1_xval_model_train_perfect_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "861406e6",
   "metadata": {},
   "source": [
    "# This notebook trains a model with cross-val on the entire test dataset. Uses the true labels as labels (instead of consensus, making this perfect model)\n",
    "- Gets pred_probs on holdout and saves results as numpy files\n",
    "- Make sure you run ``preprocess_data`` and ``create_labels_df`` on local and push/pull newest ``cifar10_test_consensus_dataset`` first"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cd34358d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ulyana/virtual/multi/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "# %load_ext autoreload\n",
    "# %autoreload 2\n",
    "\n",
    "import sys\n",
    "\n",
    "sys.path.insert(0, \"../\")\n",
    "\n",
    "from autogluon.vision import ImagePredictor, ImageDataset\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import datetime\n",
    "from pathlib import Path\n",
    "import cleanlab\n",
    "from utils.cross_validation_autogluon import cross_val_predict_autogluon_image_dataset\n",
    "\n",
    "pd.set_option('display.max_rows', None)\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7f358c4",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "52251f3a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>image</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label  image\n",
       "0      0     15\n",
       "1      1     15\n",
       "2      2     15\n",
       "3      3     15\n",
       "4      4     15\n",
       "5      5     15\n",
       "6      6     15\n",
       "7      7     15\n",
       "8      8     15\n",
       "9      9     15"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load consensus test files\n",
    "data_filepath = './data/benchmark_data/cifar10_test_dataset.csv'\n",
    "df = pd.read_csv(data_filepath)\n",
    "\n",
    "# Create mini train dataset for testing\n",
    "num_from_each_group = 15\n",
    "mini_df = df.groupby(\"label\").head(num_from_each_group)\n",
    "mini_df.groupby(\"label\")[\"image\"].count().reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f46d6ab0",
   "metadata": {},
   "source": [
    "**Model and data saving params**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f1e9ce8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save/load folders\n",
    "model_folder = './data/cifar10_truelabels' # + [model_type]\n",
    "\n",
    "# generate cross-validated predicted probabilities for various models\n",
    "models = [\n",
    "    \"swin_base_patch4_window7_224\"\n",
    "]\n",
    "\n",
    "# xvalidation parameters\n",
    "num_cv_folds = 5 # number K in stratified K-folds cross-validation\n",
    "verbose = 1 # verbose for dataloading duing crossval to numpy save\n",
    "\n",
    "# shared model parameters\n",
    "epochs = 100\n",
    "holdout_frac = 0.2\n",
    "time_limit = 21600\n",
    "random_state = 123"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78aafe39",
   "metadata": {},
   "source": [
    "## Run cross validation on `models`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3e793c24",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "modified configs(<old> != <new>): {\n",
      "root.img_cls.model   resnet101 != swin_base_patch4_window7_224\n",
      "root.train.early_stop_patience -1 != 10\n",
      "root.train.early_stop_baseline 0.0 != -inf\n",
      "root.train.early_stop_max_value 1.0 != inf\n",
      "root.train.epochs    200 != 100\n",
      "root.train.batch_size 32 != 16\n",
      "root.misc.seed       42 != 331\n",
      "root.misc.num_workers 4 != 64\n",
      "}\n",
      "Saved config to /datasets/ulyana/multiannotator_benchmarks/09373de0/.trial_0/config.yaml\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----\n",
      "Running cross-validation for model: swin_base_patch4_window7_224\n",
      "----\n",
      "Running Cross-Validation on Split: 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model swin_base_patch4_window7_224 created, param count:                                         86753474\n",
      "AMP not enabled. Training in float32.\n",
      "Disable EMA as it is not supported for now.\n",
      "Start training from [Epoch 0]\n",
      "Epoch[0] Batch [49]\tSpeed: 70.610260 samples/sec\taccuracy=0.145000\tlr=0.000100\n",
      "Epoch[0] Batch [99]\tSpeed: 103.269302 samples/sec\taccuracy=0.180625\tlr=0.000100\n",
      "Epoch[0] Batch [149]\tSpeed: 103.583590 samples/sec\taccuracy=0.235417\tlr=0.000100\n",
      "Epoch[0] Batch [199]\tSpeed: 103.142438 samples/sec\taccuracy=0.301563\tlr=0.000100\n",
      "Epoch[0] Batch [249]\tSpeed: 102.781737 samples/sec\taccuracy=0.361000\tlr=0.000100\n",
      "Epoch[0] Batch [299]\tSpeed: 102.498664 samples/sec\taccuracy=0.410625\tlr=0.000100\n",
      "Epoch[0] Batch [349]\tSpeed: 102.620982 samples/sec\taccuracy=0.455357\tlr=0.000100\n",
      "Epoch[0] Batch [399]\tSpeed: 102.558473 samples/sec\taccuracy=0.490938\tlr=0.000100\n",
      "Epoch[0] Batch [449]\tSpeed: 102.394425 samples/sec\taccuracy=0.524444\tlr=0.000100\n",
      "[Epoch 0] training: accuracy=0.524444\n",
      "[Epoch 0] speed: 97 samples/sec\ttime cost: 73.562599\n",
      "[Epoch 0] validation: top1=0.932500 top5=0.993750\n",
      "[Epoch 0] Current best top-1: 0.932500 vs previous -inf, saved to /datasets/ulyana/multiannotator_benchmarks/09373de0/.trial_0/best_checkpoint.pkl\n",
      "Epoch[1] Batch [49]\tSpeed: 99.320471 samples/sec\taccuracy=0.601250\tlr=0.003400\n",
      "Epoch[1] Batch [99]\tSpeed: 102.031794 samples/sec\taccuracy=0.665000\tlr=0.003400\n",
      "Epoch[1] Batch [149]\tSpeed: 101.902832 samples/sec\taccuracy=0.682083\tlr=0.003400\n",
      "Epoch[1] Batch [199]\tSpeed: 101.650378 samples/sec\taccuracy=0.690312\tlr=0.003400\n",
      "Epoch[1] Batch [249]\tSpeed: 101.542740 samples/sec\taccuracy=0.709000\tlr=0.003400\n",
      "Epoch[1] Batch [299]\tSpeed: 101.485009 samples/sec\taccuracy=0.719167\tlr=0.003400\n",
      "Epoch[1] Batch [349]\tSpeed: 101.511959 samples/sec\taccuracy=0.722679\tlr=0.003400\n",
      "Epoch[1] Batch [399]\tSpeed: 101.566072 samples/sec\taccuracy=0.728906\tlr=0.003400\n",
      "Epoch[1] Batch [449]\tSpeed: 101.472877 samples/sec\taccuracy=0.735694\tlr=0.003400\n",
      "[Epoch 1] training: accuracy=0.735694\n",
      "[Epoch 1] speed: 101 samples/sec\ttime cost: 71.027282\n",
      "[Epoch 1] validation: top1=0.945000 top5=1.000000\n",
      "[Epoch 1] Current best top-1: 0.945000 vs previous 0.932500, saved to /datasets/ulyana/multiannotator_benchmarks/09373de0/.trial_0/best_checkpoint.pkl\n",
      "Epoch[2] Batch [49]\tSpeed: 98.131682 samples/sec\taccuracy=0.747500\tlr=0.006700\n",
      "Epoch[2] Batch [99]\tSpeed: 101.451695 samples/sec\taccuracy=0.745625\tlr=0.006700\n",
      "Epoch[2] Batch [149]\tSpeed: 101.587437 samples/sec\taccuracy=0.751250\tlr=0.006700\n",
      "Epoch[2] Batch [199]\tSpeed: 101.173387 samples/sec\taccuracy=0.753750\tlr=0.006700\n",
      "Epoch[2] Batch [249]\tSpeed: 101.127737 samples/sec\taccuracy=0.747000\tlr=0.006700\n",
      "Epoch[2] Batch [299]\tSpeed: 101.047314 samples/sec\taccuracy=0.747292\tlr=0.006700\n",
      "Epoch[2] Batch [349]\tSpeed: 101.209445 samples/sec\taccuracy=0.744107\tlr=0.006700\n",
      "Epoch[2] Batch [399]\tSpeed: 101.128472 samples/sec\taccuracy=0.748437\tlr=0.006700\n",
      "Epoch[2] Batch [449]\tSpeed: 101.086544 samples/sec\taccuracy=0.752778\tlr=0.006700\n",
      "[Epoch 2] training: accuracy=0.752778\n",
      "[Epoch 2] speed: 100 samples/sec\ttime cost: 71.385689\n",
      "[Epoch 2] validation: top1=0.927500 top5=0.998750\n",
      "Epoch[3] Batch [49]\tSpeed: 97.550787 samples/sec\taccuracy=0.760000\tlr=0.010000\n",
      "Epoch[3] Batch [99]\tSpeed: 101.049575 samples/sec\taccuracy=0.723125\tlr=0.010000\n",
      "Epoch[3] Batch [149]\tSpeed: 101.238611 samples/sec\taccuracy=0.712917\tlr=0.010000\n",
      "Epoch[3] Batch [199]\tSpeed: 100.939030 samples/sec\taccuracy=0.712812\tlr=0.010000\n",
      "Epoch[3] Batch [249]\tSpeed: 100.955008 samples/sec\taccuracy=0.714000\tlr=0.010000\n",
      "Epoch[3] Batch [299]\tSpeed: 100.899490 samples/sec\taccuracy=0.716875\tlr=0.010000\n",
      "Epoch[3] Batch [349]\tSpeed: 101.058995 samples/sec\taccuracy=0.721786\tlr=0.010000\n",
      "Epoch[3] Batch [399]\tSpeed: 101.095550 samples/sec\taccuracy=0.725156\tlr=0.010000\n",
      "Epoch[3] Batch [449]\tSpeed: 100.935721 samples/sec\taccuracy=0.724167\tlr=0.010000\n",
      "[Epoch 3] training: accuracy=0.724167\n",
      "[Epoch 3] speed: 100 samples/sec\ttime cost: 71.562165\n",
      "[Epoch 3] validation: top1=0.860000 top5=0.996250\n",
      "Epoch[4] Batch [49]\tSpeed: 97.757062 samples/sec\taccuracy=0.768750\tlr=0.010000\n",
      "Epoch[4] Batch [99]\tSpeed: 101.055004 samples/sec\taccuracy=0.773750\tlr=0.010000\n",
      "Epoch[4] Batch [149]\tSpeed: 101.216745 samples/sec\taccuracy=0.766250\tlr=0.010000\n",
      "Epoch[4] Batch [199]\tSpeed: 100.859098 samples/sec\taccuracy=0.772188\tlr=0.010000\n",
      "Epoch[4] Batch [249]\tSpeed: 100.882047 samples/sec\taccuracy=0.768500\tlr=0.010000\n",
      "Epoch[4] Batch [299]\tSpeed: 100.899526 samples/sec\taccuracy=0.772917\tlr=0.010000\n",
      "Epoch[4] Batch [349]\tSpeed: 100.963331 samples/sec\taccuracy=0.775179\tlr=0.010000\n",
      "Epoch[4] Batch [399]\tSpeed: 101.026602 samples/sec\taccuracy=0.776250\tlr=0.010000\n",
      "Epoch[4] Batch [449]\tSpeed: 100.913521 samples/sec\taccuracy=0.778333\tlr=0.010000\n",
      "[Epoch 4] training: accuracy=0.778333\n",
      "[Epoch 4] speed: 100 samples/sec\ttime cost: 71.572757\n",
      "[Epoch 4] validation: top1=0.933750 top5=1.000000\n",
      "Epoch[5] Batch [49]\tSpeed: 97.321489 samples/sec\taccuracy=0.816250\tlr=0.010000\n",
      "Epoch[5] Batch [99]\tSpeed: 100.845909 samples/sec\taccuracy=0.794375\tlr=0.010000\n",
      "Epoch[5] Batch [149]\tSpeed: 101.058976 samples/sec\taccuracy=0.796250\tlr=0.010000\n",
      "Epoch[5] Batch [199]\tSpeed: 100.812199 samples/sec\taccuracy=0.797500\tlr=0.010000\n",
      "Epoch[5] Batch [249]\tSpeed: 100.813710 samples/sec\taccuracy=0.801750\tlr=0.010000\n",
      "Epoch[5] Batch [299]\tSpeed: 100.781275 samples/sec\taccuracy=0.800417\tlr=0.010000\n",
      "Epoch[5] Batch [349]\tSpeed: 100.936628 samples/sec\taccuracy=0.798393\tlr=0.010000\n",
      "Epoch[5] Batch [399]\tSpeed: 100.947554 samples/sec\taccuracy=0.798750\tlr=0.010000\n",
      "Epoch[5] Batch [449]\tSpeed: 100.918016 samples/sec\taccuracy=0.800694\tlr=0.010000\n",
      "[Epoch 5] training: accuracy=0.800694\n",
      "[Epoch 5] speed: 100 samples/sec\ttime cost: 71.664692\n",
      "[Epoch 5] validation: top1=0.921250 top5=0.998750\n",
      "Epoch[6] Batch [49]\tSpeed: 98.528424 samples/sec\taccuracy=0.836250\tlr=0.010000\n",
      "Epoch[6] Batch [99]\tSpeed: 100.880181 samples/sec\taccuracy=0.816250\tlr=0.010000\n",
      "Epoch[6] Batch [149]\tSpeed: 101.080576 samples/sec\taccuracy=0.819583\tlr=0.010000\n",
      "Epoch[6] Batch [199]\tSpeed: 100.863840 samples/sec\taccuracy=0.823438\tlr=0.010000\n",
      "Epoch[6] Batch [249]\tSpeed: 100.915521 samples/sec\taccuracy=0.786500\tlr=0.010000\n",
      "Epoch[6] Batch [299]\tSpeed: 100.849791 samples/sec\taccuracy=0.772500\tlr=0.010000\n",
      "Epoch[6] Batch [349]\tSpeed: 100.983014 samples/sec\taccuracy=0.768929\tlr=0.010000\n",
      "Epoch[6] Batch [399]\tSpeed: 101.035959 samples/sec\taccuracy=0.767031\tlr=0.010000\n",
      "Epoch[6] Batch [449]\tSpeed: 101.025467 samples/sec\taccuracy=0.768056\tlr=0.010000\n",
      "[Epoch 6] training: accuracy=0.768056\n",
      "[Epoch 6] speed: 100 samples/sec\ttime cost: 71.523068\n",
      "[Epoch 6] validation: top1=0.895000 top5=0.997500\n",
      "Epoch[7] Batch [49]\tSpeed: 98.342542 samples/sec\taccuracy=0.807500\tlr=0.010000\n",
      "Epoch[7] Batch [99]\tSpeed: 100.997030 samples/sec\taccuracy=0.796875\tlr=0.010000\n",
      "Epoch[7] Batch [149]\tSpeed: 101.097558 samples/sec\taccuracy=0.799583\tlr=0.010000\n",
      "Epoch[7] Batch [199]\tSpeed: 100.780297 samples/sec\taccuracy=0.798125\tlr=0.010000\n",
      "Epoch[7] Batch [249]\tSpeed: 100.874595 samples/sec\taccuracy=0.797500\tlr=0.010000\n",
      "Epoch[7] Batch [299]\tSpeed: 100.866853 samples/sec\taccuracy=0.798958\tlr=0.010000\n",
      "Epoch[7] Batch [349]\tSpeed: 100.922833 samples/sec\taccuracy=0.799286\tlr=0.010000\n",
      "Epoch[7] Batch [399]\tSpeed: 100.978082 samples/sec\taccuracy=0.800781\tlr=0.010000\n",
      "Epoch[7] Batch [449]\tSpeed: 100.916086 samples/sec\taccuracy=0.802639\tlr=0.010000\n",
      "[Epoch 7] training: accuracy=0.802639\n",
      "[Epoch 7] speed: 100 samples/sec\ttime cost: 71.554122\n",
      "[Epoch 7] validation: top1=0.896250 top5=0.993750\n",
      "Epoch[8] Batch [49]\tSpeed: 97.309576 samples/sec\taccuracy=0.807500\tlr=0.010000\n",
      "Epoch[8] Batch [99]\tSpeed: 100.909385 samples/sec\taccuracy=0.821250\tlr=0.010000\n",
      "Epoch[8] Batch [149]\tSpeed: 101.192240 samples/sec\taccuracy=0.823333\tlr=0.010000\n",
      "Epoch[8] Batch [199]\tSpeed: 100.857712 samples/sec\taccuracy=0.830000\tlr=0.010000\n",
      "Epoch[8] Batch [249]\tSpeed: 100.867751 samples/sec\taccuracy=0.832000\tlr=0.010000\n",
      "Epoch[8] Batch [299]\tSpeed: 100.863815 samples/sec\taccuracy=0.831250\tlr=0.010000\n",
      "Epoch[8] Batch [349]\tSpeed: 100.950652 samples/sec\taccuracy=0.833929\tlr=0.010000\n",
      "Epoch[8] Batch [399]\tSpeed: 101.009945 samples/sec\taccuracy=0.833438\tlr=0.010000\n",
      "Epoch[8] Batch [449]\tSpeed: 100.936750 samples/sec\taccuracy=0.831389\tlr=0.010000\n",
      "[Epoch 8] training: accuracy=0.831389\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Epoch 8] speed: 100 samples/sec\ttime cost: 71.628242\n",
      "[Epoch 8] validation: top1=0.931250 top5=0.996250\n",
      "Epoch[9] Batch [49]\tSpeed: 97.604310 samples/sec\taccuracy=0.837500\tlr=0.010000\n",
      "Epoch[9] Batch [99]\tSpeed: 101.038402 samples/sec\taccuracy=0.836250\tlr=0.010000\n",
      "Epoch[9] Batch [149]\tSpeed: 101.202269 samples/sec\taccuracy=0.840000\tlr=0.010000\n",
      "Epoch[9] Batch [199]\tSpeed: 100.852692 samples/sec\taccuracy=0.832812\tlr=0.010000\n",
      "Epoch[9] Batch [249]\tSpeed: 100.828572 samples/sec\taccuracy=0.834500\tlr=0.010000\n",
      "Epoch[9] Batch [299]\tSpeed: 100.762317 samples/sec\taccuracy=0.835833\tlr=0.010000\n",
      "Epoch[9] Batch [349]\tSpeed: 101.006904 samples/sec\taccuracy=0.832321\tlr=0.010000\n",
      "Epoch[9] Batch [399]\tSpeed: 101.050747 samples/sec\taccuracy=0.833750\tlr=0.010000\n",
      "Epoch[9] Batch [449]\tSpeed: 100.985984 samples/sec\taccuracy=0.835417\tlr=0.010000\n",
      "[Epoch 9] training: accuracy=0.835417\n",
      "[Epoch 9] speed: 100 samples/sec\ttime cost: 71.592460\n",
      "[Epoch 9] validation: top1=0.936250 top5=0.996250\n",
      "Epoch[10] Batch [49]\tSpeed: 97.908910 samples/sec\taccuracy=0.840000\tlr=0.010000\n",
      "Epoch[10] Batch [99]\tSpeed: 100.981589 samples/sec\taccuracy=0.858125\tlr=0.010000\n",
      "Epoch[10] Batch [149]\tSpeed: 101.204302 samples/sec\taccuracy=0.853750\tlr=0.010000\n",
      "Epoch[10] Batch [199]\tSpeed: 100.710928 samples/sec\taccuracy=0.848125\tlr=0.010000\n",
      "Epoch[10] Batch [249]\tSpeed: 100.894241 samples/sec\taccuracy=0.847000\tlr=0.010000\n",
      "Epoch[10] Batch [299]\tSpeed: 100.836238 samples/sec\taccuracy=0.846250\tlr=0.010000\n",
      "Epoch[10] Batch [349]\tSpeed: 100.934069 samples/sec\taccuracy=0.845357\tlr=0.010000\n",
      "Epoch[10] Batch [399]\tSpeed: 101.072017 samples/sec\taccuracy=0.845313\tlr=0.010000\n",
      "Epoch[10] Batch [449]\tSpeed: 101.043553 samples/sec\taccuracy=0.845000\tlr=0.010000\n",
      "[Epoch 10] training: accuracy=0.845000\n",
      "[Epoch 10] speed: 100 samples/sec\ttime cost: 71.570966\n",
      "[Epoch 10] validation: top1=0.931250 top5=0.998750\n",
      "Epoch[11] Batch [49]\tSpeed: 97.848980 samples/sec\taccuracy=0.862500\tlr=0.010000\n",
      "Epoch[11] Batch [99]\tSpeed: 101.024062 samples/sec\taccuracy=0.858125\tlr=0.010000\n",
      "Epoch[11] Batch [149]\tSpeed: 101.188374 samples/sec\taccuracy=0.859583\tlr=0.010000\n",
      "Epoch[11] Batch [199]\tSpeed: 100.951187 samples/sec\taccuracy=0.850625\tlr=0.010000\n",
      "Epoch[11] Batch [249]\tSpeed: 100.964950 samples/sec\taccuracy=0.849750\tlr=0.010000\n",
      "Epoch[11] Batch [299]\tSpeed: 100.878583 samples/sec\taccuracy=0.851042\tlr=0.010000\n",
      "Epoch[11] Batch [349]\tSpeed: 101.024832 samples/sec\taccuracy=0.849107\tlr=0.010000\n",
      "Epoch[11] Batch [399]\tSpeed: 100.982960 samples/sec\taccuracy=0.847344\tlr=0.010000\n",
      "Epoch[11] Batch [449]\tSpeed: 100.932718 samples/sec\taccuracy=0.846111\tlr=0.010000\n",
      "[Epoch 11] training: accuracy=0.846111\n",
      "[Epoch 11] speed: 100 samples/sec\ttime cost: 71.554585\n",
      "[Epoch 11] validation: top1=0.920000 top5=0.997500\n",
      "[Epoch 12] EarlyStop after 10 epochs: no better than 0.945\n",
      "Applying the state from the best checkpoint...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Folder ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_0/ already exists!\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_0/_test_pred_probs_split_0\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_0/_test_pred_features_split_0\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_0/_test_labels_split_0\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_0/_test_image_files_split_0\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_0/_test_indices_split_0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "modified configs(<old> != <new>): {\n",
      "root.img_cls.model   resnet101 != swin_base_patch4_window7_224\n",
      "root.train.early_stop_patience -1 != 10\n",
      "root.train.early_stop_baseline 0.0 != -inf\n",
      "root.train.early_stop_max_value 1.0 != inf\n",
      "root.train.epochs    200 != 100\n",
      "root.train.batch_size 32 != 16\n",
      "root.misc.seed       42 != 468\n",
      "root.misc.num_workers 4 != 64\n",
      "}\n",
      "Saved config to /datasets/ulyana/multiannotator_benchmarks/e5f44713/.trial_0/config.yaml\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----\n",
      "Running Cross-Validation on Split: 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model swin_base_patch4_window7_224 created, param count:                                         86753474\n",
      "AMP not enabled. Training in float32.\n",
      "Disable EMA as it is not supported for now.\n",
      "Start training from [Epoch 0]\n",
      "Epoch[0] Batch [49]\tSpeed: 72.825903 samples/sec\taccuracy=0.126250\tlr=0.000100\n",
      "Epoch[0] Batch [99]\tSpeed: 101.462994 samples/sec\taccuracy=0.201875\tlr=0.000100\n",
      "Epoch[0] Batch [149]\tSpeed: 101.430170 samples/sec\taccuracy=0.294583\tlr=0.000100\n",
      "Epoch[0] Batch [199]\tSpeed: 101.270042 samples/sec\taccuracy=0.362812\tlr=0.000100\n",
      "Epoch[0] Batch [249]\tSpeed: 101.161101 samples/sec\taccuracy=0.416500\tlr=0.000100\n",
      "Epoch[0] Batch [299]\tSpeed: 101.064580 samples/sec\taccuracy=0.461458\tlr=0.000100\n",
      "Epoch[0] Batch [349]\tSpeed: 101.052430 samples/sec\taccuracy=0.497321\tlr=0.000100\n",
      "Epoch[0] Batch [399]\tSpeed: 101.050762 samples/sec\taccuracy=0.527500\tlr=0.000100\n",
      "Epoch[0] Batch [449]\tSpeed: 101.013205 samples/sec\taccuracy=0.555833\tlr=0.000100\n",
      "[Epoch 0] training: accuracy=0.555833\n",
      "[Epoch 0] speed: 96 samples/sec\ttime cost: 74.243153\n",
      "[Epoch 0] validation: top1=0.927500 top5=0.997500\n",
      "[Epoch 0] Current best top-1: 0.927500 vs previous -inf, saved to /datasets/ulyana/multiannotator_benchmarks/e5f44713/.trial_0/best_checkpoint.pkl\n",
      "Epoch[1] Batch [49]\tSpeed: 97.989120 samples/sec\taccuracy=0.571250\tlr=0.003400\n",
      "Epoch[1] Batch [99]\tSpeed: 101.264844 samples/sec\taccuracy=0.544375\tlr=0.003400\n",
      "Epoch[1] Batch [149]\tSpeed: 101.282704 samples/sec\taccuracy=0.593333\tlr=0.003400\n",
      "Epoch[1] Batch [199]\tSpeed: 100.870610 samples/sec\taccuracy=0.619375\tlr=0.003400\n",
      "Epoch[1] Batch [249]\tSpeed: 100.739982 samples/sec\taccuracy=0.646750\tlr=0.003400\n",
      "Epoch[1] Batch [299]\tSpeed: 100.855727 samples/sec\taccuracy=0.670000\tlr=0.003400\n",
      "Epoch[1] Batch [349]\tSpeed: 100.930201 samples/sec\taccuracy=0.682321\tlr=0.003400\n",
      "Epoch[1] Batch [399]\tSpeed: 101.013761 samples/sec\taccuracy=0.692500\tlr=0.003400\n",
      "Epoch[1] Batch [449]\tSpeed: 100.956645 samples/sec\taccuracy=0.701806\tlr=0.003400\n",
      "[Epoch 1] training: accuracy=0.701806\n",
      "[Epoch 1] speed: 100 samples/sec\ttime cost: 71.545749\n",
      "[Epoch 1] validation: top1=0.950000 top5=0.997500\n",
      "[Epoch 1] Current best top-1: 0.950000 vs previous 0.927500, saved to /datasets/ulyana/multiannotator_benchmarks/e5f44713/.trial_0/best_checkpoint.pkl\n",
      "Epoch[2] Batch [49]\tSpeed: 97.221541 samples/sec\taccuracy=0.765000\tlr=0.006700\n",
      "Epoch[2] Batch [99]\tSpeed: 101.039583 samples/sec\taccuracy=0.753750\tlr=0.006700\n",
      "Epoch[2] Batch [149]\tSpeed: 101.117287 samples/sec\taccuracy=0.742083\tlr=0.006700\n",
      "Epoch[2] Batch [199]\tSpeed: 100.864137 samples/sec\taccuracy=0.744687\tlr=0.006700\n",
      "Epoch[2] Batch [249]\tSpeed: 100.868451 samples/sec\taccuracy=0.740500\tlr=0.006700\n",
      "Epoch[2] Batch [299]\tSpeed: 100.914186 samples/sec\taccuracy=0.744167\tlr=0.006700\n",
      "Epoch[2] Batch [349]\tSpeed: 100.916480 samples/sec\taccuracy=0.743036\tlr=0.006700\n",
      "Epoch[2] Batch [399]\tSpeed: 101.000641 samples/sec\taccuracy=0.748594\tlr=0.006700\n",
      "Epoch[2] Batch [449]\tSpeed: 100.924057 samples/sec\taccuracy=0.751944\tlr=0.006700\n",
      "[Epoch 2] training: accuracy=0.751944\n",
      "[Epoch 2] speed: 100 samples/sec\ttime cost: 71.631206\n",
      "[Epoch 2] validation: top1=0.927500 top5=0.997500\n",
      "Epoch[3] Batch [49]\tSpeed: 97.134221 samples/sec\taccuracy=0.748750\tlr=0.010000\n",
      "Epoch[3] Batch [99]\tSpeed: 101.008549 samples/sec\taccuracy=0.741875\tlr=0.010000\n",
      "Epoch[3] Batch [149]\tSpeed: 101.103686 samples/sec\taccuracy=0.729583\tlr=0.010000\n",
      "Epoch[3] Batch [199]\tSpeed: 100.904387 samples/sec\taccuracy=0.735625\tlr=0.010000\n",
      "Epoch[3] Batch [249]\tSpeed: 100.752287 samples/sec\taccuracy=0.741500\tlr=0.010000\n",
      "Epoch[3] Batch [299]\tSpeed: 100.806708 samples/sec\taccuracy=0.742708\tlr=0.010000\n",
      "Epoch[3] Batch [349]\tSpeed: 100.853211 samples/sec\taccuracy=0.744107\tlr=0.010000\n",
      "Epoch[3] Batch [399]\tSpeed: 101.002301 samples/sec\taccuracy=0.743594\tlr=0.010000\n",
      "Epoch[3] Batch [449]\tSpeed: 100.925347 samples/sec\taccuracy=0.744306\tlr=0.010000\n",
      "[Epoch 3] training: accuracy=0.744306\n",
      "[Epoch 3] speed: 100 samples/sec\ttime cost: 71.661402\n",
      "[Epoch 3] validation: top1=0.901250 top5=0.996250\n",
      "Epoch[4] Batch [49]\tSpeed: 97.539940 samples/sec\taccuracy=0.797500\tlr=0.010000\n",
      "Epoch[4] Batch [99]\tSpeed: 101.027353 samples/sec\taccuracy=0.793750\tlr=0.010000\n",
      "Epoch[4] Batch [149]\tSpeed: 101.152739 samples/sec\taccuracy=0.796667\tlr=0.010000\n",
      "Epoch[4] Batch [199]\tSpeed: 100.954054 samples/sec\taccuracy=0.792813\tlr=0.010000\n",
      "Epoch[4] Batch [249]\tSpeed: 100.934573 samples/sec\taccuracy=0.790250\tlr=0.010000\n",
      "Epoch[4] Batch [299]\tSpeed: 100.901492 samples/sec\taccuracy=0.785208\tlr=0.010000\n",
      "Epoch[4] Batch [349]\tSpeed: 101.080451 samples/sec\taccuracy=0.788929\tlr=0.010000\n",
      "Epoch[4] Batch [399]\tSpeed: 101.076085 samples/sec\taccuracy=0.790000\tlr=0.010000\n",
      "Epoch[4] Batch [449]\tSpeed: 101.070985 samples/sec\taccuracy=0.789028\tlr=0.010000\n",
      "[Epoch 4] training: accuracy=0.789028\n",
      "[Epoch 4] speed: 100 samples/sec\ttime cost: 71.561242\n",
      "[Epoch 4] validation: top1=0.926250 top5=1.000000\n",
      "Epoch[5] Batch [49]\tSpeed: 96.643448 samples/sec\taccuracy=0.827500\tlr=0.010000\n",
      "Epoch[5] Batch [99]\tSpeed: 101.015175 samples/sec\taccuracy=0.810625\tlr=0.010000\n",
      "Epoch[5] Batch [149]\tSpeed: 101.120731 samples/sec\taccuracy=0.807500\tlr=0.010000\n",
      "Epoch[5] Batch [199]\tSpeed: 100.851683 samples/sec\taccuracy=0.801250\tlr=0.010000\n",
      "Epoch[5] Batch [249]\tSpeed: 100.857567 samples/sec\taccuracy=0.798750\tlr=0.010000\n",
      "Epoch[5] Batch [299]\tSpeed: 100.869616 samples/sec\taccuracy=0.798542\tlr=0.010000\n",
      "Epoch[5] Batch [349]\tSpeed: 100.966588 samples/sec\taccuracy=0.801429\tlr=0.010000\n",
      "Epoch[5] Batch [399]\tSpeed: 101.028503 samples/sec\taccuracy=0.800781\tlr=0.010000\n",
      "Epoch[5] Batch [449]\tSpeed: 100.960484 samples/sec\taccuracy=0.798472\tlr=0.010000\n",
      "[Epoch 5] training: accuracy=0.798472\n",
      "[Epoch 5] speed: 100 samples/sec\ttime cost: 71.678699\n",
      "[Epoch 5] validation: top1=0.910000 top5=0.995000\n",
      "Epoch[6] Batch [49]\tSpeed: 96.738693 samples/sec\taccuracy=0.807500\tlr=0.010000\n",
      "Epoch[6] Batch [99]\tSpeed: 100.924360 samples/sec\taccuracy=0.815000\tlr=0.010000\n",
      "Epoch[6] Batch [149]\tSpeed: 101.147345 samples/sec\taccuracy=0.825417\tlr=0.010000\n",
      "Epoch[6] Batch [199]\tSpeed: 100.823837 samples/sec\taccuracy=0.823125\tlr=0.010000\n",
      "Epoch[6] Batch [249]\tSpeed: 100.866738 samples/sec\taccuracy=0.824250\tlr=0.010000\n",
      "Epoch[6] Batch [299]\tSpeed: 100.798895 samples/sec\taccuracy=0.824167\tlr=0.010000\n",
      "Epoch[6] Batch [349]\tSpeed: 100.968827 samples/sec\taccuracy=0.825000\tlr=0.010000\n",
      "Epoch[6] Batch [399]\tSpeed: 101.036151 samples/sec\taccuracy=0.821562\tlr=0.010000\n",
      "Epoch[6] Batch [449]\tSpeed: 100.965703 samples/sec\taccuracy=0.821806\tlr=0.010000\n",
      "[Epoch 6] training: accuracy=0.821806\n",
      "[Epoch 6] speed: 100 samples/sec\ttime cost: 71.681423\n",
      "[Epoch 6] validation: top1=0.940000 top5=1.000000\n",
      "Epoch[7] Batch [49]\tSpeed: 97.579646 samples/sec\taccuracy=0.825000\tlr=0.010000\n",
      "Epoch[7] Batch [99]\tSpeed: 100.957419 samples/sec\taccuracy=0.820000\tlr=0.010000\n",
      "Epoch[7] Batch [149]\tSpeed: 101.084973 samples/sec\taccuracy=0.823750\tlr=0.010000\n",
      "Epoch[7] Batch [199]\tSpeed: 100.818444 samples/sec\taccuracy=0.825000\tlr=0.010000\n",
      "Epoch[7] Batch [249]\tSpeed: 100.910031 samples/sec\taccuracy=0.831000\tlr=0.010000\n",
      "Epoch[7] Batch [299]\tSpeed: 100.699074 samples/sec\taccuracy=0.830417\tlr=0.010000\n",
      "Epoch[7] Batch [349]\tSpeed: 101.003906 samples/sec\taccuracy=0.826964\tlr=0.010000\n",
      "Epoch[7] Batch [399]\tSpeed: 101.094131 samples/sec\taccuracy=0.830156\tlr=0.010000\n",
      "Epoch[7] Batch [449]\tSpeed: 100.974141 samples/sec\taccuracy=0.833750\tlr=0.010000\n",
      "[Epoch 7] training: accuracy=0.833750\n",
      "[Epoch 7] speed: 100 samples/sec\ttime cost: 71.608811\n",
      "[Epoch 7] validation: top1=0.937500 top5=0.998750\n",
      "Epoch[8] Batch [49]\tSpeed: 97.303258 samples/sec\taccuracy=0.828750\tlr=0.010000\n",
      "Epoch[8] Batch [99]\tSpeed: 101.054937 samples/sec\taccuracy=0.834375\tlr=0.010000\n",
      "Epoch[8] Batch [149]\tSpeed: 101.135842 samples/sec\taccuracy=0.835417\tlr=0.010000\n",
      "Epoch[8] Batch [199]\tSpeed: 100.907937 samples/sec\taccuracy=0.832812\tlr=0.010000\n",
      "Epoch[8] Batch [249]\tSpeed: 100.891238 samples/sec\taccuracy=0.829500\tlr=0.010000\n",
      "Epoch[8] Batch [299]\tSpeed: 100.756384 samples/sec\taccuracy=0.829375\tlr=0.010000\n",
      "Epoch[8] Batch [349]\tSpeed: 100.895224 samples/sec\taccuracy=0.829464\tlr=0.010000\n",
      "Epoch[8] Batch [399]\tSpeed: 100.955579 samples/sec\taccuracy=0.823438\tlr=0.010000\n",
      "Epoch[8] Batch [449]\tSpeed: 100.886633 samples/sec\taccuracy=0.818472\tlr=0.010000\n",
      "[Epoch 8] training: accuracy=0.818472\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Epoch 8] speed: 100 samples/sec\ttime cost: 71.636907\n",
      "[Epoch 8] validation: top1=0.916250 top5=0.991250\n",
      "Epoch[9] Batch [49]\tSpeed: 96.963370 samples/sec\taccuracy=0.803750\tlr=0.010000\n",
      "Epoch[9] Batch [99]\tSpeed: 100.956447 samples/sec\taccuracy=0.816875\tlr=0.010000\n",
      "Epoch[9] Batch [149]\tSpeed: 101.111806 samples/sec\taccuracy=0.824583\tlr=0.010000\n",
      "Epoch[9] Batch [199]\tSpeed: 100.886991 samples/sec\taccuracy=0.824063\tlr=0.010000\n",
      "Epoch[9] Batch [249]\tSpeed: 100.929478 samples/sec\taccuracy=0.784000\tlr=0.010000\n",
      "Epoch[9] Batch [299]\tSpeed: 100.893746 samples/sec\taccuracy=0.683542\tlr=0.010000\n",
      "Epoch[9] Batch [349]\tSpeed: 100.887346 samples/sec\taccuracy=0.620714\tlr=0.010000\n",
      "Epoch[9] Batch [399]\tSpeed: 101.032558 samples/sec\taccuracy=0.577812\tlr=0.010000\n",
      "Epoch[9] Batch [449]\tSpeed: 100.957769 samples/sec\taccuracy=0.544722\tlr=0.010000\n",
      "[Epoch 9] training: accuracy=0.544722\n",
      "[Epoch 9] speed: 100 samples/sec\ttime cost: 71.652579\n",
      "[Epoch 9] validation: top1=0.413750 top5=0.866250\n",
      "Epoch[10] Batch [49]\tSpeed: 96.499826 samples/sec\taccuracy=0.377500\tlr=0.010000\n",
      "Epoch[10] Batch [99]\tSpeed: 100.810978 samples/sec\taccuracy=0.393125\tlr=0.010000\n",
      "Epoch[10] Batch [149]\tSpeed: 100.976247 samples/sec\taccuracy=0.411250\tlr=0.010000\n",
      "Epoch[10] Batch [199]\tSpeed: 100.684800 samples/sec\taccuracy=0.423438\tlr=0.010000\n",
      "Epoch[10] Batch [249]\tSpeed: 100.727263 samples/sec\taccuracy=0.430500\tlr=0.010000\n",
      "Epoch[10] Batch [299]\tSpeed: 100.734248 samples/sec\taccuracy=0.447917\tlr=0.010000\n",
      "Epoch[10] Batch [349]\tSpeed: 100.833850 samples/sec\taccuracy=0.456786\tlr=0.010000\n",
      "Epoch[10] Batch [399]\tSpeed: 100.996191 samples/sec\taccuracy=0.472813\tlr=0.010000\n",
      "Epoch[10] Batch [449]\tSpeed: 100.895961 samples/sec\taccuracy=0.484167\tlr=0.010000\n",
      "[Epoch 10] training: accuracy=0.484167\n",
      "[Epoch 10] speed: 100 samples/sec\ttime cost: 71.770640\n",
      "[Epoch 10] validation: top1=0.753750 top5=0.972500\n",
      "Epoch[11] Batch [49]\tSpeed: 97.198585 samples/sec\taccuracy=0.595000\tlr=0.010000\n",
      "Epoch[11] Batch [99]\tSpeed: 100.871735 samples/sec\taccuracy=0.619375\tlr=0.010000\n",
      "Epoch[11] Batch [149]\tSpeed: 101.154748 samples/sec\taccuracy=0.634167\tlr=0.010000\n",
      "Epoch[11] Batch [199]\tSpeed: 100.847606 samples/sec\taccuracy=0.639687\tlr=0.010000\n",
      "Epoch[11] Batch [249]\tSpeed: 100.936856 samples/sec\taccuracy=0.648000\tlr=0.010000\n",
      "Epoch[11] Batch [299]\tSpeed: 100.744447 samples/sec\taccuracy=0.663958\tlr=0.010000\n",
      "Epoch[11] Batch [349]\tSpeed: 100.981045 samples/sec\taccuracy=0.669107\tlr=0.010000\n",
      "Epoch[11] Batch [399]\tSpeed: 101.010745 samples/sec\taccuracy=0.674687\tlr=0.010000\n",
      "Epoch[11] Batch [449]\tSpeed: 100.940785 samples/sec\taccuracy=0.675694\tlr=0.010000\n",
      "[Epoch 11] training: accuracy=0.675694\n",
      "[Epoch 11] speed: 100 samples/sec\ttime cost: 71.645767\n",
      "[Epoch 11] validation: top1=0.852500 top5=0.993750\n",
      "[Epoch 12] EarlyStop after 10 epochs: no better than 0.95\n",
      "Applying the state from the best checkpoint...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Folder ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_1/ already exists!\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_1/_test_pred_probs_split_1\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_1/_test_pred_features_split_1\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_1/_test_labels_split_1\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_1/_test_image_files_split_1\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_1/_test_indices_split_1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "modified configs(<old> != <new>): {\n",
      "root.img_cls.model   resnet101 != swin_base_patch4_window7_224\n",
      "root.train.early_stop_patience -1 != 10\n",
      "root.train.early_stop_baseline 0.0 != -inf\n",
      "root.train.early_stop_max_value 1.0 != inf\n",
      "root.train.epochs    200 != 100\n",
      "root.train.batch_size 32 != 16\n",
      "root.misc.seed       42 != 525\n",
      "root.misc.num_workers 4 != 64\n",
      "}\n",
      "Saved config to /datasets/ulyana/multiannotator_benchmarks/01ea71e9/.trial_0/config.yaml\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----\n",
      "Running Cross-Validation on Split: 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model swin_base_patch4_window7_224 created, param count:                                         86753474\n",
      "AMP not enabled. Training in float32.\n",
      "Disable EMA as it is not supported for now.\n",
      "Start training from [Epoch 0]\n",
      "Epoch[0] Batch [49]\tSpeed: 72.772137 samples/sec\taccuracy=0.130000\tlr=0.000100\n",
      "Epoch[0] Batch [99]\tSpeed: 101.578924 samples/sec\taccuracy=0.210625\tlr=0.000100\n",
      "Epoch[0] Batch [149]\tSpeed: 101.384438 samples/sec\taccuracy=0.283750\tlr=0.000100\n",
      "Epoch[0] Batch [199]\tSpeed: 101.152156 samples/sec\taccuracy=0.345313\tlr=0.000100\n",
      "Epoch[0] Batch [249]\tSpeed: 101.005901 samples/sec\taccuracy=0.396500\tlr=0.000100\n",
      "Epoch[0] Batch [299]\tSpeed: 100.903653 samples/sec\taccuracy=0.450417\tlr=0.000100\n",
      "Epoch[0] Batch [349]\tSpeed: 101.004332 samples/sec\taccuracy=0.496964\tlr=0.000100\n",
      "Epoch[0] Batch [399]\tSpeed: 101.157670 samples/sec\taccuracy=0.532031\tlr=0.000100\n",
      "Epoch[0] Batch [449]\tSpeed: 101.045044 samples/sec\taccuracy=0.563194\tlr=0.000100\n",
      "[Epoch 0] training: accuracy=0.563194\n",
      "[Epoch 0] speed: 96 samples/sec\ttime cost: 74.272612\n",
      "[Epoch 0] validation: top1=0.938750 top5=0.997500\n",
      "[Epoch 0] Current best top-1: 0.938750 vs previous -inf, saved to /datasets/ulyana/multiannotator_benchmarks/01ea71e9/.trial_0/best_checkpoint.pkl\n",
      "Epoch[1] Batch [49]\tSpeed: 98.707564 samples/sec\taccuracy=0.636250\tlr=0.003400\n",
      "Epoch[1] Batch [99]\tSpeed: 101.247231 samples/sec\taccuracy=0.668750\tlr=0.003400\n",
      "Epoch[1] Batch [149]\tSpeed: 101.289965 samples/sec\taccuracy=0.661667\tlr=0.003400\n",
      "Epoch[1] Batch [199]\tSpeed: 100.979307 samples/sec\taccuracy=0.682187\tlr=0.003400\n",
      "Epoch[1] Batch [249]\tSpeed: 100.987719 samples/sec\taccuracy=0.696750\tlr=0.003400\n",
      "Epoch[1] Batch [299]\tSpeed: 100.969273 samples/sec\taccuracy=0.712917\tlr=0.003400\n",
      "Epoch[1] Batch [349]\tSpeed: 101.021921 samples/sec\taccuracy=0.722321\tlr=0.003400\n",
      "Epoch[1] Batch [399]\tSpeed: 101.031566 samples/sec\taccuracy=0.731719\tlr=0.003400\n",
      "Epoch[1] Batch [449]\tSpeed: 101.025701 samples/sec\taccuracy=0.735972\tlr=0.003400\n",
      "[Epoch 1] training: accuracy=0.735972\n",
      "[Epoch 1] speed: 100 samples/sec\ttime cost: 71.436686\n",
      "[Epoch 1] validation: top1=0.921250 top5=0.998750\n",
      "Epoch[2] Batch [49]\tSpeed: 97.103008 samples/sec\taccuracy=0.730000\tlr=0.006700\n",
      "Epoch[2] Batch [99]\tSpeed: 101.018277 samples/sec\taccuracy=0.744375\tlr=0.006700\n",
      "Epoch[2] Batch [149]\tSpeed: 101.145125 samples/sec\taccuracy=0.741667\tlr=0.006700\n",
      "Epoch[2] Batch [199]\tSpeed: 100.866881 samples/sec\taccuracy=0.736563\tlr=0.006700\n",
      "Epoch[2] Batch [249]\tSpeed: 100.827933 samples/sec\taccuracy=0.736750\tlr=0.006700\n",
      "Epoch[2] Batch [299]\tSpeed: 100.800730 samples/sec\taccuracy=0.741042\tlr=0.006700\n",
      "Epoch[2] Batch [349]\tSpeed: 100.906881 samples/sec\taccuracy=0.742679\tlr=0.006700\n",
      "Epoch[2] Batch [399]\tSpeed: 100.993829 samples/sec\taccuracy=0.744062\tlr=0.006700\n",
      "Epoch[2] Batch [449]\tSpeed: 100.951290 samples/sec\taccuracy=0.747500\tlr=0.006700\n",
      "[Epoch 2] training: accuracy=0.747500\n",
      "[Epoch 2] speed: 100 samples/sec\ttime cost: 71.652596\n",
      "[Epoch 2] validation: top1=0.912500 top5=0.993750\n",
      "Epoch[3] Batch [49]\tSpeed: 97.080070 samples/sec\taccuracy=0.720000\tlr=0.010000\n",
      "Epoch[3] Batch [99]\tSpeed: 100.904223 samples/sec\taccuracy=0.723125\tlr=0.010000\n",
      "Epoch[3] Batch [149]\tSpeed: 101.117306 samples/sec\taccuracy=0.730833\tlr=0.010000\n",
      "Epoch[3] Batch [199]\tSpeed: 100.791156 samples/sec\taccuracy=0.730313\tlr=0.010000\n",
      "Epoch[3] Batch [249]\tSpeed: 100.789048 samples/sec\taccuracy=0.736250\tlr=0.010000\n",
      "Epoch[3] Batch [299]\tSpeed: 100.729086 samples/sec\taccuracy=0.736458\tlr=0.010000\n",
      "Epoch[3] Batch [349]\tSpeed: 100.873042 samples/sec\taccuracy=0.739643\tlr=0.010000\n",
      "Epoch[3] Batch [399]\tSpeed: 100.955220 samples/sec\taccuracy=0.741875\tlr=0.010000\n",
      "Epoch[3] Batch [449]\tSpeed: 100.943943 samples/sec\taccuracy=0.745556\tlr=0.010000\n",
      "[Epoch 3] training: accuracy=0.745556\n",
      "[Epoch 3] speed: 100 samples/sec\ttime cost: 71.686217\n",
      "[Epoch 3] validation: top1=0.931250 top5=0.996250\n",
      "Epoch[4] Batch [49]\tSpeed: 97.278329 samples/sec\taccuracy=0.763750\tlr=0.010000\n",
      "Epoch[4] Batch [99]\tSpeed: 100.893950 samples/sec\taccuracy=0.761875\tlr=0.010000\n",
      "Epoch[4] Batch [149]\tSpeed: 101.086462 samples/sec\taccuracy=0.762083\tlr=0.010000\n",
      "Epoch[4] Batch [199]\tSpeed: 100.809479 samples/sec\taccuracy=0.765312\tlr=0.010000\n",
      "Epoch[4] Batch [249]\tSpeed: 100.876054 samples/sec\taccuracy=0.774500\tlr=0.010000\n",
      "Epoch[4] Batch [299]\tSpeed: 100.782664 samples/sec\taccuracy=0.776667\tlr=0.010000\n",
      "Epoch[4] Batch [349]\tSpeed: 100.899280 samples/sec\taccuracy=0.779286\tlr=0.010000\n",
      "Epoch[4] Batch [399]\tSpeed: 100.974162 samples/sec\taccuracy=0.775937\tlr=0.010000\n",
      "Epoch[4] Batch [449]\tSpeed: 100.864191 samples/sec\taccuracy=0.775694\tlr=0.010000\n",
      "[Epoch 4] training: accuracy=0.775694\n",
      "[Epoch 4] speed: 100 samples/sec\ttime cost: 71.662767\n",
      "[Epoch 4] validation: top1=0.915000 top5=1.000000\n",
      "Epoch[5] Batch [49]\tSpeed: 97.951796 samples/sec\taccuracy=0.780000\tlr=0.010000\n",
      "Epoch[5] Batch [99]\tSpeed: 101.031350 samples/sec\taccuracy=0.788125\tlr=0.010000\n",
      "Epoch[5] Batch [149]\tSpeed: 101.065925 samples/sec\taccuracy=0.788750\tlr=0.010000\n",
      "Epoch[5] Batch [199]\tSpeed: 100.836905 samples/sec\taccuracy=0.795937\tlr=0.010000\n",
      "Epoch[5] Batch [249]\tSpeed: 100.870547 samples/sec\taccuracy=0.793500\tlr=0.010000\n",
      "Epoch[5] Batch [299]\tSpeed: 100.866935 samples/sec\taccuracy=0.798542\tlr=0.010000\n",
      "Epoch[5] Batch [349]\tSpeed: 100.934627 samples/sec\taccuracy=0.802679\tlr=0.010000\n",
      "Epoch[5] Batch [399]\tSpeed: 100.993716 samples/sec\taccuracy=0.804688\tlr=0.010000\n",
      "Epoch[5] Batch [449]\tSpeed: 100.898613 samples/sec\taccuracy=0.806389\tlr=0.010000\n",
      "[Epoch 5] training: accuracy=0.806389\n",
      "[Epoch 5] speed: 100 samples/sec\ttime cost: 71.581930\n",
      "[Epoch 5] validation: top1=0.946250 top5=0.997500\n",
      "[Epoch 5] Current best top-1: 0.946250 vs previous 0.938750, saved to /datasets/ulyana/multiannotator_benchmarks/01ea71e9/.trial_0/best_checkpoint.pkl\n",
      "Epoch[6] Batch [49]\tSpeed: 97.412969 samples/sec\taccuracy=0.817500\tlr=0.010000\n",
      "Epoch[6] Batch [99]\tSpeed: 101.066446 samples/sec\taccuracy=0.806250\tlr=0.010000\n",
      "Epoch[6] Batch [149]\tSpeed: 101.236170 samples/sec\taccuracy=0.814583\tlr=0.010000\n",
      "Epoch[6] Batch [199]\tSpeed: 100.854626 samples/sec\taccuracy=0.819375\tlr=0.010000\n",
      "Epoch[6] Batch [249]\tSpeed: 100.958343 samples/sec\taccuracy=0.814500\tlr=0.010000\n",
      "Epoch[6] Batch [299]\tSpeed: 100.909800 samples/sec\taccuracy=0.813333\tlr=0.010000\n",
      "Epoch[6] Batch [349]\tSpeed: 101.001383 samples/sec\taccuracy=0.809464\tlr=0.010000\n",
      "Epoch[6] Batch [399]\tSpeed: 101.041834 samples/sec\taccuracy=0.810312\tlr=0.010000\n",
      "Epoch[6] Batch [449]\tSpeed: 100.991391 samples/sec\taccuracy=0.806111\tlr=0.010000\n",
      "[Epoch 6] training: accuracy=0.806111\n",
      "[Epoch 6] speed: 100 samples/sec\ttime cost: 71.582888\n",
      "[Epoch 6] validation: top1=0.923750 top5=0.996250\n",
      "Epoch[7] Batch [49]\tSpeed: 97.199771 samples/sec\taccuracy=0.832500\tlr=0.010000\n",
      "Epoch[7] Batch [99]\tSpeed: 101.060434 samples/sec\taccuracy=0.829375\tlr=0.010000\n",
      "Epoch[7] Batch [149]\tSpeed: 101.204763 samples/sec\taccuracy=0.833750\tlr=0.010000\n",
      "Epoch[7] Batch [199]\tSpeed: 100.721557 samples/sec\taccuracy=0.830625\tlr=0.010000\n",
      "Epoch[7] Batch [249]\tSpeed: 100.844945 samples/sec\taccuracy=0.833500\tlr=0.010000\n",
      "Epoch[7] Batch [299]\tSpeed: 100.786705 samples/sec\taccuracy=0.838958\tlr=0.010000\n",
      "Epoch[7] Batch [349]\tSpeed: 100.874892 samples/sec\taccuracy=0.838036\tlr=0.010000\n",
      "Epoch[7] Batch [399]\tSpeed: 101.047810 samples/sec\taccuracy=0.836406\tlr=0.010000\n",
      "Epoch[7] Batch [449]\tSpeed: 100.874631 samples/sec\taccuracy=0.835139\tlr=0.010000\n",
      "[Epoch 7] training: accuracy=0.835139\n",
      "[Epoch 7] speed: 100 samples/sec\ttime cost: 71.651903\n",
      "[Epoch 7] validation: top1=0.947500 top5=0.997500\n",
      "[Epoch 7] Current best top-1: 0.947500 vs previous 0.946250, saved to /datasets/ulyana/multiannotator_benchmarks/01ea71e9/.trial_0/best_checkpoint.pkl\n",
      "Epoch[8] Batch [49]\tSpeed: 98.576980 samples/sec\taccuracy=0.831250\tlr=0.010000\n",
      "Epoch[8] Batch [99]\tSpeed: 101.028363 samples/sec\taccuracy=0.834375\tlr=0.010000\n",
      "Epoch[8] Batch [149]\tSpeed: 101.230221 samples/sec\taccuracy=0.838750\tlr=0.010000\n",
      "Epoch[8] Batch [199]\tSpeed: 100.952578 samples/sec\taccuracy=0.844063\tlr=0.010000\n",
      "Epoch[8] Batch [249]\tSpeed: 100.810324 samples/sec\taccuracy=0.840250\tlr=0.010000\n",
      "Epoch[8] Batch [299]\tSpeed: 100.855205 samples/sec\taccuracy=0.842292\tlr=0.010000\n",
      "Epoch[8] Batch [349]\tSpeed: 100.940855 samples/sec\taccuracy=0.841071\tlr=0.010000\n",
      "Epoch[8] Batch [399]\tSpeed: 101.015853 samples/sec\taccuracy=0.840000\tlr=0.010000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch[8] Batch [449]\tSpeed: 101.004730 samples/sec\taccuracy=0.839028\tlr=0.010000\n",
      "[Epoch 8] training: accuracy=0.839028\n",
      "[Epoch 8] speed: 100 samples/sec\ttime cost: 71.503580\n",
      "[Epoch 8] validation: top1=0.941250 top5=0.998750\n",
      "Epoch[9] Batch [49]\tSpeed: 97.271429 samples/sec\taccuracy=0.857500\tlr=0.010000\n",
      "Epoch[9] Batch [99]\tSpeed: 101.028041 samples/sec\taccuracy=0.846250\tlr=0.010000\n",
      "Epoch[9] Batch [149]\tSpeed: 101.192332 samples/sec\taccuracy=0.844167\tlr=0.010000\n",
      "Epoch[9] Batch [199]\tSpeed: 100.834581 samples/sec\taccuracy=0.844375\tlr=0.010000\n",
      "Epoch[9] Batch [249]\tSpeed: 100.919667 samples/sec\taccuracy=0.845250\tlr=0.010000\n",
      "Epoch[9] Batch [299]\tSpeed: 100.881652 samples/sec\taccuracy=0.846875\tlr=0.010000\n",
      "Epoch[9] Batch [349]\tSpeed: 100.925243 samples/sec\taccuracy=0.845536\tlr=0.010000\n",
      "Epoch[9] Batch [399]\tSpeed: 100.997817 samples/sec\taccuracy=0.844531\tlr=0.010000\n",
      "Epoch[9] Batch [449]\tSpeed: 101.003180 samples/sec\taccuracy=0.844861\tlr=0.010000\n",
      "[Epoch 9] training: accuracy=0.844861\n",
      "[Epoch 9] speed: 100 samples/sec\ttime cost: 71.616589\n",
      "[Epoch 9] validation: top1=0.943750 top5=0.996250\n",
      "Epoch[10] Batch [49]\tSpeed: 97.686436 samples/sec\taccuracy=0.877500\tlr=0.010000\n",
      "Epoch[10] Batch [99]\tSpeed: 100.886930 samples/sec\taccuracy=0.859375\tlr=0.010000\n",
      "Epoch[10] Batch [149]\tSpeed: 101.075049 samples/sec\taccuracy=0.858333\tlr=0.010000\n",
      "Epoch[10] Batch [199]\tSpeed: 100.822670 samples/sec\taccuracy=0.856875\tlr=0.010000\n",
      "Epoch[10] Batch [249]\tSpeed: 100.917558 samples/sec\taccuracy=0.857500\tlr=0.010000\n",
      "Epoch[10] Batch [299]\tSpeed: 100.849482 samples/sec\taccuracy=0.854167\tlr=0.010000\n",
      "Epoch[10] Batch [349]\tSpeed: 101.043842 samples/sec\taccuracy=0.857679\tlr=0.010000\n",
      "Epoch[10] Batch [399]\tSpeed: 101.001872 samples/sec\taccuracy=0.853750\tlr=0.010000\n",
      "Epoch[10] Batch [449]\tSpeed: 100.977550 samples/sec\taccuracy=0.851389\tlr=0.010000\n",
      "[Epoch 10] training: accuracy=0.851389\n",
      "[Epoch 10] speed: 100 samples/sec\ttime cost: 71.597388\n",
      "[Epoch 10] validation: top1=0.930000 top5=0.998750\n",
      "Epoch[11] Batch [49]\tSpeed: 97.035173 samples/sec\taccuracy=0.853750\tlr=0.010000\n",
      "Epoch[11] Batch [99]\tSpeed: 101.020500 samples/sec\taccuracy=0.855625\tlr=0.010000\n",
      "Epoch[11] Batch [149]\tSpeed: 101.147265 samples/sec\taccuracy=0.853750\tlr=0.010000\n",
      "Epoch[11] Batch [199]\tSpeed: 100.854990 samples/sec\taccuracy=0.856875\tlr=0.010000\n",
      "Epoch[11] Batch [249]\tSpeed: 100.869297 samples/sec\taccuracy=0.858750\tlr=0.010000\n",
      "Epoch[11] Batch [299]\tSpeed: 100.848597 samples/sec\taccuracy=0.860000\tlr=0.010000\n",
      "Epoch[11] Batch [349]\tSpeed: 100.849476 samples/sec\taccuracy=0.857500\tlr=0.010000\n",
      "Epoch[11] Batch [399]\tSpeed: 100.969434 samples/sec\taccuracy=0.856563\tlr=0.010000\n",
      "Epoch[11] Batch [449]\tSpeed: 100.977006 samples/sec\taccuracy=0.855278\tlr=0.010000\n",
      "[Epoch 11] training: accuracy=0.855278\n",
      "[Epoch 11] speed: 100 samples/sec\ttime cost: 71.656100\n",
      "[Epoch 11] validation: top1=0.938750 top5=0.998750\n",
      "Epoch[12] Batch [49]\tSpeed: 97.327570 samples/sec\taccuracy=0.853750\tlr=0.010000\n",
      "Epoch[12] Batch [99]\tSpeed: 100.931737 samples/sec\taccuracy=0.851875\tlr=0.010000\n",
      "Epoch[12] Batch [149]\tSpeed: 101.212984 samples/sec\taccuracy=0.857500\tlr=0.010000\n",
      "Epoch[12] Batch [199]\tSpeed: 100.875332 samples/sec\taccuracy=0.857812\tlr=0.010000\n",
      "Epoch[12] Batch [249]\tSpeed: 100.819368 samples/sec\taccuracy=0.860000\tlr=0.010000\n",
      "Epoch[12] Batch [299]\tSpeed: 100.781226 samples/sec\taccuracy=0.860000\tlr=0.010000\n",
      "Epoch[12] Batch [349]\tSpeed: 100.973652 samples/sec\taccuracy=0.860536\tlr=0.010000\n",
      "Epoch[12] Batch [399]\tSpeed: 101.016778 samples/sec\taccuracy=0.860000\tlr=0.010000\n",
      "Epoch[12] Batch [449]\tSpeed: 100.970015 samples/sec\taccuracy=0.860278\tlr=0.010000\n",
      "[Epoch 12] training: accuracy=0.860278\n",
      "[Epoch 12] speed: 100 samples/sec\ttime cost: 71.628294\n",
      "[Epoch 12] validation: top1=0.958750 top5=0.998750\n",
      "[Epoch 12] Current best top-1: 0.958750 vs previous 0.947500, saved to /datasets/ulyana/multiannotator_benchmarks/01ea71e9/.trial_0/best_checkpoint.pkl\n",
      "Epoch[13] Batch [49]\tSpeed: 97.404777 samples/sec\taccuracy=0.877500\tlr=0.010000\n",
      "Epoch[13] Batch [99]\tSpeed: 101.144085 samples/sec\taccuracy=0.870000\tlr=0.010000\n",
      "Epoch[13] Batch [149]\tSpeed: 101.163650 samples/sec\taccuracy=0.868333\tlr=0.010000\n",
      "Epoch[13] Batch [199]\tSpeed: 100.867878 samples/sec\taccuracy=0.868437\tlr=0.010000\n",
      "Epoch[13] Batch [249]\tSpeed: 100.937466 samples/sec\taccuracy=0.862000\tlr=0.010000\n",
      "Epoch[13] Batch [299]\tSpeed: 100.878735 samples/sec\taccuracy=0.863125\tlr=0.010000\n",
      "Epoch[13] Batch [349]\tSpeed: 100.983011 samples/sec\taccuracy=0.863036\tlr=0.010000\n",
      "Epoch[13] Batch [399]\tSpeed: 101.027210 samples/sec\taccuracy=0.861875\tlr=0.010000\n",
      "Epoch[13] Batch [449]\tSpeed: 100.951919 samples/sec\taccuracy=0.862222\tlr=0.010000\n",
      "[Epoch 13] training: accuracy=0.862222\n",
      "[Epoch 13] speed: 100 samples/sec\ttime cost: 71.591893\n",
      "[Epoch 13] validation: top1=0.952500 top5=0.998750\n",
      "Epoch[14] Batch [49]\tSpeed: 97.693924 samples/sec\taccuracy=0.871250\tlr=0.010000\n",
      "Epoch[14] Batch [99]\tSpeed: 100.947736 samples/sec\taccuracy=0.870000\tlr=0.010000\n",
      "Epoch[14] Batch [149]\tSpeed: 101.123653 samples/sec\taccuracy=0.854167\tlr=0.010000\n",
      "Epoch[14] Batch [199]\tSpeed: 100.837359 samples/sec\taccuracy=0.858125\tlr=0.010000\n",
      "Epoch[14] Batch [249]\tSpeed: 100.859665 samples/sec\taccuracy=0.859000\tlr=0.010000\n",
      "Epoch[14] Batch [299]\tSpeed: 100.740315 samples/sec\taccuracy=0.855625\tlr=0.010000\n",
      "Epoch[14] Batch [349]\tSpeed: 100.951281 samples/sec\taccuracy=0.858036\tlr=0.010000\n",
      "Epoch[14] Batch [399]\tSpeed: 101.051778 samples/sec\taccuracy=0.862031\tlr=0.010000\n",
      "Epoch[14] Batch [449]\tSpeed: 100.940269 samples/sec\taccuracy=0.864583\tlr=0.010000\n",
      "[Epoch 14] training: accuracy=0.864583\n",
      "[Epoch 14] speed: 100 samples/sec\ttime cost: 71.607182\n",
      "[Epoch 14] validation: top1=0.933750 top5=1.000000\n",
      "Epoch[15] Batch [49]\tSpeed: 97.132357 samples/sec\taccuracy=0.856250\tlr=0.010000\n",
      "Epoch[15] Batch [99]\tSpeed: 100.802383 samples/sec\taccuracy=0.881250\tlr=0.010000\n",
      "Epoch[15] Batch [149]\tSpeed: 101.131471 samples/sec\taccuracy=0.874583\tlr=0.010000\n",
      "Epoch[15] Batch [199]\tSpeed: 100.882835 samples/sec\taccuracy=0.877188\tlr=0.010000\n",
      "Epoch[15] Batch [249]\tSpeed: 100.881255 samples/sec\taccuracy=0.878750\tlr=0.010000\n",
      "Epoch[15] Batch [299]\tSpeed: 100.806802 samples/sec\taccuracy=0.879167\tlr=0.010000\n",
      "Epoch[15] Batch [349]\tSpeed: 100.879202 samples/sec\taccuracy=0.874643\tlr=0.010000\n",
      "Epoch[15] Batch [399]\tSpeed: 101.000203 samples/sec\taccuracy=0.876094\tlr=0.010000\n",
      "Epoch[15] Batch [449]\tSpeed: 100.972928 samples/sec\taccuracy=0.876111\tlr=0.010000\n",
      "[Epoch 15] training: accuracy=0.876111\n",
      "[Epoch 15] speed: 100 samples/sec\ttime cost: 71.662197\n",
      "[Epoch 15] validation: top1=0.930000 top5=0.997500\n",
      "Epoch[16] Batch [49]\tSpeed: 97.642301 samples/sec\taccuracy=0.888750\tlr=0.010000\n",
      "Epoch[16] Batch [99]\tSpeed: 101.116184 samples/sec\taccuracy=0.875000\tlr=0.010000\n",
      "Epoch[16] Batch [149]\tSpeed: 101.181893 samples/sec\taccuracy=0.872500\tlr=0.010000\n",
      "Epoch[16] Batch [199]\tSpeed: 100.945908 samples/sec\taccuracy=0.873125\tlr=0.010000\n",
      "Epoch[16] Batch [249]\tSpeed: 100.965588 samples/sec\taccuracy=0.871250\tlr=0.010000\n",
      "Epoch[16] Batch [299]\tSpeed: 100.896680 samples/sec\taccuracy=0.868958\tlr=0.010000\n",
      "Epoch[16] Batch [349]\tSpeed: 100.991224 samples/sec\taccuracy=0.868214\tlr=0.010000\n",
      "Epoch[16] Batch [399]\tSpeed: 100.934728 samples/sec\taccuracy=0.871094\tlr=0.010000\n",
      "Epoch[16] Batch [449]\tSpeed: 100.904748 samples/sec\taccuracy=0.870139\tlr=0.010000\n",
      "[Epoch 16] training: accuracy=0.870139\n",
      "[Epoch 16] speed: 100 samples/sec\ttime cost: 71.573117\n",
      "[Epoch 16] validation: top1=0.915000 top5=0.997500\n",
      "Epoch[17] Batch [49]\tSpeed: 97.683319 samples/sec\taccuracy=0.870000\tlr=0.010000\n",
      "Epoch[17] Batch [99]\tSpeed: 101.021279 samples/sec\taccuracy=0.863125\tlr=0.010000\n",
      "Epoch[17] Batch [149]\tSpeed: 101.109103 samples/sec\taccuracy=0.868333\tlr=0.010000\n",
      "Epoch[17] Batch [199]\tSpeed: 100.885968 samples/sec\taccuracy=0.875313\tlr=0.010000\n",
      "Epoch[17] Batch [249]\tSpeed: 100.949562 samples/sec\taccuracy=0.875750\tlr=0.010000\n",
      "Epoch[17] Batch [299]\tSpeed: 100.891359 samples/sec\taccuracy=0.878542\tlr=0.010000\n",
      "Epoch[17] Batch [349]\tSpeed: 100.979893 samples/sec\taccuracy=0.875000\tlr=0.010000\n",
      "Epoch[17] Batch [399]\tSpeed: 101.035320 samples/sec\taccuracy=0.874687\tlr=0.010000\n",
      "Epoch[17] Batch [449]\tSpeed: 101.011395 samples/sec\taccuracy=0.876389\tlr=0.010000\n",
      "[Epoch 17] training: accuracy=0.876389\n",
      "[Epoch 17] speed: 100 samples/sec\ttime cost: 71.574428\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Epoch 17] validation: top1=0.951250 top5=0.997500\n",
      "Epoch[18] Batch [49]\tSpeed: 98.043480 samples/sec\taccuracy=0.873750\tlr=0.010000\n",
      "Epoch[18] Batch [99]\tSpeed: 101.015540 samples/sec\taccuracy=0.881250\tlr=0.010000\n",
      "Epoch[18] Batch [149]\tSpeed: 101.196455 samples/sec\taccuracy=0.888750\tlr=0.010000\n",
      "Epoch[18] Batch [199]\tSpeed: 100.913166 samples/sec\taccuracy=0.887500\tlr=0.010000\n",
      "Epoch[18] Batch [249]\tSpeed: 100.960578 samples/sec\taccuracy=0.887000\tlr=0.010000\n",
      "Epoch[18] Batch [299]\tSpeed: 100.914908 samples/sec\taccuracy=0.886042\tlr=0.010000\n",
      "Epoch[18] Batch [349]\tSpeed: 101.073953 samples/sec\taccuracy=0.884286\tlr=0.010000\n",
      "Epoch[18] Batch [399]\tSpeed: 101.035068 samples/sec\taccuracy=0.884844\tlr=0.010000\n",
      "Epoch[18] Batch [449]\tSpeed: 101.046611 samples/sec\taccuracy=0.885556\tlr=0.010000\n",
      "[Epoch 18] training: accuracy=0.885556\n",
      "[Epoch 18] speed: 100 samples/sec\ttime cost: 71.521473\n",
      "[Epoch 18] validation: top1=0.938750 top5=1.000000\n",
      "Epoch[19] Batch [49]\tSpeed: 97.738561 samples/sec\taccuracy=0.885000\tlr=0.010000\n",
      "Epoch[19] Batch [99]\tSpeed: 101.019059 samples/sec\taccuracy=0.891250\tlr=0.010000\n",
      "Epoch[19] Batch [149]\tSpeed: 101.132279 samples/sec\taccuracy=0.887500\tlr=0.010000\n",
      "Epoch[19] Batch [199]\tSpeed: 100.851713 samples/sec\taccuracy=0.888125\tlr=0.010000\n",
      "Epoch[19] Batch [249]\tSpeed: 100.779207 samples/sec\taccuracy=0.887750\tlr=0.010000\n",
      "Epoch[19] Batch [299]\tSpeed: 100.937743 samples/sec\taccuracy=0.888958\tlr=0.010000\n",
      "Epoch[19] Batch [349]\tSpeed: 100.950995 samples/sec\taccuracy=0.889464\tlr=0.010000\n",
      "Epoch[19] Batch [399]\tSpeed: 101.058413 samples/sec\taccuracy=0.888906\tlr=0.010000\n",
      "Epoch[19] Batch [449]\tSpeed: 101.048939 samples/sec\taccuracy=0.886250\tlr=0.010000\n",
      "[Epoch 19] training: accuracy=0.886250\n",
      "[Epoch 19] speed: 100 samples/sec\ttime cost: 71.577619\n",
      "[Epoch 19] validation: top1=0.943750 top5=1.000000\n",
      "Epoch[20] Batch [49]\tSpeed: 97.930316 samples/sec\taccuracy=0.908750\tlr=0.010000\n",
      "Epoch[20] Batch [99]\tSpeed: 100.924694 samples/sec\taccuracy=0.906250\tlr=0.010000\n",
      "Epoch[20] Batch [149]\tSpeed: 101.074590 samples/sec\taccuracy=0.895833\tlr=0.010000\n",
      "Epoch[20] Batch [199]\tSpeed: 100.802874 samples/sec\taccuracy=0.893437\tlr=0.010000\n",
      "Epoch[20] Batch [249]\tSpeed: 100.864061 samples/sec\taccuracy=0.891000\tlr=0.010000\n",
      "Epoch[20] Batch [299]\tSpeed: 100.866411 samples/sec\taccuracy=0.888125\tlr=0.010000\n",
      "Epoch[20] Batch [349]\tSpeed: 100.979173 samples/sec\taccuracy=0.889107\tlr=0.010000\n",
      "Epoch[20] Batch [399]\tSpeed: 101.062117 samples/sec\taccuracy=0.889531\tlr=0.010000\n",
      "Epoch[20] Batch [449]\tSpeed: 100.951679 samples/sec\taccuracy=0.889722\tlr=0.010000\n",
      "[Epoch 20] training: accuracy=0.889722\n",
      "[Epoch 20] speed: 100 samples/sec\ttime cost: 71.581147\n",
      "[Epoch 20] validation: top1=0.940000 top5=0.996250\n",
      "Epoch[21] Batch [49]\tSpeed: 97.027123 samples/sec\taccuracy=0.896250\tlr=0.010000\n",
      "Epoch[21] Batch [99]\tSpeed: 100.938654 samples/sec\taccuracy=0.896250\tlr=0.010000\n",
      "Epoch[21] Batch [149]\tSpeed: 101.023369 samples/sec\taccuracy=0.891667\tlr=0.010000\n",
      "Epoch[21] Batch [199]\tSpeed: 100.767852 samples/sec\taccuracy=0.892813\tlr=0.010000\n",
      "Epoch[21] Batch [249]\tSpeed: 100.886245 samples/sec\taccuracy=0.890000\tlr=0.010000\n",
      "Epoch[21] Batch [299]\tSpeed: 100.846685 samples/sec\taccuracy=0.891667\tlr=0.010000\n",
      "Epoch[21] Batch [349]\tSpeed: 100.890288 samples/sec\taccuracy=0.894107\tlr=0.010000\n",
      "Epoch[21] Batch [399]\tSpeed: 101.000957 samples/sec\taccuracy=0.893750\tlr=0.010000\n",
      "Epoch[21] Batch [449]\tSpeed: 100.987835 samples/sec\taccuracy=0.893056\tlr=0.010000\n",
      "[Epoch 21] training: accuracy=0.893056\n",
      "[Epoch 21] speed: 100 samples/sec\ttime cost: 71.672169\n",
      "[Epoch 21] validation: top1=0.948750 top5=0.998750\n",
      "Epoch[22] Batch [49]\tSpeed: 97.912198 samples/sec\taccuracy=0.891250\tlr=0.010000\n",
      "Epoch[22] Batch [99]\tSpeed: 100.974976 samples/sec\taccuracy=0.890625\tlr=0.010000\n",
      "Epoch[22] Batch [149]\tSpeed: 100.998136 samples/sec\taccuracy=0.886667\tlr=0.010000\n",
      "Epoch[22] Batch [199]\tSpeed: 100.745191 samples/sec\taccuracy=0.892813\tlr=0.010000\n",
      "Epoch[22] Batch [249]\tSpeed: 100.891899 samples/sec\taccuracy=0.890250\tlr=0.010000\n",
      "Epoch[22] Batch [299]\tSpeed: 100.814252 samples/sec\taccuracy=0.890000\tlr=0.010000\n",
      "Epoch[22] Batch [349]\tSpeed: 100.810115 samples/sec\taccuracy=0.890714\tlr=0.010000\n",
      "Epoch[22] Batch [399]\tSpeed: 100.932414 samples/sec\taccuracy=0.891406\tlr=0.010000\n",
      "Epoch[22] Batch [449]\tSpeed: 100.955715 samples/sec\taccuracy=0.890278\tlr=0.010000\n",
      "[Epoch 22] training: accuracy=0.890278\n",
      "[Epoch 22] speed: 100 samples/sec\ttime cost: 71.614591\n",
      "[Epoch 22] validation: top1=0.957500 top5=0.996250\n",
      "[Epoch 23] EarlyStop after 10 epochs: no better than 0.95875\n",
      "Applying the state from the best checkpoint...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Folder ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_2/ already exists!\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_2/_test_pred_probs_split_2\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_2/_test_pred_features_split_2\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_2/_test_labels_split_2\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_2/_test_image_files_split_2\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_2/_test_indices_split_2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "modified configs(<old> != <new>): {\n",
      "root.img_cls.model   resnet101 != swin_base_patch4_window7_224\n",
      "root.train.early_stop_patience -1 != 10\n",
      "root.train.early_stop_baseline 0.0 != -inf\n",
      "root.train.early_stop_max_value 1.0 != inf\n",
      "root.train.epochs    200 != 100\n",
      "root.train.batch_size 32 != 16\n",
      "root.misc.seed       42 != 554\n",
      "root.misc.num_workers 4 != 64\n",
      "}\n",
      "Saved config to /datasets/ulyana/multiannotator_benchmarks/dfc70db1/.trial_0/config.yaml\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----\n",
      "Running Cross-Validation on Split: 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model swin_base_patch4_window7_224 created, param count:                                         86753474\n",
      "AMP not enabled. Training in float32.\n",
      "Disable EMA as it is not supported for now.\n",
      "Start training from [Epoch 0]\n",
      "Epoch[0] Batch [49]\tSpeed: 72.362471 samples/sec\taccuracy=0.123750\tlr=0.000100\n",
      "Epoch[0] Batch [99]\tSpeed: 101.640667 samples/sec\taccuracy=0.214375\tlr=0.000100\n",
      "Epoch[0] Batch [149]\tSpeed: 101.345929 samples/sec\taccuracy=0.300000\tlr=0.000100\n",
      "Epoch[0] Batch [199]\tSpeed: 101.024439 samples/sec\taccuracy=0.375937\tlr=0.000100\n",
      "Epoch[0] Batch [249]\tSpeed: 100.925829 samples/sec\taccuracy=0.438000\tlr=0.000100\n",
      "Epoch[0] Batch [299]\tSpeed: 100.782697 samples/sec\taccuracy=0.486667\tlr=0.000100\n",
      "Epoch[0] Batch [349]\tSpeed: 100.882432 samples/sec\taccuracy=0.523214\tlr=0.000100\n",
      "Epoch[0] Batch [399]\tSpeed: 101.006898 samples/sec\taccuracy=0.560000\tlr=0.000100\n",
      "Epoch[0] Batch [449]\tSpeed: 100.917552 samples/sec\taccuracy=0.589028\tlr=0.000100\n",
      "[Epoch 0] training: accuracy=0.589028\n",
      "[Epoch 0] speed: 96 samples/sec\ttime cost: 74.390724\n",
      "[Epoch 0] validation: top1=0.952500 top5=0.998750\n",
      "[Epoch 0] Current best top-1: 0.952500 vs previous -inf, saved to /datasets/ulyana/multiannotator_benchmarks/dfc70db1/.trial_0/best_checkpoint.pkl\n",
      "Epoch[1] Batch [49]\tSpeed: 97.946730 samples/sec\taccuracy=0.587500\tlr=0.003400\n",
      "Epoch[1] Batch [99]\tSpeed: 101.140793 samples/sec\taccuracy=0.655625\tlr=0.003400\n",
      "Epoch[1] Batch [149]\tSpeed: 101.225179 samples/sec\taccuracy=0.677083\tlr=0.003400\n",
      "Epoch[1] Batch [199]\tSpeed: 100.916043 samples/sec\taccuracy=0.699375\tlr=0.003400\n",
      "Epoch[1] Batch [249]\tSpeed: 100.866823 samples/sec\taccuracy=0.710000\tlr=0.003400\n",
      "Epoch[1] Batch [299]\tSpeed: 100.924199 samples/sec\taccuracy=0.715417\tlr=0.003400\n",
      "Epoch[1] Batch [349]\tSpeed: 100.928225 samples/sec\taccuracy=0.721607\tlr=0.003400\n",
      "Epoch[1] Batch [399]\tSpeed: 100.940867 samples/sec\taccuracy=0.729375\tlr=0.003400\n",
      "Epoch[1] Batch [449]\tSpeed: 100.886818 samples/sec\taccuracy=0.738056\tlr=0.003400\n",
      "[Epoch 1] training: accuracy=0.738056\n",
      "[Epoch 1] speed: 100 samples/sec\ttime cost: 71.556266\n",
      "[Epoch 1] validation: top1=0.938750 top5=0.998750\n",
      "Epoch[2] Batch [49]\tSpeed: 97.275628 samples/sec\taccuracy=0.733750\tlr=0.006700\n",
      "Epoch[2] Batch [99]\tSpeed: 101.008297 samples/sec\taccuracy=0.755000\tlr=0.006700\n",
      "Epoch[2] Batch [149]\tSpeed: 101.192463 samples/sec\taccuracy=0.752083\tlr=0.006700\n",
      "Epoch[2] Batch [199]\tSpeed: 100.871590 samples/sec\taccuracy=0.750000\tlr=0.006700\n",
      "Epoch[2] Batch [249]\tSpeed: 100.877652 samples/sec\taccuracy=0.744000\tlr=0.006700\n",
      "Epoch[2] Batch [299]\tSpeed: 100.921895 samples/sec\taccuracy=0.747083\tlr=0.006700\n",
      "Epoch[2] Batch [349]\tSpeed: 100.942531 samples/sec\taccuracy=0.750893\tlr=0.006700\n",
      "Epoch[2] Batch [399]\tSpeed: 100.988862 samples/sec\taccuracy=0.752656\tlr=0.006700\n",
      "Epoch[2] Batch [449]\tSpeed: 100.977474 samples/sec\taccuracy=0.750972\tlr=0.006700\n",
      "[Epoch 2] training: accuracy=0.750972\n",
      "[Epoch 2] speed: 100 samples/sec\ttime cost: 71.616678\n",
      "[Epoch 2] validation: top1=0.918750 top5=0.996250\n",
      "Epoch[3] Batch [49]\tSpeed: 97.675132 samples/sec\taccuracy=0.751250\tlr=0.010000\n",
      "Epoch[3] Batch [99]\tSpeed: 100.918119 samples/sec\taccuracy=0.744375\tlr=0.010000\n",
      "Epoch[3] Batch [149]\tSpeed: 101.072194 samples/sec\taccuracy=0.751667\tlr=0.010000\n",
      "Epoch[3] Batch [199]\tSpeed: 100.762447 samples/sec\taccuracy=0.747812\tlr=0.010000\n",
      "Epoch[3] Batch [249]\tSpeed: 100.863709 samples/sec\taccuracy=0.749500\tlr=0.010000\n",
      "Epoch[3] Batch [299]\tSpeed: 100.838372 samples/sec\taccuracy=0.749792\tlr=0.010000\n",
      "Epoch[3] Batch [349]\tSpeed: 100.875129 samples/sec\taccuracy=0.749286\tlr=0.010000\n",
      "Epoch[3] Batch [399]\tSpeed: 100.991135 samples/sec\taccuracy=0.749687\tlr=0.010000\n",
      "Epoch[3] Batch [449]\tSpeed: 100.889891 samples/sec\taccuracy=0.752222\tlr=0.010000\n",
      "[Epoch 3] training: accuracy=0.752222\n",
      "[Epoch 3] speed: 100 samples/sec\ttime cost: 71.627649\n",
      "[Epoch 3] validation: top1=0.902500 top5=1.000000\n",
      "Epoch[4] Batch [49]\tSpeed: 97.400092 samples/sec\taccuracy=0.770000\tlr=0.010000\n",
      "Epoch[4] Batch [99]\tSpeed: 100.980817 samples/sec\taccuracy=0.774375\tlr=0.010000\n",
      "Epoch[4] Batch [149]\tSpeed: 101.045208 samples/sec\taccuracy=0.780000\tlr=0.010000\n",
      "Epoch[4] Batch [199]\tSpeed: 100.752448 samples/sec\taccuracy=0.780937\tlr=0.010000\n",
      "Epoch[4] Batch [249]\tSpeed: 100.889857 samples/sec\taccuracy=0.785750\tlr=0.010000\n",
      "Epoch[4] Batch [299]\tSpeed: 100.870559 samples/sec\taccuracy=0.781042\tlr=0.010000\n",
      "Epoch[4] Batch [349]\tSpeed: 100.896419 samples/sec\taccuracy=0.781071\tlr=0.010000\n",
      "Epoch[4] Batch [399]\tSpeed: 101.008038 samples/sec\taccuracy=0.783125\tlr=0.010000\n",
      "Epoch[4] Batch [449]\tSpeed: 100.907182 samples/sec\taccuracy=0.784861\tlr=0.010000\n",
      "[Epoch 4] training: accuracy=0.784861\n",
      "[Epoch 4] speed: 100 samples/sec\ttime cost: 71.640146\n",
      "[Epoch 4] validation: top1=0.927500 top5=0.997500\n",
      "Epoch[5] Batch [49]\tSpeed: 97.554485 samples/sec\taccuracy=0.828750\tlr=0.010000\n",
      "Epoch[5] Batch [99]\tSpeed: 100.673073 samples/sec\taccuracy=0.811250\tlr=0.010000\n",
      "Epoch[5] Batch [149]\tSpeed: 101.113421 samples/sec\taccuracy=0.807500\tlr=0.010000\n",
      "Epoch[5] Batch [199]\tSpeed: 100.794211 samples/sec\taccuracy=0.806562\tlr=0.010000\n",
      "Epoch[5] Batch [249]\tSpeed: 100.855645 samples/sec\taccuracy=0.804000\tlr=0.010000\n",
      "Epoch[5] Batch [299]\tSpeed: 100.806035 samples/sec\taccuracy=0.803958\tlr=0.010000\n",
      "Epoch[5] Batch [349]\tSpeed: 100.965785 samples/sec\taccuracy=0.802857\tlr=0.010000\n",
      "Epoch[5] Batch [399]\tSpeed: 101.004846 samples/sec\taccuracy=0.802969\tlr=0.010000\n",
      "Epoch[5] Batch [449]\tSpeed: 100.980398 samples/sec\taccuracy=0.804861\tlr=0.010000\n",
      "[Epoch 5] training: accuracy=0.804861\n",
      "[Epoch 5] speed: 100 samples/sec\ttime cost: 71.638674\n",
      "[Epoch 5] validation: top1=0.950000 top5=1.000000\n",
      "Epoch[6] Batch [49]\tSpeed: 97.026453 samples/sec\taccuracy=0.853750\tlr=0.010000\n",
      "Epoch[6] Batch [99]\tSpeed: 100.911782 samples/sec\taccuracy=0.855000\tlr=0.010000\n",
      "Epoch[6] Batch [149]\tSpeed: 101.204436 samples/sec\taccuracy=0.845833\tlr=0.010000\n",
      "Epoch[6] Batch [199]\tSpeed: 100.795458 samples/sec\taccuracy=0.839688\tlr=0.010000\n",
      "Epoch[6] Batch [249]\tSpeed: 100.985737 samples/sec\taccuracy=0.839500\tlr=0.010000\n",
      "Epoch[6] Batch [299]\tSpeed: 100.823276 samples/sec\taccuracy=0.835000\tlr=0.010000\n",
      "Epoch[6] Batch [349]\tSpeed: 100.976517 samples/sec\taccuracy=0.831607\tlr=0.010000\n",
      "Epoch[6] Batch [399]\tSpeed: 101.014561 samples/sec\taccuracy=0.831875\tlr=0.010000\n",
      "Epoch[6] Batch [449]\tSpeed: 100.983321 samples/sec\taccuracy=0.830278\tlr=0.010000\n",
      "[Epoch 6] training: accuracy=0.830278\n",
      "[Epoch 6] speed: 100 samples/sec\ttime cost: 71.643620\n",
      "[Epoch 6] validation: top1=0.936250 top5=0.997500\n",
      "Epoch[7] Batch [49]\tSpeed: 97.446487 samples/sec\taccuracy=0.817500\tlr=0.010000\n",
      "Epoch[7] Batch [99]\tSpeed: 100.945398 samples/sec\taccuracy=0.819375\tlr=0.010000\n",
      "Epoch[7] Batch [149]\tSpeed: 101.144948 samples/sec\taccuracy=0.823333\tlr=0.010000\n",
      "Epoch[7] Batch [199]\tSpeed: 100.858558 samples/sec\taccuracy=0.817187\tlr=0.010000\n",
      "Epoch[7] Batch [249]\tSpeed: 100.868912 samples/sec\taccuracy=0.815000\tlr=0.010000\n",
      "Epoch[7] Batch [299]\tSpeed: 100.830269 samples/sec\taccuracy=0.811250\tlr=0.010000\n",
      "Epoch[7] Batch [349]\tSpeed: 101.018843 samples/sec\taccuracy=0.817321\tlr=0.010000\n",
      "Epoch[7] Batch [399]\tSpeed: 101.034986 samples/sec\taccuracy=0.814531\tlr=0.010000\n",
      "Epoch[7] Batch [449]\tSpeed: 100.983105 samples/sec\taccuracy=0.814861\tlr=0.010000\n",
      "[Epoch 7] training: accuracy=0.814861\n",
      "[Epoch 7] speed: 100 samples/sec\ttime cost: 71.610226\n",
      "[Epoch 7] validation: top1=0.945000 top5=0.996250\n",
      "Epoch[8] Batch [49]\tSpeed: 97.825120 samples/sec\taccuracy=0.801250\tlr=0.010000\n",
      "Epoch[8] Batch [99]\tSpeed: 101.000881 samples/sec\taccuracy=0.816250\tlr=0.010000\n",
      "Epoch[8] Batch [149]\tSpeed: 101.233794 samples/sec\taccuracy=0.822500\tlr=0.010000\n",
      "Epoch[8] Batch [199]\tSpeed: 100.806114 samples/sec\taccuracy=0.823438\tlr=0.010000\n",
      "Epoch[8] Batch [249]\tSpeed: 100.679915 samples/sec\taccuracy=0.823750\tlr=0.010000\n",
      "Epoch[8] Batch [299]\tSpeed: 100.859747 samples/sec\taccuracy=0.826667\tlr=0.010000\n",
      "Epoch[8] Batch [349]\tSpeed: 100.888207 samples/sec\taccuracy=0.828929\tlr=0.010000\n",
      "Epoch[8] Batch [399]\tSpeed: 100.947390 samples/sec\taccuracy=0.829688\tlr=0.010000\n",
      "Epoch[8] Batch [449]\tSpeed: 100.943925 samples/sec\taccuracy=0.831389\tlr=0.010000\n",
      "[Epoch 8] training: accuracy=0.831389\n",
      "[Epoch 8] speed: 100 samples/sec\ttime cost: 71.603530\n",
      "[Epoch 8] validation: top1=0.935000 top5=1.000000\n",
      "Epoch[9] Batch [49]\tSpeed: 97.485554 samples/sec\taccuracy=0.868750\tlr=0.010000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch[9] Batch [99]\tSpeed: 100.998887 samples/sec\taccuracy=0.858125\tlr=0.010000\n",
      "Epoch[9] Batch [149]\tSpeed: 101.104981 samples/sec\taccuracy=0.860000\tlr=0.010000\n",
      "Epoch[9] Batch [199]\tSpeed: 100.798752 samples/sec\taccuracy=0.852187\tlr=0.010000\n",
      "Epoch[9] Batch [249]\tSpeed: 100.864661 samples/sec\taccuracy=0.850750\tlr=0.010000\n",
      "Epoch[9] Batch [299]\tSpeed: 100.842651 samples/sec\taccuracy=0.852292\tlr=0.010000\n",
      "Epoch[9] Batch [349]\tSpeed: 100.860929 samples/sec\taccuracy=0.851786\tlr=0.010000\n",
      "Epoch[9] Batch [399]\tSpeed: 101.029163 samples/sec\taccuracy=0.847812\tlr=0.010000\n",
      "Epoch[9] Batch [449]\tSpeed: 100.970394 samples/sec\taccuracy=0.841667\tlr=0.010000\n",
      "[Epoch 9] training: accuracy=0.841667\n",
      "[Epoch 9] speed: 100 samples/sec\ttime cost: 71.623932\n",
      "[Epoch 9] validation: top1=0.942500 top5=1.000000\n",
      "Epoch[10] Batch [49]\tSpeed: 97.135191 samples/sec\taccuracy=0.870000\tlr=0.010000\n",
      "Epoch[10] Batch [99]\tSpeed: 101.077373 samples/sec\taccuracy=0.865000\tlr=0.010000\n",
      "Epoch[10] Batch [149]\tSpeed: 101.200969 samples/sec\taccuracy=0.860000\tlr=0.010000\n",
      "Epoch[10] Batch [199]\tSpeed: 100.846333 samples/sec\taccuracy=0.859688\tlr=0.010000\n",
      "Epoch[10] Batch [249]\tSpeed: 100.905634 samples/sec\taccuracy=0.855500\tlr=0.010000\n",
      "Epoch[10] Batch [299]\tSpeed: 100.876560 samples/sec\taccuracy=0.853542\tlr=0.010000\n",
      "Epoch[10] Batch [349]\tSpeed: 101.001201 samples/sec\taccuracy=0.850179\tlr=0.010000\n",
      "Epoch[10] Batch [399]\tSpeed: 101.068333 samples/sec\taccuracy=0.839375\tlr=0.010000\n",
      "Epoch[10] Batch [449]\tSpeed: 101.000462 samples/sec\taccuracy=0.836250\tlr=0.010000\n",
      "[Epoch 10] training: accuracy=0.836250\n",
      "[Epoch 10] speed: 100 samples/sec\ttime cost: 71.613357\n",
      "[Epoch 10] validation: top1=0.936250 top5=0.997500\n",
      "[Epoch 11] EarlyStop after 10 epochs: no better than 0.9525\n",
      "Applying the state from the best checkpoint...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_3/_test_pred_probs_split_3\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_3/_test_pred_features_split_3\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_3/_test_labels_split_3\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_3/_test_image_files_split_3\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_3/_test_indices_split_3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "modified configs(<old> != <new>): {\n",
      "root.img_cls.model   resnet101 != swin_base_patch4_window7_224\n",
      "root.train.early_stop_patience -1 != 10\n",
      "root.train.early_stop_baseline 0.0 != -inf\n",
      "root.train.early_stop_max_value 1.0 != inf\n",
      "root.train.epochs    200 != 100\n",
      "root.train.batch_size 32 != 16\n",
      "root.misc.seed       42 != 479\n",
      "root.misc.num_workers 4 != 64\n",
      "}\n",
      "Saved config to /datasets/ulyana/multiannotator_benchmarks/c2314db8/.trial_0/config.yaml\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----\n",
      "Running Cross-Validation on Split: 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model swin_base_patch4_window7_224 created, param count:                                         86753474\n",
      "AMP not enabled. Training in float32.\n",
      "Disable EMA as it is not supported for now.\n",
      "Start training from [Epoch 0]\n",
      "Epoch[0] Batch [49]\tSpeed: 72.559596 samples/sec\taccuracy=0.130000\tlr=0.000100\n",
      "Epoch[0] Batch [99]\tSpeed: 101.671449 samples/sec\taccuracy=0.193750\tlr=0.000100\n",
      "Epoch[0] Batch [149]\tSpeed: 101.485629 samples/sec\taccuracy=0.285000\tlr=0.000100\n",
      "Epoch[0] Batch [199]\tSpeed: 101.226962 samples/sec\taccuracy=0.366563\tlr=0.000100\n",
      "Epoch[0] Batch [249]\tSpeed: 101.073573 samples/sec\taccuracy=0.426750\tlr=0.000100\n",
      "Epoch[0] Batch [299]\tSpeed: 100.993601 samples/sec\taccuracy=0.475000\tlr=0.000100\n",
      "Epoch[0] Batch [349]\tSpeed: 101.069676 samples/sec\taccuracy=0.517857\tlr=0.000100\n",
      "Epoch[0] Batch [399]\tSpeed: 101.076721 samples/sec\taccuracy=0.545625\tlr=0.000100\n",
      "Epoch[0] Batch [449]\tSpeed: 101.086264 samples/sec\taccuracy=0.572639\tlr=0.000100\n",
      "[Epoch 0] training: accuracy=0.572639\n",
      "[Epoch 0] speed: 96 samples/sec\ttime cost: 74.269226\n",
      "[Epoch 0] validation: top1=0.950000 top5=0.998750\n",
      "[Epoch 0] Current best top-1: 0.950000 vs previous -inf, saved to /datasets/ulyana/multiannotator_benchmarks/c2314db8/.trial_0/best_checkpoint.pkl\n",
      "Epoch[1] Batch [49]\tSpeed: 97.811883 samples/sec\taccuracy=0.665000\tlr=0.003400\n",
      "Epoch[1] Batch [99]\tSpeed: 101.059570 samples/sec\taccuracy=0.688125\tlr=0.003400\n",
      "Epoch[1] Batch [149]\tSpeed: 101.280291 samples/sec\taccuracy=0.702917\tlr=0.003400\n",
      "Epoch[1] Batch [199]\tSpeed: 100.981370 samples/sec\taccuracy=0.715000\tlr=0.003400\n",
      "Epoch[1] Batch [249]\tSpeed: 100.797223 samples/sec\taccuracy=0.727250\tlr=0.003400\n",
      "Epoch[1] Batch [299]\tSpeed: 100.938790 samples/sec\taccuracy=0.730625\tlr=0.003400\n",
      "Epoch[1] Batch [349]\tSpeed: 100.959494 samples/sec\taccuracy=0.737143\tlr=0.003400\n",
      "Epoch[1] Batch [399]\tSpeed: 101.062008 samples/sec\taccuracy=0.747969\tlr=0.003400\n",
      "Epoch[1] Batch [449]\tSpeed: 101.032905 samples/sec\taccuracy=0.752083\tlr=0.003400\n",
      "[Epoch 1] training: accuracy=0.752083\n",
      "[Epoch 1] speed: 100 samples/sec\ttime cost: 71.545440\n",
      "[Epoch 1] validation: top1=0.925000 top5=1.000000\n",
      "Epoch[2] Batch [49]\tSpeed: 97.585535 samples/sec\taccuracy=0.731250\tlr=0.006700\n",
      "Epoch[2] Batch [99]\tSpeed: 101.005311 samples/sec\taccuracy=0.726250\tlr=0.006700\n",
      "Epoch[2] Batch [149]\tSpeed: 101.137250 samples/sec\taccuracy=0.732917\tlr=0.006700\n",
      "Epoch[2] Batch [199]\tSpeed: 100.813501 samples/sec\taccuracy=0.732187\tlr=0.006700\n",
      "Epoch[2] Batch [249]\tSpeed: 100.851959 samples/sec\taccuracy=0.735750\tlr=0.006700\n",
      "Epoch[2] Batch [299]\tSpeed: 100.825155 samples/sec\taccuracy=0.731667\tlr=0.006700\n",
      "Epoch[2] Batch [349]\tSpeed: 100.906235 samples/sec\taccuracy=0.731964\tlr=0.006700\n",
      "Epoch[2] Batch [399]\tSpeed: 100.923452 samples/sec\taccuracy=0.735469\tlr=0.006700\n",
      "Epoch[2] Batch [449]\tSpeed: 100.929424 samples/sec\taccuracy=0.735833\tlr=0.006700\n",
      "[Epoch 2] training: accuracy=0.735833\n",
      "[Epoch 2] speed: 100 samples/sec\ttime cost: 71.621109\n",
      "[Epoch 2] validation: top1=0.928750 top5=0.996250\n",
      "Epoch[3] Batch [49]\tSpeed: 97.522942 samples/sec\taccuracy=0.738750\tlr=0.010000\n",
      "Epoch[3] Batch [99]\tSpeed: 100.995826 samples/sec\taccuracy=0.721875\tlr=0.010000\n",
      "Epoch[3] Batch [149]\tSpeed: 101.107857 samples/sec\taccuracy=0.725000\tlr=0.010000\n",
      "Epoch[3] Batch [199]\tSpeed: 100.839905 samples/sec\taccuracy=0.729688\tlr=0.010000\n",
      "Epoch[3] Batch [249]\tSpeed: 100.921142 samples/sec\taccuracy=0.730500\tlr=0.010000\n",
      "Epoch[3] Batch [299]\tSpeed: 100.813913 samples/sec\taccuracy=0.736250\tlr=0.010000\n",
      "Epoch[3] Batch [349]\tSpeed: 100.941229 samples/sec\taccuracy=0.741607\tlr=0.010000\n",
      "Epoch[3] Batch [399]\tSpeed: 101.018375 samples/sec\taccuracy=0.744219\tlr=0.010000\n",
      "Epoch[3] Batch [449]\tSpeed: 100.948113 samples/sec\taccuracy=0.744306\tlr=0.010000\n",
      "[Epoch 3] training: accuracy=0.744306\n",
      "[Epoch 3] speed: 100 samples/sec\ttime cost: 71.611354\n",
      "[Epoch 3] validation: top1=0.875000 top5=0.986250\n",
      "Epoch[4] Batch [49]\tSpeed: 97.153183 samples/sec\taccuracy=0.748750\tlr=0.010000\n",
      "Epoch[4] Batch [99]\tSpeed: 100.966275 samples/sec\taccuracy=0.743750\tlr=0.010000\n",
      "Epoch[4] Batch [149]\tSpeed: 101.133260 samples/sec\taccuracy=0.755000\tlr=0.010000\n",
      "Epoch[4] Batch [199]\tSpeed: 100.788186 samples/sec\taccuracy=0.753750\tlr=0.010000\n",
      "Epoch[4] Batch [249]\tSpeed: 100.851328 samples/sec\taccuracy=0.757750\tlr=0.010000\n",
      "Epoch[4] Batch [299]\tSpeed: 100.833741 samples/sec\taccuracy=0.761250\tlr=0.010000\n",
      "Epoch[4] Batch [349]\tSpeed: 100.847782 samples/sec\taccuracy=0.762321\tlr=0.010000\n",
      "Epoch[4] Batch [399]\tSpeed: 100.945878 samples/sec\taccuracy=0.767969\tlr=0.010000\n",
      "Epoch[4] Batch [449]\tSpeed: 100.913943 samples/sec\taccuracy=0.764861\tlr=0.010000\n",
      "[Epoch 4] training: accuracy=0.764861\n",
      "[Epoch 4] speed: 100 samples/sec\ttime cost: 71.666581\n",
      "[Epoch 4] validation: top1=0.905000 top5=0.995000\n",
      "Epoch[5] Batch [49]\tSpeed: 97.666367 samples/sec\taccuracy=0.790000\tlr=0.010000\n",
      "Epoch[5] Batch [99]\tSpeed: 101.043660 samples/sec\taccuracy=0.776875\tlr=0.010000\n",
      "Epoch[5] Batch [149]\tSpeed: 101.170141 samples/sec\taccuracy=0.760417\tlr=0.010000\n",
      "Epoch[5] Batch [199]\tSpeed: 100.940955 samples/sec\taccuracy=0.755938\tlr=0.010000\n",
      "Epoch[5] Batch [249]\tSpeed: 100.947566 samples/sec\taccuracy=0.764500\tlr=0.010000\n",
      "Epoch[5] Batch [299]\tSpeed: 100.919024 samples/sec\taccuracy=0.765625\tlr=0.010000\n",
      "Epoch[5] Batch [349]\tSpeed: 101.007005 samples/sec\taccuracy=0.770714\tlr=0.010000\n",
      "Epoch[5] Batch [399]\tSpeed: 101.066029 samples/sec\taccuracy=0.765781\tlr=0.010000\n",
      "Epoch[5] Batch [449]\tSpeed: 100.961669 samples/sec\taccuracy=0.761944\tlr=0.010000\n",
      "[Epoch 5] training: accuracy=0.761944\n",
      "[Epoch 5] speed: 100 samples/sec\ttime cost: 71.562163\n",
      "[Epoch 5] validation: top1=0.900000 top5=0.996250\n",
      "Epoch[6] Batch [49]\tSpeed: 97.788799 samples/sec\taccuracy=0.797500\tlr=0.010000\n",
      "Epoch[6] Batch [99]\tSpeed: 101.087799 samples/sec\taccuracy=0.800000\tlr=0.010000\n",
      "Epoch[6] Batch [149]\tSpeed: 101.122855 samples/sec\taccuracy=0.806667\tlr=0.010000\n",
      "Epoch[6] Batch [199]\tSpeed: 100.827412 samples/sec\taccuracy=0.807500\tlr=0.010000\n",
      "Epoch[6] Batch [249]\tSpeed: 100.853289 samples/sec\taccuracy=0.804000\tlr=0.010000\n",
      "Epoch[6] Batch [299]\tSpeed: 100.904068 samples/sec\taccuracy=0.807292\tlr=0.010000\n",
      "Epoch[6] Batch [349]\tSpeed: 100.944189 samples/sec\taccuracy=0.810179\tlr=0.010000\n",
      "Epoch[6] Batch [399]\tSpeed: 101.033026 samples/sec\taccuracy=0.811406\tlr=0.010000\n",
      "Epoch[6] Batch [449]\tSpeed: 100.971172 samples/sec\taccuracy=0.809028\tlr=0.010000\n",
      "[Epoch 6] training: accuracy=0.809028\n",
      "[Epoch 6] speed: 100 samples/sec\ttime cost: 71.576859\n",
      "[Epoch 6] validation: top1=0.930000 top5=0.998750\n",
      "Epoch[7] Batch [49]\tSpeed: 97.568362 samples/sec\taccuracy=0.831250\tlr=0.010000\n",
      "Epoch[7] Batch [99]\tSpeed: 100.912289 samples/sec\taccuracy=0.826250\tlr=0.010000\n",
      "Epoch[7] Batch [149]\tSpeed: 101.126177 samples/sec\taccuracy=0.827500\tlr=0.010000\n",
      "Epoch[7] Batch [199]\tSpeed: 100.935213 samples/sec\taccuracy=0.829688\tlr=0.010000\n",
      "Epoch[7] Batch [249]\tSpeed: 100.926949 samples/sec\taccuracy=0.828250\tlr=0.010000\n",
      "Epoch[7] Batch [299]\tSpeed: 100.820068 samples/sec\taccuracy=0.829792\tlr=0.010000\n",
      "Epoch[7] Batch [349]\tSpeed: 100.955484 samples/sec\taccuracy=0.831786\tlr=0.010000\n",
      "Epoch[7] Batch [399]\tSpeed: 100.984908 samples/sec\taccuracy=0.832969\tlr=0.010000\n",
      "Epoch[7] Batch [449]\tSpeed: 100.885826 samples/sec\taccuracy=0.833750\tlr=0.010000\n",
      "[Epoch 7] training: accuracy=0.833750\n",
      "[Epoch 7] speed: 100 samples/sec\ttime cost: 71.611111\n",
      "[Epoch 7] validation: top1=0.951250 top5=0.996250\n",
      "[Epoch 7] Current best top-1: 0.951250 vs previous 0.950000, saved to /datasets/ulyana/multiannotator_benchmarks/c2314db8/.trial_0/best_checkpoint.pkl\n",
      "Epoch[8] Batch [49]\tSpeed: 97.724357 samples/sec\taccuracy=0.841250\tlr=0.010000\n",
      "Epoch[8] Batch [99]\tSpeed: 101.063451 samples/sec\taccuracy=0.833750\tlr=0.010000\n",
      "Epoch[8] Batch [149]\tSpeed: 101.217463 samples/sec\taccuracy=0.835417\tlr=0.010000\n",
      "Epoch[8] Batch [199]\tSpeed: 100.916744 samples/sec\taccuracy=0.830625\tlr=0.010000\n",
      "Epoch[8] Batch [249]\tSpeed: 100.881980 samples/sec\taccuracy=0.833000\tlr=0.010000\n",
      "Epoch[8] Batch [299]\tSpeed: 100.828524 samples/sec\taccuracy=0.833750\tlr=0.010000\n",
      "Epoch[8] Batch [349]\tSpeed: 100.828642 samples/sec\taccuracy=0.833929\tlr=0.010000\n",
      "Epoch[8] Batch [399]\tSpeed: 100.958914 samples/sec\taccuracy=0.833594\tlr=0.010000\n",
      "Epoch[8] Batch [449]\tSpeed: 100.953361 samples/sec\taccuracy=0.830833\tlr=0.010000\n",
      "[Epoch 8] training: accuracy=0.830833\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Epoch 8] speed: 100 samples/sec\ttime cost: 71.589633\n",
      "[Epoch 8] validation: top1=0.942500 top5=0.997500\n",
      "Epoch[9] Batch [49]\tSpeed: 97.291750 samples/sec\taccuracy=0.856250\tlr=0.010000\n",
      "Epoch[9] Batch [99]\tSpeed: 101.109530 samples/sec\taccuracy=0.860625\tlr=0.010000\n",
      "Epoch[9] Batch [149]\tSpeed: 101.198673 samples/sec\taccuracy=0.857500\tlr=0.010000\n",
      "Epoch[9] Batch [199]\tSpeed: 100.934807 samples/sec\taccuracy=0.853750\tlr=0.010000\n",
      "Epoch[9] Batch [249]\tSpeed: 100.877409 samples/sec\taccuracy=0.852250\tlr=0.010000\n",
      "Epoch[9] Batch [299]\tSpeed: 100.906532 samples/sec\taccuracy=0.851667\tlr=0.010000\n",
      "Epoch[9] Batch [349]\tSpeed: 100.975444 samples/sec\taccuracy=0.854286\tlr=0.010000\n",
      "Epoch[9] Batch [399]\tSpeed: 101.091609 samples/sec\taccuracy=0.851562\tlr=0.010000\n",
      "Epoch[9] Batch [449]\tSpeed: 100.972099 samples/sec\taccuracy=0.850694\tlr=0.010000\n",
      "[Epoch 9] training: accuracy=0.850694\n",
      "[Epoch 9] speed: 100 samples/sec\ttime cost: 71.593703\n",
      "[Epoch 9] validation: top1=0.952500 top5=1.000000\n",
      "[Epoch 9] Current best top-1: 0.952500 vs previous 0.951250, saved to /datasets/ulyana/multiannotator_benchmarks/c2314db8/.trial_0/best_checkpoint.pkl\n",
      "Epoch[10] Batch [49]\tSpeed: 97.920630 samples/sec\taccuracy=0.850000\tlr=0.010000\n",
      "Epoch[10] Batch [99]\tSpeed: 101.047822 samples/sec\taccuracy=0.858125\tlr=0.010000\n",
      "Epoch[10] Batch [149]\tSpeed: 101.122102 samples/sec\taccuracy=0.853750\tlr=0.010000\n",
      "Epoch[10] Batch [199]\tSpeed: 100.833608 samples/sec\taccuracy=0.850313\tlr=0.010000\n",
      "Epoch[10] Batch [249]\tSpeed: 100.924618 samples/sec\taccuracy=0.849750\tlr=0.010000\n",
      "Epoch[10] Batch [299]\tSpeed: 100.864570 samples/sec\taccuracy=0.850625\tlr=0.010000\n",
      "Epoch[10] Batch [349]\tSpeed: 101.075628 samples/sec\taccuracy=0.847679\tlr=0.010000\n",
      "Epoch[10] Batch [399]\tSpeed: 101.077966 samples/sec\taccuracy=0.848906\tlr=0.010000\n",
      "Epoch[10] Batch [449]\tSpeed: 101.044652 samples/sec\taccuracy=0.847778\tlr=0.010000\n",
      "[Epoch 10] training: accuracy=0.847778\n",
      "[Epoch 10] speed: 100 samples/sec\ttime cost: 71.546618\n",
      "[Epoch 10] validation: top1=0.927500 top5=0.996250\n",
      "Epoch[11] Batch [49]\tSpeed: 97.480289 samples/sec\taccuracy=0.856250\tlr=0.010000\n",
      "Epoch[11] Batch [99]\tSpeed: 100.971993 samples/sec\taccuracy=0.860000\tlr=0.010000\n",
      "Epoch[11] Batch [149]\tSpeed: 101.164904 samples/sec\taccuracy=0.854167\tlr=0.010000\n",
      "Epoch[11] Batch [199]\tSpeed: 100.782334 samples/sec\taccuracy=0.854375\tlr=0.010000\n",
      "Epoch[11] Batch [249]\tSpeed: 100.955299 samples/sec\taccuracy=0.853750\tlr=0.010000\n",
      "Epoch[11] Batch [299]\tSpeed: 100.876748 samples/sec\taccuracy=0.852292\tlr=0.010000\n",
      "Epoch[11] Batch [349]\tSpeed: 100.985564 samples/sec\taccuracy=0.849464\tlr=0.010000\n",
      "Epoch[11] Batch [399]\tSpeed: 101.058623 samples/sec\taccuracy=0.849531\tlr=0.010000\n",
      "Epoch[11] Batch [449]\tSpeed: 101.079559 samples/sec\taccuracy=0.844028\tlr=0.010000\n",
      "[Epoch 11] training: accuracy=0.844028\n",
      "[Epoch 11] speed: 100 samples/sec\ttime cost: 71.592330\n",
      "[Epoch 11] validation: top1=0.916250 top5=0.993750\n",
      "Epoch[12] Batch [49]\tSpeed: 97.899666 samples/sec\taccuracy=0.813750\tlr=0.010000\n",
      "Epoch[12] Batch [99]\tSpeed: 101.057174 samples/sec\taccuracy=0.830000\tlr=0.010000\n",
      "Epoch[12] Batch [149]\tSpeed: 101.185829 samples/sec\taccuracy=0.845833\tlr=0.010000\n",
      "Epoch[12] Batch [199]\tSpeed: 100.855051 samples/sec\taccuracy=0.845625\tlr=0.010000\n",
      "Epoch[12] Batch [249]\tSpeed: 100.867630 samples/sec\taccuracy=0.837500\tlr=0.010000\n",
      "Epoch[12] Batch [299]\tSpeed: 100.904833 samples/sec\taccuracy=0.837500\tlr=0.010000\n",
      "Epoch[12] Batch [349]\tSpeed: 100.899963 samples/sec\taccuracy=0.837857\tlr=0.010000\n",
      "Epoch[12] Batch [399]\tSpeed: 101.026815 samples/sec\taccuracy=0.840156\tlr=0.010000\n",
      "Epoch[12] Batch [449]\tSpeed: 100.979936 samples/sec\taccuracy=0.840694\tlr=0.010000\n",
      "[Epoch 12] training: accuracy=0.840694\n",
      "[Epoch 12] speed: 100 samples/sec\ttime cost: 71.565134\n",
      "[Epoch 12] validation: top1=0.927500 top5=0.996250\n",
      "Epoch[13] Batch [49]\tSpeed: 97.154415 samples/sec\taccuracy=0.855000\tlr=0.010000\n",
      "Epoch[13] Batch [99]\tSpeed: 100.947275 samples/sec\taccuracy=0.869375\tlr=0.010000\n",
      "Epoch[13] Batch [149]\tSpeed: 101.015732 samples/sec\taccuracy=0.865833\tlr=0.010000\n",
      "Epoch[13] Batch [199]\tSpeed: 100.819959 samples/sec\taccuracy=0.867812\tlr=0.010000\n",
      "Epoch[13] Batch [249]\tSpeed: 100.869925 samples/sec\taccuracy=0.867000\tlr=0.010000\n",
      "Epoch[13] Batch [299]\tSpeed: 100.810481 samples/sec\taccuracy=0.866250\tlr=0.010000\n",
      "Epoch[13] Batch [349]\tSpeed: 100.918526 samples/sec\taccuracy=0.865714\tlr=0.010000\n",
      "Epoch[13] Batch [399]\tSpeed: 101.029403 samples/sec\taccuracy=0.863906\tlr=0.010000\n",
      "Epoch[13] Batch [449]\tSpeed: 100.955724 samples/sec\taccuracy=0.862917\tlr=0.010000\n",
      "[Epoch 13] training: accuracy=0.862917\n",
      "[Epoch 13] speed: 100 samples/sec\ttime cost: 71.657671\n",
      "[Epoch 13] validation: top1=0.950000 top5=1.000000\n",
      "Epoch[14] Batch [49]\tSpeed: 97.714761 samples/sec\taccuracy=0.871250\tlr=0.010000\n",
      "Epoch[14] Batch [99]\tSpeed: 101.013147 samples/sec\taccuracy=0.866250\tlr=0.010000\n",
      "Epoch[14] Batch [149]\tSpeed: 101.235141 samples/sec\taccuracy=0.859167\tlr=0.010000\n",
      "Epoch[14] Batch [199]\tSpeed: 100.918295 samples/sec\taccuracy=0.860000\tlr=0.010000\n",
      "Epoch[14] Batch [249]\tSpeed: 100.934867 samples/sec\taccuracy=0.864000\tlr=0.010000\n",
      "Epoch[14] Batch [299]\tSpeed: 100.936889 samples/sec\taccuracy=0.864792\tlr=0.010000\n",
      "Epoch[14] Batch [349]\tSpeed: 100.993555 samples/sec\taccuracy=0.867857\tlr=0.010000\n",
      "Epoch[14] Batch [399]\tSpeed: 100.998401 samples/sec\taccuracy=0.866094\tlr=0.010000\n",
      "Epoch[14] Batch [449]\tSpeed: 101.018226 samples/sec\taccuracy=0.865278\tlr=0.010000\n",
      "[Epoch 14] training: accuracy=0.865278\n",
      "[Epoch 14] speed: 100 samples/sec\ttime cost: 71.558230\n",
      "[Epoch 14] validation: top1=0.937500 top5=0.997500\n",
      "Epoch[15] Batch [49]\tSpeed: 98.076106 samples/sec\taccuracy=0.855000\tlr=0.010000\n",
      "Epoch[15] Batch [99]\tSpeed: 101.058456 samples/sec\taccuracy=0.855625\tlr=0.010000\n",
      "Epoch[15] Batch [149]\tSpeed: 101.209906 samples/sec\taccuracy=0.863333\tlr=0.010000\n",
      "Epoch[15] Batch [199]\tSpeed: 100.895536 samples/sec\taccuracy=0.866250\tlr=0.010000\n",
      "Epoch[15] Batch [249]\tSpeed: 100.958428 samples/sec\taccuracy=0.867250\tlr=0.010000\n",
      "Epoch[15] Batch [299]\tSpeed: 100.866320 samples/sec\taccuracy=0.867083\tlr=0.010000\n",
      "Epoch[15] Batch [349]\tSpeed: 101.001675 samples/sec\taccuracy=0.867679\tlr=0.010000\n",
      "Epoch[15] Batch [399]\tSpeed: 101.015647 samples/sec\taccuracy=0.866094\tlr=0.010000\n",
      "Epoch[15] Batch [449]\tSpeed: 101.015075 samples/sec\taccuracy=0.868333\tlr=0.010000\n",
      "[Epoch 15] training: accuracy=0.868333\n",
      "[Epoch 15] speed: 100 samples/sec\ttime cost: 71.530635\n",
      "[Epoch 15] validation: top1=0.943750 top5=0.998750\n",
      "Epoch[16] Batch [49]\tSpeed: 97.840991 samples/sec\taccuracy=0.882500\tlr=0.010000\n",
      "Epoch[16] Batch [99]\tSpeed: 101.017456 samples/sec\taccuracy=0.883125\tlr=0.010000\n",
      "Epoch[16] Batch [149]\tSpeed: 101.204064 samples/sec\taccuracy=0.883333\tlr=0.010000\n",
      "Epoch[16] Batch [199]\tSpeed: 100.873888 samples/sec\taccuracy=0.880938\tlr=0.010000\n",
      "Epoch[16] Batch [249]\tSpeed: 100.912611 samples/sec\taccuracy=0.881750\tlr=0.010000\n",
      "Epoch[16] Batch [299]\tSpeed: 100.877200 samples/sec\taccuracy=0.878125\tlr=0.010000\n",
      "Epoch[16] Batch [349]\tSpeed: 100.936088 samples/sec\taccuracy=0.877857\tlr=0.010000\n",
      "Epoch[16] Batch [399]\tSpeed: 100.988929 samples/sec\taccuracy=0.877188\tlr=0.010000\n",
      "Epoch[16] Batch [449]\tSpeed: 100.982109 samples/sec\taccuracy=0.877222\tlr=0.010000\n",
      "[Epoch 16] training: accuracy=0.877222\n",
      "[Epoch 16] speed: 100 samples/sec\ttime cost: 71.568277\n",
      "[Epoch 16] validation: top1=0.940000 top5=0.995000\n",
      "Epoch[17] Batch [49]\tSpeed: 97.625335 samples/sec\taccuracy=0.873750\tlr=0.010000\n",
      "Epoch[17] Batch [99]\tSpeed: 100.919412 samples/sec\taccuracy=0.873125\tlr=0.010000\n",
      "Epoch[17] Batch [149]\tSpeed: 101.186817 samples/sec\taccuracy=0.876667\tlr=0.010000\n",
      "Epoch[17] Batch [199]\tSpeed: 100.846403 samples/sec\taccuracy=0.877812\tlr=0.010000\n",
      "Epoch[17] Batch [249]\tSpeed: 100.879569 samples/sec\taccuracy=0.882250\tlr=0.010000\n",
      "Epoch[17] Batch [299]\tSpeed: 100.857321 samples/sec\taccuracy=0.881250\tlr=0.010000\n",
      "Epoch[17] Batch [349]\tSpeed: 101.035640 samples/sec\taccuracy=0.879821\tlr=0.010000\n",
      "Epoch[17] Batch [399]\tSpeed: 101.043669 samples/sec\taccuracy=0.879531\tlr=0.010000\n",
      "Epoch[17] Batch [449]\tSpeed: 100.992193 samples/sec\taccuracy=0.880694\tlr=0.010000\n",
      "[Epoch 17] training: accuracy=0.880694\n",
      "[Epoch 17] speed: 100 samples/sec\ttime cost: 71.588001\n",
      "[Epoch 17] validation: top1=0.925000 top5=0.998750\n",
      "Epoch[18] Batch [49]\tSpeed: 97.592466 samples/sec\taccuracy=0.882500\tlr=0.010000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch[18] Batch [99]\tSpeed: 100.988920 samples/sec\taccuracy=0.875000\tlr=0.010000\n",
      "Epoch[18] Batch [149]\tSpeed: 101.216391 samples/sec\taccuracy=0.876667\tlr=0.010000\n",
      "Epoch[18] Batch [199]\tSpeed: 100.942862 samples/sec\taccuracy=0.877188\tlr=0.010000\n",
      "Epoch[18] Batch [249]\tSpeed: 100.933574 samples/sec\taccuracy=0.877500\tlr=0.010000\n",
      "Epoch[18] Batch [299]\tSpeed: 100.902697 samples/sec\taccuracy=0.878542\tlr=0.010000\n",
      "Epoch[18] Batch [349]\tSpeed: 101.039528 samples/sec\taccuracy=0.878750\tlr=0.010000\n",
      "Epoch[18] Batch [399]\tSpeed: 101.072568 samples/sec\taccuracy=0.879531\tlr=0.010000\n",
      "Epoch[18] Batch [449]\tSpeed: 101.029449 samples/sec\taccuracy=0.879306\tlr=0.010000\n",
      "[Epoch 18] training: accuracy=0.879306\n",
      "[Epoch 18] speed: 100 samples/sec\ttime cost: 71.562829\n",
      "[Epoch 18] validation: top1=0.932500 top5=0.996250\n",
      "Epoch[19] Batch [49]\tSpeed: 97.809194 samples/sec\taccuracy=0.900000\tlr=0.010000\n",
      "Epoch[19] Batch [99]\tSpeed: 101.056858 samples/sec\taccuracy=0.891875\tlr=0.010000\n",
      "Epoch[19] Batch [149]\tSpeed: 101.328349 samples/sec\taccuracy=0.892917\tlr=0.010000\n",
      "Epoch[19] Batch [199]\tSpeed: 100.941420 samples/sec\taccuracy=0.885312\tlr=0.010000\n",
      "Epoch[19] Batch [249]\tSpeed: 100.941493 samples/sec\taccuracy=0.886250\tlr=0.010000\n",
      "Epoch[19] Batch [299]\tSpeed: 100.936170 samples/sec\taccuracy=0.886875\tlr=0.010000\n",
      "Epoch[19] Batch [349]\tSpeed: 101.008169 samples/sec\taccuracy=0.885536\tlr=0.010000\n",
      "Epoch[19] Batch [399]\tSpeed: 100.986628 samples/sec\taccuracy=0.883281\tlr=0.010000\n",
      "Epoch[19] Batch [449]\tSpeed: 100.974311 samples/sec\taccuracy=0.883194\tlr=0.010000\n",
      "[Epoch 19] training: accuracy=0.883194\n",
      "[Epoch 19] speed: 100 samples/sec\ttime cost: 71.541051\n",
      "[Epoch 19] validation: top1=0.943750 top5=0.998750\n",
      "[Epoch 20] EarlyStop after 10 epochs: no better than 0.9525\n",
      "Applying the state from the best checkpoint...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_4/_test_pred_probs_split_4\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_4/_test_pred_features_split_4\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_4/_test_labels_split_4\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_4/_test_image_files_split_4\n",
      "Saving ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_4/_test_indices_split_4\n",
      "CPU times: user 1h 21min 1s, sys: 19min 38s, total: 1h 40min 39s\n",
      "Wall time: 1h 39min 7s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# run cross-validation for each model\n",
    "for model in models:\n",
    "    \n",
    "    print(\"----\")\n",
    "    print(f\"Running cross-validation for model: {model}\")\n",
    "\n",
    "    MODEL_PARAMS = {\n",
    "        \"model\": model,\n",
    "        \"epochs\": epochs,\n",
    "        \"holdout_frac\": holdout_frac,\n",
    "    }\n",
    "\n",
    "    # results of cross-validation will be saved to pickle files for each model/fold\n",
    "    _ = \\\n",
    "        cross_val_predict_autogluon_image_dataset(\n",
    "            dataset=df,\n",
    "            out_folder=f\"{model_folder}_{model}/\", # save results of cross-validation in pickle files for each fold\n",
    "            n_splits=num_cv_folds,\n",
    "            model_params=MODEL_PARAMS,\n",
    "            time_limit=time_limit,\n",
    "            random_state=random_state,\n",
    "        )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc171378",
   "metadata": {},
   "source": [
    "## Read per-fold pickle files from xvalidation and save data as numpy arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "23da39c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load pickle file util\n",
    "def load_pickle(pickle_file_name, verbose=1):\n",
    "    \"\"\"Load pickle file\"\"\"\n",
    "\n",
    "    if verbose:\n",
    "        print(f\"Loading {pickle_file_name}\")\n",
    "\n",
    "    with open(pickle_file_name, 'rb') as handle:\n",
    "        out = pickle.load(handle)\n",
    "        \n",
    "    return out\n",
    "\n",
    "# get the original label from file path (aka \"true labels\" y)\n",
    "get_orig_label_idx_from_file_path = np.vectorize(lambda f: label_name_to_idx_map[Path(f).parts[-2]])\n",
    "\n",
    "# get original label name to idx mapping\n",
    "label_name_to_idx_map = {'airplane': 0,\n",
    "                         'automobile': 1,\n",
    "                         'bird': 2,\n",
    "                         'cat': 3,\n",
    "                         'deer': 4,\n",
    "                         'dog': 5,\n",
    "                         'frog': 6,\n",
    "                         'horse': 7,\n",
    "                         'ship': 8,\n",
    "                         'truck': 9}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02692c32",
   "metadata": {},
   "source": [
    "#### Save pickle files per fold as single files per model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f685c794",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_0/_test_pred_probs_split_0\n",
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_0/_test_labels_split_0\n",
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_0/_test_image_files_split_0\n",
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_0/_test_indices_split_0\n",
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_1/_test_pred_probs_split_1\n",
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_1/_test_labels_split_1\n",
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_1/_test_image_files_split_1\n",
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_1/_test_indices_split_1\n",
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_2/_test_pred_probs_split_2\n",
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_2/_test_labels_split_2\n",
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_2/_test_image_files_split_2\n",
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_2/_test_indices_split_2\n",
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_3/_test_pred_probs_split_3\n",
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_3/_test_labels_split_3\n",
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_3/_test_image_files_split_3\n",
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_3/_test_indices_split_3\n",
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_4/_test_pred_probs_split_4\n",
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_4/_test_labels_split_4\n",
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_4/_test_image_files_split_4\n",
      "Loading ./data/cifar10_truelabels_swin_base_patch4_window7_224/split_4/_test_indices_split_4\n",
      "Saving to numpy files in this folder: ./data/cifar10_truelabels_swin_base_patch4_window7_224/\n",
      "Model: swin_base_patch4_window7_224\n",
      "  Accuracy (argmax pred vs labels): 0.9484\n",
      "  Accuracy (argmax pred vs true labels) : 0.9484\n",
      "  Accuracy (labels vs true labels)       : 1.0\n"
     ]
    }
   ],
   "source": [
    "results_list = []\n",
    "\n",
    "for model in models:\n",
    "\n",
    "    pred_probs = []\n",
    "    labels = []\n",
    "    images = []\n",
    "    \n",
    "    for split_num in range(num_cv_folds):\n",
    "\n",
    "        out_subfolder = f\"{model_folder}_{model}/split_{split_num}/\"\n",
    "        \n",
    "        # pickle file name to read\n",
    "        get_pickle_file_name = (\n",
    "            lambda object_name: f\"{out_subfolder}_{object_name}_split_{split_num}\"\n",
    "        )\n",
    "\n",
    "        # NOTE: the \"test_\" prefix in the pickle name correspond to the \"test\" split during cross-validation.\n",
    "        pred_probs_split = load_pickle(get_pickle_file_name(\"test_pred_probs\"), verbose=verbose)\n",
    "        labels_split = load_pickle(get_pickle_file_name(\"test_labels\"), verbose=verbose)\n",
    "        images_split = load_pickle(get_pickle_file_name(\"test_image_files\"), verbose=verbose)\n",
    "        indices_split = load_pickle(get_pickle_file_name(\"test_indices\"), verbose=verbose)\n",
    "\n",
    "        # append to list so we can combine data from all the splits\n",
    "        pred_probs.append(pred_probs_split)\n",
    "        labels.append(labels_split)\n",
    "        images.append(images_split)    \n",
    "\n",
    "    # convert list to array\n",
    "    pred_probs = np.vstack(pred_probs)\n",
    "    labels = np.hstack(labels) # remember that this is the noisy labels (s)\n",
    "    images = np.hstack(images)\n",
    "    \n",
    "    # get the true labels (y) from the original file path\n",
    "    true_labels = get_orig_label_idx_from_file_path(images)\n",
    "    \n",
    "    # save to Numpy files\n",
    "    numpy_out_folder = f\"{model_folder}_{model}/\"\n",
    "    \n",
    "    print(f\"Saving to numpy files in this folder: {numpy_out_folder}\")\n",
    "    \n",
    "    np.save(numpy_out_folder + \"pred_probs\", pred_probs)\n",
    "    np.save(numpy_out_folder + \"labels\", labels)\n",
    "    np.save(numpy_out_folder + \"images\", images)\n",
    "    np.save(numpy_out_folder + \"true_labels\", true_labels)\n",
    "\n",
    "    # check the accuracy\n",
    "    acc_labels = (pred_probs.argmax(axis=1) == labels).mean() # noisy labels (s)\n",
    "    acc_true_labels = (pred_probs.argmax(axis=1) == true_labels).mean() # true labels (y)    \n",
    "    acc_noisy_vs_true_labels = (labels == true_labels).mean()\n",
    "    \n",
    "    print(f\"Model: {model}\")\n",
    "    print(f\"  Accuracy (argmax pred vs labels): {acc_labels}\")\n",
    "    print(f\"  Accuracy (argmax pred vs true labels) : {acc_true_labels}\")\n",
    "    print(f\"  Accuracy (labels vs true labels)       : {acc_noisy_vs_true_labels}\")\n",
    "    \n",
    "    results = {\n",
    "        \"model\": model,\n",
    "        \"Accuracy (argmax pred vs noisy labels)\": acc_labels,\n",
    "        \"Accuracy (argmax pred vs true labels)\": acc_true_labels,\n",
    "        \"Accuracy (noisy vs true labels)\": acc_noisy_vs_true_labels\n",
    "    }\n",
    "    \n",
    "    results_list.append(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "41ecf31c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "swin_base_patch4_window7_224\n",
      " pred_probs[(10000, 10)],labels[(10000,)], true_labels[(10000,)]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model in models:\n",
    "    numpy_out_folder = f\"{model_folder}_{model}/\"\n",
    "\n",
    "    pred_probs = np.load(numpy_out_folder + 'pred_probs.npy')\n",
    "    labels = np.load(numpy_out_folder + 'labels.npy')\n",
    "    true_labels =  np.load(numpy_out_folder + 'true_labels.npy')\n",
    "    print(f'{model}\\n pred_probs[{pred_probs.shape}],labels[{labels.shape}], true_labels[{true_labels.shape}]\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.6 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}