text.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os \n",
    "import sys\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"MIG-GPU-bb1ccb6e-2bc9-c7a1-b25d-3eef9033e192/6/0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "This requires torchtext\n",
    "\"\"\"\n",
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "from torchtext.data.utils import get_tokenizer\n",
    "from torchtext.vocab import build_vocab_from_iterator\n",
    "\n",
    "from pytorch_ood.dataset.txt import NewsGroup20, Reuters52, WMT16Sentences, Multi30k\n",
    "from pytorch_ood.model.gruclf import GRUClassifier\n",
    "from pytorch_ood.utils import ToUnknown, OODMetrics\n",
    "from pytorch_ood.detector import MaxSoftmax, EnergyBased\n",
    "\n",
    "torch.manual_seed(123)\n",
    "\n",
    "n_epochs = 10\n",
    "lr = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# download datasets\n",
    "train_dataset = NewsGroup20(\"data/\", train=True, download=True)\n",
    "test_dataset = NewsGroup20(\"data/\", train=False, download=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "11293lines [00:00, 15999.73lines/s]\n"
     ]
    }
   ],
   "source": [
    "tokenizer = get_tokenizer('basic_english')\n",
    "\n",
    "def yield_tokens(data_iter):\n",
    "    for text, _ in data_iter:\n",
    "        yield tokenizer(text)\n",
    "\n",
    "vocab = build_vocab_from_iterator(yield_tokens(train_dataset)) # , max_tokens=10000# , specials=[\"<unk>\"]\n",
    "# vocab.set_default_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def prep(x):\n",
    "    return torch.tensor([vocab[v] for v in tokenizer(x)], dtype=torch.int64)\n",
    "\n",
    "train_dataset = NewsGroup20(\"data/\", train=True, transform=prep)\n",
    "test_dataset = NewsGroup20(\"data/\", train=False, transform=prep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# add padding, etc.\n",
    "def collate_batch(batch):\n",
    "    texts = [i[0] for i in batch]\n",
    "    labels = torch.tensor([i[1] for i in batch],  dtype=torch.int64)\n",
    "    t_lengths = torch.tensor([len(t) for t in texts])\n",
    "    max_t_length = torch.max(t_lengths)\n",
    "\n",
    "    padded = []\n",
    "    for text in texts:\n",
    "        t = torch.cat([torch.zeros(max_t_length-len(text), dtype=torch.long), text])\n",
    "        padded.append(t)\n",
    "    return torch.stack(padded,dim=0), labels\n",
    "\n",
    "train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True, collate_fn=collate_batch)\n",
    "test_loader = DataLoader(test_dataset, batch_size=256, shuffle=True, collate_fn=collate_batch)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "model = GRUClassifier(num_classes=20, n_vocab=len(vocab))\n",
    "model.cuda()\n",
    "\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=n_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0\n",
      "Loss: 0.30 Accuracy 12.50%\n",
      "Loss: 2.05 Accuracy 5.40%\n",
      "Loss: 2.67 Accuracy 5.80%\n",
      "Loss: 2.86 Accuracy 6.65%\n",
      "Loss: 2.93 Accuracy 7.39%\n",
      "Loss: 2.96 Accuracy 8.21%\n",
      "Loss: 2.96 Accuracy 8.76%\n",
      "Loss: 2.94 Accuracy 9.24%\n",
      "Loss: 2.95 Accuracy 9.34%\n",
      "Loss: 2.93 Accuracy 9.65%\n",
      "Loss: 2.89 Accuracy 9.93%\n",
      "Loss: 2.88 Accuracy 10.28%\n",
      "Loss: 2.90 Accuracy 10.15%\n",
      "Loss: 2.88 Accuracy 10.47%\n",
      "Loss: 2.88 Accuracy 10.73%\n",
      "Loss: 2.88 Accuracy 10.78%\n",
      "Loss: 2.88 Accuracy 10.85%\n",
      "Loss: 2.86 Accuracy 10.89%\n",
      "Loss: 2.86 Accuracy 11.08%\n",
      "Loss: 2.83 Accuracy 11.65%\n",
      "Loss: 2.81 Accuracy 11.94%\n",
      "Loss: 2.77 Accuracy 12.20%\n",
      "Loss: 2.78 Accuracy 12.34%\n",
      "Loss: 2.73 Accuracy 12.70%\n",
      "Loss: 2.73 Accuracy 12.97%\n",
      "Loss: 2.67 Accuracy 13.22%\n",
      "Loss: 2.67 Accuracy 13.47%\n",
      "Loss: 2.65 Accuracy 13.63%\n",
      "Loss: 2.69 Accuracy 13.69%\n",
      "Loss: 2.59 Accuracy 14.01%\n",
      "Loss: 2.56 Accuracy 14.34%\n",
      "Loss: 2.62 Accuracy 14.52%\n",
      "Loss: 2.56 Accuracy 14.80%\n",
      "Loss: 2.58 Accuracy 14.96%\n",
      "Loss: 2.57 Accuracy 15.19%\n",
      "Loss: 2.55 Accuracy 15.40%\n",
      "Test Accuracy: 19.63%\n",
      "Epoch 1\n",
      "Loss: 0.23 Accuracy 34.38%\n",
      "Loss: 1.64 Accuracy 26.70%\n",
      "Loss: 2.07 Accuracy 27.98%\n",
      "Loss: 2.29 Accuracy 27.32%\n",
      "Loss: 2.35 Accuracy 27.29%\n",
      "Loss: 2.28 Accuracy 27.82%\n",
      "Loss: 2.21 Accuracy 28.33%\n",
      "Loss: 2.17 Accuracy 28.83%\n",
      "Loss: 2.18 Accuracy 29.40%\n",
      "Loss: 2.21 Accuracy 29.60%\n",
      "Loss: 2.12 Accuracy 30.14%\n",
      "Loss: 2.12 Accuracy 30.46%\n",
      "Loss: 2.13 Accuracy 30.86%\n",
      "Loss: 2.14 Accuracy 30.84%\n",
      "Loss: 2.06 Accuracy 31.21%\n",
      "Loss: 2.08 Accuracy 31.42%\n",
      "Loss: 2.08 Accuracy 31.54%\n",
      "Loss: 2.11 Accuracy 31.58%\n",
      "Loss: 2.06 Accuracy 31.94%\n",
      "Loss: 1.93 Accuracy 32.51%\n",
      "Loss: 1.96 Accuracy 32.79%\n",
      "Loss: 1.96 Accuracy 33.19%\n",
      "Loss: 2.04 Accuracy 33.34%\n",
      "Loss: 1.94 Accuracy 33.64%\n",
      "Loss: 1.96 Accuracy 33.83%\n",
      "Loss: 1.88 Accuracy 34.04%\n",
      "Loss: 1.96 Accuracy 34.26%\n",
      "Loss: 1.90 Accuracy 34.40%\n",
      "Loss: 1.82 Accuracy 34.58%\n",
      "Loss: 1.76 Accuracy 34.91%\n",
      "Loss: 1.74 Accuracy 35.18%\n",
      "Loss: 1.78 Accuracy 35.37%\n",
      "Loss: 1.80 Accuracy 35.54%\n",
      "Loss: 1.79 Accuracy 35.64%\n",
      "Loss: 1.74 Accuracy 35.89%\n",
      "Loss: 1.80 Accuracy 36.00%\n",
      "Test Accuracy: 32.88%\n",
      "Epoch 2\n",
      "Loss: 0.15 Accuracy 53.12%\n",
      "Loss: 1.13 Accuracy 47.73%\n",
      "Loss: 1.36 Accuracy 49.11%\n",
      "Loss: 1.50 Accuracy 49.29%\n",
      "Loss: 1.47 Accuracy 49.92%\n",
      "Loss: 1.49 Accuracy 50.43%\n",
      "Loss: 1.47 Accuracy 50.67%\n",
      "Loss: 1.53 Accuracy 50.62%\n",
      "Loss: 1.52 Accuracy 51.04%\n",
      "Loss: 1.40 Accuracy 51.99%\n",
      "Loss: 1.41 Accuracy 52.10%\n",
      "Loss: 1.45 Accuracy 52.03%\n",
      "Loss: 1.47 Accuracy 52.20%\n",
      "Loss: 1.44 Accuracy 52.27%\n",
      "Loss: 1.46 Accuracy 52.39%\n",
      "Loss: 1.32 Accuracy 52.84%\n",
      "Loss: 1.36 Accuracy 53.05%\n",
      "Loss: 1.25 Accuracy 53.45%\n",
      "Loss: 1.24 Accuracy 53.92%\n",
      "Loss: 1.25 Accuracy 54.22%\n",
      "Loss: 1.26 Accuracy 54.45%\n",
      "Loss: 1.29 Accuracy 54.50%\n",
      "Loss: 1.31 Accuracy 54.62%\n",
      "Loss: 1.31 Accuracy 54.71%\n",
      "Loss: 1.32 Accuracy 54.72%\n",
      "Loss: 1.40 Accuracy 54.76%\n",
      "Loss: 1.28 Accuracy 54.84%\n",
      "Loss: 1.26 Accuracy 54.98%\n",
      "Loss: 1.30 Accuracy 55.13%\n",
      "Loss: 1.21 Accuracy 55.47%\n",
      "Loss: 1.28 Accuracy 55.55%\n",
      "Loss: 1.28 Accuracy 55.70%\n",
      "Loss: 1.25 Accuracy 55.91%\n",
      "Loss: 1.18 Accuracy 56.13%\n",
      "Loss: 1.22 Accuracy 56.25%\n",
      "Loss: 1.16 Accuracy 56.43%\n",
      "Test Accuracy: 45.83%\n",
      "Epoch 3\n",
      "Loss: 0.09 Accuracy 65.62%\n",
      "Loss: 0.71 Accuracy 63.35%\n",
      "Loss: 0.84 Accuracy 66.96%\n",
      "Loss: 0.88 Accuracy 68.65%\n",
      "Loss: 0.88 Accuracy 69.21%\n",
      "Loss: 0.90 Accuracy 69.49%\n",
      "Loss: 0.91 Accuracy 69.77%\n",
      "Loss: 0.95 Accuracy 69.28%\n",
      "Loss: 0.95 Accuracy 69.37%\n",
      "Loss: 0.95 Accuracy 69.02%\n",
      "Loss: 0.91 Accuracy 69.34%\n",
      "Loss: 0.90 Accuracy 69.37%\n",
      "Loss: 0.89 Accuracy 69.71%\n",
      "Loss: 0.92 Accuracy 69.90%\n",
      "Loss: 0.92 Accuracy 69.90%\n",
      "Loss: 0.89 Accuracy 70.07%\n",
      "Loss: 0.86 Accuracy 70.21%\n",
      "Loss: 0.87 Accuracy 70.45%\n",
      "Loss: 0.85 Accuracy 70.48%\n",
      "Loss: 0.84 Accuracy 70.40%\n",
      "Loss: 0.87 Accuracy 70.52%\n",
      "Loss: 0.83 Accuracy 70.72%\n",
      "Loss: 0.86 Accuracy 70.66%\n",
      "Loss: 0.83 Accuracy 70.83%\n",
      "Loss: 0.91 Accuracy 70.72%\n",
      "Loss: 0.92 Accuracy 70.73%\n",
      "Loss: 1.01 Accuracy 70.64%\n",
      "Loss: 0.94 Accuracy 70.72%\n",
      "Loss: 0.88 Accuracy 70.84%\n",
      "Loss: 0.87 Accuracy 70.78%\n",
      "Loss: 0.88 Accuracy 70.76%\n",
      "Loss: 0.86 Accuracy 70.77%\n",
      "Loss: 0.90 Accuracy 70.68%\n",
      "Loss: 0.86 Accuracy 70.90%\n",
      "Loss: 0.79 Accuracy 71.05%\n",
      "Loss: 0.77 Accuracy 71.16%\n",
      "Test Accuracy: 51.55%\n",
      "Epoch 4\n",
      "Loss: 0.05 Accuracy 84.38%\n",
      "Loss: 0.34 Accuracy 87.50%\n",
      "Loss: 0.49 Accuracy 84.97%\n",
      "Loss: 0.54 Accuracy 84.17%\n",
      "Loss: 0.53 Accuracy 83.38%\n",
      "Loss: 0.57 Accuracy 83.09%\n",
      "Loss: 0.58 Accuracy 82.94%\n",
      "Loss: 0.57 Accuracy 82.83%\n",
      "Loss: 0.58 Accuracy 82.64%\n",
      "Loss: 0.60 Accuracy 82.45%\n",
      "Loss: 0.57 Accuracy 82.46%\n",
      "Loss: 0.57 Accuracy 82.46%\n",
      "Loss: 0.56 Accuracy 82.46%\n",
      "Loss: 0.56 Accuracy 82.35%\n",
      "Loss: 0.56 Accuracy 82.25%\n",
      "Loss: 0.55 Accuracy 82.47%\n",
      "Loss: 0.60 Accuracy 82.57%\n",
      "Loss: 0.58 Accuracy 82.55%\n",
      "Loss: 0.60 Accuracy 82.37%\n",
      "Loss: 0.63 Accuracy 82.07%\n",
      "Loss: 0.54 Accuracy 82.21%\n",
      "Loss: 0.56 Accuracy 82.09%\n",
      "Loss: 0.53 Accuracy 82.23%\n",
      "Loss: 0.51 Accuracy 82.29%\n",
      "Loss: 0.49 Accuracy 82.42%\n",
      "Loss: 0.54 Accuracy 82.43%\n",
      "Loss: 0.59 Accuracy 82.32%\n",
      "Loss: 0.54 Accuracy 82.31%\n",
      "Loss: 0.62 Accuracy 82.25%\n",
      "Loss: 0.59 Accuracy 82.18%\n",
      "Loss: 0.54 Accuracy 82.28%\n",
      "Loss: 0.54 Accuracy 82.32%\n",
      "Loss: 0.53 Accuracy 82.38%\n",
      "Loss: 0.58 Accuracy 82.27%\n",
      "Loss: 0.58 Accuracy 82.17%\n",
      "Loss: 0.67 Accuracy 82.02%\n",
      "Test Accuracy: 50.58%\n",
      "Epoch 5\n",
      "Loss: 0.04 Accuracy 87.50%\n",
      "Loss: 0.24 Accuracy 90.34%\n",
      "Loss: 0.32 Accuracy 90.18%\n",
      "Loss: 0.34 Accuracy 90.02%\n",
      "Loss: 0.38 Accuracy 90.02%\n",
      "Loss: 0.32 Accuracy 90.20%\n",
      "Loss: 0.34 Accuracy 90.06%\n",
      "Loss: 0.31 Accuracy 90.40%\n",
      "Loss: 0.34 Accuracy 90.39%\n",
      "Loss: 0.32 Accuracy 90.38%\n",
      "Loss: 0.36 Accuracy 90.41%\n",
      "Loss: 0.31 Accuracy 90.46%\n",
      "Loss: 0.31 Accuracy 90.65%\n",
      "Loss: 0.30 Accuracy 90.62%\n",
      "Loss: 0.34 Accuracy 90.45%\n",
      "Loss: 0.36 Accuracy 90.31%\n",
      "Loss: 0.37 Accuracy 90.24%\n",
      "Loss: 0.33 Accuracy 90.19%\n",
      "Loss: 0.32 Accuracy 90.25%\n",
      "Loss: 0.31 Accuracy 90.27%\n",
      "Loss: 0.32 Accuracy 90.28%\n",
      "Loss: 0.33 Accuracy 90.21%\n",
      "Loss: 0.35 Accuracy 90.09%\n",
      "Loss: 0.31 Accuracy 90.17%\n",
      "Loss: 0.27 Accuracy 90.27%\n",
      "Loss: 0.33 Accuracy 90.21%\n",
      "Loss: 0.33 Accuracy 90.18%\n",
      "Loss: 0.39 Accuracy 90.06%\n",
      "Loss: 0.40 Accuracy 89.95%\n",
      "Loss: 0.32 Accuracy 89.97%\n",
      "Loss: 0.31 Accuracy 90.06%\n",
      "Loss: 0.31 Accuracy 90.06%\n",
      "Loss: 0.32 Accuracy 90.09%\n",
      "Loss: 0.35 Accuracy 89.99%\n",
      "Loss: 0.31 Accuracy 89.95%\n",
      "Loss: 0.35 Accuracy 89.89%\n",
      "Test Accuracy: 53.27%\n",
      "Epoch 6\n",
      "Loss: 0.01 Accuracy 100.00%\n",
      "Loss: 0.14 Accuracy 94.60%\n",
      "Loss: 0.18 Accuracy 95.24%\n",
      "Loss: 0.16 Accuracy 95.67%\n",
      "Loss: 0.17 Accuracy 95.66%\n",
      "Loss: 0.17 Accuracy 95.59%\n",
      "Loss: 0.19 Accuracy 95.34%\n",
      "Loss: 0.20 Accuracy 95.29%\n",
      "Loss: 0.19 Accuracy 95.22%\n",
      "Loss: 0.23 Accuracy 94.92%\n",
      "Loss: 0.23 Accuracy 94.80%\n",
      "Loss: 0.27 Accuracy 94.65%\n",
      "Loss: 0.26 Accuracy 94.58%\n",
      "Loss: 0.20 Accuracy 94.58%\n",
      "Loss: 0.19 Accuracy 94.50%\n",
      "Loss: 0.17 Accuracy 94.52%\n",
      "Loss: 0.15 Accuracy 94.62%\n",
      "Loss: 0.22 Accuracy 94.55%\n",
      "Loss: 0.21 Accuracy 94.54%\n",
      "Loss: 0.19 Accuracy 94.63%\n",
      "Loss: 0.18 Accuracy 94.73%\n",
      "Loss: 0.16 Accuracy 94.82%\n",
      "Loss: 0.20 Accuracy 94.71%\n",
      "Loss: 0.20 Accuracy 94.70%\n",
      "Loss: 0.22 Accuracy 94.68%\n",
      "Loss: 0.19 Accuracy 94.77%\n",
      "Loss: 0.22 Accuracy 94.64%\n",
      "Loss: 0.18 Accuracy 94.64%\n",
      "Loss: 0.17 Accuracy 94.66%\n",
      "Loss: 0.22 Accuracy 94.57%\n",
      "Loss: 0.17 Accuracy 94.64%\n",
      "Loss: 0.20 Accuracy 94.62%\n",
      "Loss: 0.25 Accuracy 94.50%\n",
      "Loss: 0.19 Accuracy 94.51%\n",
      "Loss: 0.19 Accuracy 94.50%\n",
      "Loss: 0.18 Accuracy 94.47%\n",
      "Test Accuracy: 55.35%\n",
      "Epoch 7\n",
      "Loss: 0.00 Accuracy 100.00%\n",
      "Loss: 0.07 Accuracy 97.73%\n",
      "Loss: 0.11 Accuracy 97.32%\n",
      "Loss: 0.10 Accuracy 97.38%\n",
      "Loss: 0.12 Accuracy 97.26%\n",
      "Loss: 0.13 Accuracy 97.37%\n",
      "Loss: 0.09 Accuracy 97.59%\n",
      "Loss: 0.09 Accuracy 97.58%\n",
      "Loss: 0.10 Accuracy 97.72%\n",
      "Loss: 0.09 Accuracy 97.77%\n",
      "Loss: 0.09 Accuracy 97.80%\n",
      "Loss: 0.08 Accuracy 97.83%\n",
      "Loss: 0.09 Accuracy 97.70%\n",
      "Loss: 0.11 Accuracy 97.57%\n",
      "Loss: 0.12 Accuracy 97.50%\n",
      "Loss: 0.13 Accuracy 97.39%\n",
      "Loss: 0.11 Accuracy 97.36%\n",
      "Loss: 0.10 Accuracy 97.37%\n",
      "Loss: 0.10 Accuracy 97.38%\n",
      "Loss: 0.10 Accuracy 97.33%\n",
      "Loss: 0.10 Accuracy 97.34%\n",
      "Loss: 0.13 Accuracy 97.27%\n",
      "Loss: 0.12 Accuracy 97.29%\n",
      "Loss: 0.15 Accuracy 97.28%\n",
      "Loss: 0.10 Accuracy 97.30%\n",
      "Loss: 0.09 Accuracy 97.35%\n",
      "Loss: 0.10 Accuracy 97.31%\n",
      "Loss: 0.11 Accuracy 97.27%\n",
      "Loss: 0.10 Accuracy 97.25%\n",
      "Loss: 0.08 Accuracy 97.28%\n",
      "Loss: 0.09 Accuracy 97.31%\n",
      "Loss: 0.10 Accuracy 97.32%\n",
      "Loss: 0.10 Accuracy 97.29%\n",
      "Loss: 0.10 Accuracy 97.31%\n",
      "Loss: 0.11 Accuracy 97.28%\n",
      "Loss: 0.11 Accuracy 97.28%\n",
      "Test Accuracy: 56.66%\n",
      "Epoch 8\n",
      "Loss: 0.00 Accuracy 100.00%\n",
      "Loss: 0.07 Accuracy 98.58%\n",
      "Loss: 0.06 Accuracy 98.81%\n",
      "Loss: 0.05 Accuracy 99.09%\n",
      "Loss: 0.05 Accuracy 98.86%\n",
      "Loss: 0.06 Accuracy 98.90%\n",
      "Loss: 0.05 Accuracy 98.87%\n",
      "Loss: 0.05 Accuracy 98.90%\n",
      "Loss: 0.06 Accuracy 98.88%\n",
      "Loss: 0.06 Accuracy 98.90%\n",
      "Loss: 0.04 Accuracy 98.98%\n",
      "Loss: 0.04 Accuracy 98.96%\n",
      "Loss: 0.05 Accuracy 98.99%\n",
      "Loss: 0.05 Accuracy 99.00%\n",
      "Loss: 0.05 Accuracy 98.96%\n",
      "Loss: 0.04 Accuracy 98.99%\n",
      "Loss: 0.05 Accuracy 98.99%\n",
      "Loss: 0.06 Accuracy 98.94%\n",
      "Loss: 0.08 Accuracy 98.86%\n",
      "Loss: 0.06 Accuracy 98.89%\n",
      "Loss: 0.06 Accuracy 98.88%\n",
      "Loss: 0.07 Accuracy 98.84%\n",
      "Loss: 0.06 Accuracy 98.84%\n",
      "Loss: 0.04 Accuracy 98.88%\n",
      "Loss: 0.06 Accuracy 98.87%\n",
      "Loss: 0.05 Accuracy 98.88%\n",
      "Loss: 0.07 Accuracy 98.89%\n",
      "Loss: 0.06 Accuracy 98.89%\n",
      "Loss: 0.05 Accuracy 98.91%\n",
      "Loss: 0.05 Accuracy 98.88%\n",
      "Loss: 0.07 Accuracy 98.87%\n",
      "Loss: 0.09 Accuracy 98.83%\n",
      "Loss: 0.08 Accuracy 98.80%\n",
      "Loss: 0.08 Accuracy 98.80%\n",
      "Loss: 0.07 Accuracy 98.78%\n",
      "Loss: 0.06 Accuracy 98.79%\n",
      "Test Accuracy: 56.15%\n",
      "Epoch 9\n",
      "Loss: 0.00 Accuracy 100.00%\n",
      "Loss: 0.03 Accuracy 99.15%\n",
      "Loss: 0.03 Accuracy 99.40%\n",
      "Loss: 0.04 Accuracy 99.19%\n",
      "Loss: 0.03 Accuracy 99.31%\n",
      "Loss: 0.03 Accuracy 99.26%\n",
      "Loss: 0.03 Accuracy 99.33%\n",
      "Loss: 0.03 Accuracy 99.38%\n",
      "Loss: 0.02 Accuracy 99.42%\n",
      "Loss: 0.03 Accuracy 99.38%\n",
      "Loss: 0.04 Accuracy 99.32%\n",
      "Loss: 0.03 Accuracy 99.32%\n",
      "Loss: 0.05 Accuracy 99.28%\n",
      "Loss: 0.04 Accuracy 99.26%\n",
      "Loss: 0.03 Accuracy 99.29%\n",
      "Loss: 0.04 Accuracy 99.28%\n",
      "Loss: 0.05 Accuracy 99.24%\n",
      "Loss: 0.05 Accuracy 99.23%\n",
      "Loss: 0.06 Accuracy 99.15%\n",
      "Loss: 0.06 Accuracy 99.13%\n",
      "Loss: 0.07 Accuracy 99.08%\n",
      "Loss: 0.05 Accuracy 99.08%\n",
      "Loss: 0.05 Accuracy 99.08%\n",
      "Loss: 0.08 Accuracy 99.04%\n",
      "Loss: 0.07 Accuracy 98.99%\n",
      "Loss: 0.05 Accuracy 99.00%\n",
      "Loss: 0.07 Accuracy 98.95%\n",
      "Loss: 0.07 Accuracy 98.93%\n",
      "Loss: 0.09 Accuracy 98.85%\n",
      "Loss: 0.07 Accuracy 98.85%\n",
      "Loss: 0.07 Accuracy 98.82%\n",
      "Loss: 0.06 Accuracy 98.80%\n",
      "Loss: 0.05 Accuracy 98.81%\n",
      "Loss: 0.08 Accuracy 98.77%\n",
      "Loss: 0.08 Accuracy 98.71%\n",
      "Loss: 0.07 Accuracy 98.67%\n",
      "Test Accuracy: 54.72%\n"
     ]
    }
   ],
   "source": [
    "import torch.nn.functional as F\n",
    "\n",
    "model.train()\n",
    "model.cuda()\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    print(f\"Epoch {epoch}\")\n",
    "    loss_ema = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "\n",
    "    model.train()\n",
    "    for n, batch in enumerate(train_loader):\n",
    "        inputs, labels = batch\n",
    "\n",
    "        inputs = inputs.cuda()\n",
    "        labels = labels.cuda()\n",
    "        logits = model(inputs)\n",
    "        loss = F.cross_entropy(logits, labels)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        loss_ema = loss_ema * 0.9 + loss.data.cpu().numpy() * 0.1\n",
    "\n",
    "        pred = logits.max(dim=1).indices\n",
    "        correct += pred.eq(labels).sum().data.cpu().numpy()\n",
    "        total += pred.shape[0]\n",
    "\n",
    "        if n % 10 == 0:\n",
    "            print(f\"Loss: {loss_ema.item():.2f} Accuracy {correct/total:.2%}\")\n",
    "\n",
    "    with torch.no_grad():\n",
    "        model.eval()\n",
    "        correct = 0\n",
    "        total = 0\n",
    "        for n, batch in enumerate(test_loader):\n",
    "            inputs, labels = batch\n",
    "\n",
    "            inputs = inputs.cuda()\n",
    "            labels = labels.cuda()\n",
    "            logits = model(inputs)\n",
    "            pred = logits.max(dim=1).indices\n",
    "            correct += pred.eq(labels).sum().data.cpu().numpy()\n",
    "            total += pred.shape[0]\n",
    "\n",
    "        print(f\"Test Accuracy: {correct/total:.2%}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def test(model, dataset, dataset_name):\n",
    "    test_loader = DataLoader(dataset, batch_size=256, shuffle=True, collate_fn=collate_batch)\n",
    "    metrics = OODMetrics()\n",
    "    metrics_energy = OODMetrics()\n",
    "    softmax = MaxSoftmax(model)\n",
    "    energy = EnergyBased(model)\n",
    "    model.eval()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for n, batch in enumerate(test_loader):\n",
    "            inputs, labels = batch\n",
    "\n",
    "            inputs = inputs.cuda()\n",
    "            labels = labels.cuda()\n",
    "            metrics.update(softmax(inputs), labels)\n",
    "            metrics_energy.update(energy(inputs), labels)\n",
    "\n",
    "    d1 = metrics.compute()\n",
    "    d1.update({\"Method\": \"Softmax\", \"Dataset\": dataset_name})\n",
    "    \n",
    "    d2 = metrics_energy.compute()\n",
    "    d2.update({\"Method\": \"Energy\", \"Dataset\": dataset_name})\n",
    "    return [d1, d2]\n",
    "\n",
    "res = []\n",
    "\n",
    "ood_dataset = Reuters52(\"data/\", train=False, download=True, transform=prep, target_transform=ToUnknown())\n",
    "res+= test(model, test_dataset + ood_dataset, dataset_name=\"Reuters52\")\n",
    "\n",
    "ood_dataset = Multi30k(\"data/\", train=False, download=True, transform=prep, target_transform=ToUnknown())\n",
    "res+= test(model, test_dataset + ood_dataset, dataset_name=\"Multi30k\")\n",
    "\n",
    "ood_dataset = WMT16Sentences(\"data/\", download=True, transform=prep, target_transform=ToUnknown())\n",
    "res+= test(model, test_dataset + ood_dataset, dataset_name=\"WMT16Sentences\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "df = pd.DataFrame(res)\n",
    "print((df.groupby(\"Method\").mean() * 100).to_latex(float_format=\"%.2f\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrrrrr}\n",
      "\\toprule\n",
      "{} &  AUROC &  AUPR-IN &  AUPR-OUT &  ACC95TPR &  FPR95TPR \\\\\n",
      "Method  &        &          &           &           &           \\\\\n",
      "\\midrule\n",
      "Energy  &  89.35 &    69.95 &     94.56 &     75.59 &     31.93 \\\\\n",
      "Softmax &  82.06 &    62.08 &     88.74 &     63.77 &     52.02 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "myenv",
   "language": "python",
   "name": "myenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}