VQA-DataLoader-Test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "from collections import defaultdict\n",
    "from nltk.tokenize import word_tokenize\n",
    "import pickle\n",
    "import h5py\n",
    "import re\n",
    "from utils import img_data_2_mini_batch, imgs2batch\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch.utils.data as Data\n",
    "from torchvision import transforms\n",
    "from matplotlib.pyplot import imshow\n",
    "import numpy as np\n",
    "from PIL import Image\n",
    "import functools as ft\n",
    "\n",
    "%matplotlib inline\n",
    "import torch.nn.functional as F\n",
    "import sys, os\n",
    "from IPython.display import display, HTML\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from data_loader import VQADataSet\n",
    "pp = lambda parsed: print(json.dumps(parsed, indent=4, sort_keys=True))\n",
    "N = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./data/data_1.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00,  6.61it/s]\n",
      "100%|██████████| 1/1 [00:00<00:00, 3554.49it/s]\n",
      "100%|██████████| 1/1 [00:00<00:00, 1266.78it/s]\n",
      "100%|██████████| 1/1 [00:00<00:00, 2037.06it/s]\n",
      "100%|██████████| 1/1 [00:00<00:00, 3253.92it/s]\n",
      "100%|██████████| 1/1 [00:00<00:00, 121.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VQADataSet init time: 10.823058843612671\n",
      "writing to ./data/data_1.pkl\n"
     ]
    }
   ],
   "source": [
    "dataset_filename = \"./data/data_{}.pkl\".format(N)\n",
    "dataset = None\n",
    "print(dataset_filename)\n",
    "if (os.path.exists(dataset_filename)):\n",
    "    with open(dataset_filename, 'rb') as handle:\n",
    "        print(\"reading from \" + dataset_filename)\n",
    "        dataset = pickle.load(handle)\n",
    "else:\n",
    "    dataset = VQADataSet(Q=N)\n",
    "    with open(dataset_filename, 'wb') as handle:\n",
    "        print(\"writing to \" + dataset_filename)\n",
    "        pickle.dump(dataset, handle)\n",
    "\n",
    "assert(dataset is not None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "# print(len(dataset.anns))\n",
    "N_sample = 4\n",
    "q_keys = list(dataset.question_maps.keys())\n",
    "# for q_key in q_keys[:N_sample]:\n",
    "#     pp(dataset.question_maps[q_key])\n",
    "# pp(dataset.answer_maps[:N_sample])\n",
    "print(len(dataset.splits['test']))\n",
    "print(len(dataset.splits['train']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train: 1 test:0 percent: 1.0 total: 1\n",
      "1\n",
      "0\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAEphJREFUeJzt3X+QXXddxvH309QUkJYfZqG0SZoyBIeASHWpOqjFoQxtlUSdqi0WqVYjQmUcEI3Tip3gL8owDA4RrNiBVqGUKhBrsEgtAzgEGhSqKVMJbSFLKg2lFGiBUvn4xz3B22WTPXdzd+/my/s1s5Pz43vPeXY3efbke+89m6pCktSWoyYdQJI0fpa7JDXIcpekBlnuktQgy12SGmS5S1KDLHcte0nemOQPF/jYxyb5QJKvJHnNuLNJy9XRkw6gtiW5Hfj1qnrfQo9RVS88jAibgS8Ax5Vv6tB3Ea/cNVFJFvsC4yTg5oMV+xKcX5oIy12LJsmVwFrgH5N8NcnvJVmXpJJckOSzwL92Y9+R5H+S3NNNozx56DhvTvLH3fIzk8wkeVmSO5PckeRXD3L+NwMvAH6vO//pSS5Jck2Sv03yZeD8JEcl2ZLk00nuSnJ1kkcPHef5ST7T7bsoye1JTp+dbTjf0PoJSf4+yf4ktyV5ydC+S7pzXdFNG+1OMj20f02Sf+gee1eS1yc5JskXk/zA0LjHJPlakqkFf7PUHMtdi6aqng98FnhuVT28qi4d2n0a8CTgOd36e4D1wGOAfwf+7hCHPh54BHAicAGwLcmj5jj/+d1xLu3Of2BqaBNwDfDIbv9LgJ/tMp0A3A1sA0iyAXgD8Pxu3/cBq/t8/kmOAv4R+ESX9VnA7yR5ztCwjcBVXZbtwOu7x64ArgU+A6zrHn9VVX2jG3/e0DHOBd5XVfv75NJ3B8tdk3JJVd1bVV8DqKrLq+orXXldAvxgkkcc5LHfBLZW1TeragfwVeD7Rzj3h6vqXVX1re78vwlcVFUzQ+c/u5uyORu4tqo+0O37Q+BbPc/zdGCqqrZW1f1VdSvw18A5Q2M+VFU7qup/gSuBH+y2n8rgh8nLu6/T16vqQ92+twDP6354wOAHz5UjfP76LuB8oyZl74GF7ir1T4BfAKb4//JcBdwzx2PvqqoHhtbvAx6+kHN3TgLemWS4tP8XeCyDgv32+Kq6N8ldPc9zEnBCki8NbVsBfHBo/X+Glu8DHtL9UFkDfGbW53kgw0eS3AucluQO4AkMrvqlb7PctdgO9gqV4e3PYzBVcjpwO4Mpl7uBLFGmvcCvVdW/zR7YleeThtYfxmBq5oB7gYcNrR8/67i3VdX6BWTcC6xNcvRcBc/g6v08Bj8crqmqry/gHGqY0zJabJ8HHj/PmGOBbwB3MSjKP13sULO8EfiTJCcBJJlKsqnbdw3wM0l+PMlKYCsP/nfzceCsJI9OcjzwO0P7Pgp8OcnvJ3lokhVJnpLk6T0yfRS4A/jzJN+b5CFJnjG0/0rg5xgU/BUL+JzVOMtdi+3PgIuTfCnJ7x5kzBUMnjj8HHAzsHOpwnVex2Ba471JvtKd/0cAqmo38GLgrQzK9m5gZuixVzJ4wvR24L3A2w/s6ObRnws8DbiNwevt38TgfyaHNPTYJzB4UnoG+KWh/TMMnnguHjzNIwEQ39chjWYcb8waU47LgX1VdfEkc2h5cs5dOgIlWQf8PHDKZJNouXJaRjrCJHkl8F/Aq6vqtknn0fLktIwkNcgrd0lq0MTm3FetWlXr1q2b1Okl6Yj0sY997AtVNe99hCZW7uvWrWPXrl2TOr0kHZGSfKbPOKdlJKlBlrskNchyl6QGWe6S1CDLXZIaZLlLUoPmLfckl3e/q/K/DrI/Sf4iyZ4kNyX5ofHHlCSNos+V+5uBMw6x/0wGv/tyPbCZwe+blCRN0LzlXlUfAL54iCGbgCtqYCfwyCSPG1dASdLoxvEO1RN58O+knOm23TF7YJLNDK7uWbt27YJPuG7LPy34sZI0abf/+U8v+jnG8YTqXL/ncs5bTVbVZVU1XVXTU1Pz3hpBkrRA4yj3GQa/qf2A1cC+MRxXkrRA4yj37cCvdK+a+VHgnqr6jikZSdLSmXfOPcnbgGcCq5LMAH8EfA9AVb0R2AGcBewB7gN+dbHCSpL6mbfcq+rcefYXg98OL0laJnyHqiQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBvco9yRlJbkmyJ8mWOfavTXJDkv9IclOSs8YfVZLU17zlnmQFsA04E9gAnJtkw6xhFwNXV9UpwDnAX447qCSpvz5X7qcCe6rq1qq6H7gK2DRrTAHHdcuPAPaNL6IkaVR9yv1EYO/Q+ky3bdglwHlJZoAdwG/PdaAkm5PsSrJr//79C4grSeqjT7lnjm01a/1c4M1VtRo4C7gyyXccu6ouq6rpqpqempoaPa0kqZc+5T4DrBlaX813TrtcAFwNUFUfBh4CrBpHQEnS6PqU+43A+iQnJ1nJ4AnT7bPGfBZ4FkCSJzEod+ddJGlC5i33qnoAuBC4Dvgkg1fF7E6yNcnGbtjLgN9I8gngbcD5VTV76kaStESO7jOoqnYweKJ0eNsrhpZvBp4x3miSpIXyHaqS1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBvUq9yRnJLklyZ4kWw4y5heT3Jxkd5K3jjemJGkUR883IMkKYBvwbGAGuDHJ9qq6eWjMeuAPgGdU1d1JHrNYgSVJ8+tz5X4qsKeqbq2q+4GrgE2zxvwGsK2q7gaoqjvHG1OSNIo+5X4isHdofabbNuyJwBOT/FuSnUnOGFdASdLo5p2WATLHtprjOOuBZwKrgQ8meUpVfelBB0o2A5sB1q5dO3JYSVI/fa7cZ4A1Q+urgX1zjHl3VX2zqm4DbmFQ9g9SVZdV1XRVTU9NTS00syRpHn3K/UZgfZKTk6wEzgG2zxrzLuCnAJKsYjBNc+s4g0qS+pu33KvqAeBC4Drgk8DVVbU7ydYkG7th1wF3JbkZuAF4eVXdtVihJUmH1mfOnaraAeyYte0VQ8sFvLT7kCRNmO9QlaQGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBvUq9yRnJLklyZ4kWw4x7uwklWR6fBElSaOat9yTrAC2AWcCG4Bzk2yYY9yxwEuAj4w7pCRpNH2u3E8F9lTVrVV1P3AVsGmOca8ELgW+PsZ8kqQF6FPuJwJ7h9Znum3fluQUYE1VXXuoAyXZnGRXkl379+8fOawkqZ8+5Z45ttW3dyZHAa8FXjbfgarqsqqarqrpqamp/iklSSPpU+4zwJqh9dXAvqH1Y4GnAO9Pcjvwo8B2n1SVpMnpU+43AuuTnJxkJXAOsP3Azqq6p6pWVdW6qloH7AQ2VtWuRUksSZrXvOVeVQ8AFwLXAZ8Erq6q3Um2Jtm42AElSaM7us+gqtoB7Ji17RUHGfvMw48lSTocvkNVkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1KBe5Z7kjCS3JNmTZMsc+1+a5OYkNyW5PslJ448qSepr3nJPsgLYBpwJbADOTbJh1rD/AKar6qnANcCl4w4qSeqvz5X7qcCeqrq1qu4HrgI2DQ+oqhuq6r5udSewerwxJUmj6FPuJwJ7h9Znum0HcwHwnrl2JNmcZFeSXfv37++fUpI0kj7lnjm21ZwDk/OAaeDVc+2vqsuqarqqpqempvqnlCSN5OgeY2aANUPrq4F9swclOR24CDitqr4xnniSpIXoc+V+I7A+yclJVgLnANuHByQ5BfgrYGNV3Tn+mJKkUcxb7lX1AHAhcB3wSeDqqtqdZGuSjd2wVwMPB96R5ONJth/kcJKkJdBnWoaq2gHsmLXtFUPLp485lyTpMPgOVUlqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJapDlLkkNstwlqUGWuyQ1yHKXpAZZ7pLUIMtdkhpkuUtSgyx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1CDLXZIaZLlLUoMsd0lqkOUuSQ2y3CWpQZa7JDXIcpekBlnuktQgy12SGmS5S1KDLHdJalCvck9yRpJbkuxJsmWO/cckeXu3/yNJ1o07qCSpv3nLPckKYBtwJrABODfJhlnDLgDurqonAK8FXjXuoJKk/vpcuZ8K7KmqW6vqfuAqYNOsMZuAt3TL1wDPSpLxxZQkjeLoHmNOBPYOrc8AP3KwMVX1QJJ7gO8DvjA8KMlmYHO3+tUktywkNLBq9rGXCXONxlyjW67ZzDWCvOqwcp3UZ1Cfcp/rCrwWMIaqugy4rMc5Dx0o2VVV04d7nHEz12jMNbrlms1co1mKXH2mZWaANUPrq4F9BxuT5GjgEcAXxxFQkjS6PuV+I7A+yclJVgLnANtnjdkOvKBbPhv416r6jit3SdLSmHdapptDvxC4DlgBXF5Vu5NsBXZV1Xbgb4Ark+xhcMV+zmKGZgxTO4vEXKMx1+iWazZzjWbRc8ULbElqj+9QlaQGWe6S1KAjotyTPDrJvyT5VPfnow4x9rgkn0vy+uWQK8lJST6W5ONJdid54TLJ9bQkH+4y3ZTkl5ZDrm7cPyf5UpJrFznPsrytRo9cP5nk35M8kOTspcjUM9dLk9zc/X26Pkmv12MvQa4XJvnP7t/gh+Z4h/1Ecg2NOztJJRnvSyOratl/AJcCW7rlLcCrDjH2dcBbgdcvh1zASuCYbvnhwO3ACcsg1xOB9d3yCcAdwCMnnavb9yzgucC1i5hlBfBp4PHd9+gTwIZZY14EvLFbPgd4+xL8neqTax3wVOAK4OzFzjRCrp8CHtYt/9Yy+nodN7S8Efjn5ZCrG3cs8AFgJzA9zgxHxJU7D769wVuAn51rUJIfBh4LvHe55Kqq+6vqG93qMSzN/5b65PrvqvpUt7wPuBOYmnSuLs/1wFcWOctyva3GvLmq6vaqugn41iJnGTXXDVV1X7e6k8F7YpZDri8PrX4vc7zBchK5Oq9kcNHz9XEHOFLK/bFVdQdA9+djZg9IchTwGuDlyylXl21NkpsY3KLhVV2ZTjzXUL5TGVxdfHo55Vpkc91W48SDjamqB4ADt9WYdK5JGDXXBcB7FjXRQK9cSV6c5NMMivQlyyFXklOANVW1KNOPfW4/sCSSvA84fo5dF/U8xIuAHVW1d5wXV2PIRVXtBZ6a5ATgXUmuqarPTzpXd5zHAVcCL6iqw74SHFeuJTC222qM2STO2UfvXEnOA6aB0xY1UXe6ObbNdeuTbcC2JM8DLub/33Q5kVzdxehrgfMXK8CyKfeqOv1g+5J8PsnjquqOrozunGPYjwE/keRFDOa2Vyb5alUd9ImMJco1fKx9SXYDP8Hgv/kTzZXkOOCfgIuraufh5BlnriUyym01Zpbwthp9ck1Cr1xJTmfwg/y0oenIiecachXwhkVNNDBfrmOBpwDv7y5Gjwe2J9lYVbvGEeBImZYZvr3BC4B3zx5QVb9cVWurah3wu8AVh1vs48iVZHWSh3bLjwKeASz0bpjjzLUSeCeDr9M7FjlP71xLaLneVqNPrkmYN1c3zfBXwMaqWqof3H1yrR9a/WngU5POVVX3VNWqqlrXddZOBl+3sRT7gZMs+w8G85zXM/imXA88uts+DbxpjvHnszSvlpk3F/Bs4CYGz5bfBGxeJrnOA74JfHzo42mTztWtfxDYD3yNwRXQcxYpz1nAfzN4ruGibttWBv/IAB4CvAPYA3wUePxif+965np693W5F7gL2L1Mcr0P+PzQ36ftyyTX64DdXaYbgCcvh1yzxr6fMb9axtsPSFKDjpRpGUnSCCx3SWqQ5S5JDbLcJalBlrskNchyl6QGWe6S1KD/A+UxR0xUMvkrAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEICAYAAABbOlNNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAEu9JREFUeJzt3X2QXXddx/H3p4m0FLRtSvqYpinTzmgKI4zXVnwYq31mLKlQx4JgULA6wjgIVYLFoZQ6tlWmyIAyERhCfGixiEbBqWmhAyKWbqCMRKgJaWvWRkhJRdtKO4Wvf9wTvb+dG3az9+7ebPt+zdy555zf95zz/WXT/ew5526aqkKSpP0Om3QDkqRDi8EgSWoYDJKkhsEgSWoYDJKkhsEgSWoYDNKAJMcn+WSS/07y9kn3I02CwaBDXpL7kpw3huO8Msk/zFJ2BfAg8D1V9YZRzyktRQaD1DoV+Jc6wG9+Jlm+yP1Ii85g0CEtyWZgNfA3SR5O8pvd9h9K8o9J/jPJF5KcM7DPK5Ps6m4H3Zvk55J8H/Ae4AXdcf5zyLk+AKwHfrOrOS/J1UluSfInSf4LeGWSw5JsSPKVJF9P8qEkKwaO84ok93djVw1e8ST5QJJrB2rPSTI9sH5Skg8n2dv1/msDY1d35/pgN7ftSXoD46ck+ctu368neVeSw5PsS/LcgbrjkvxPkpUjfXH0pGUw6JBWVa8A/g24pKqeWVU3JDkZ+ChwLbACuBL4cJKVSZ4BvBO4uKq+G/hh4O6q+hLwK8BnuuMcPeRcrwT+FLihq7mtG1oH3AIc3Y3/GnAp8OPAScBDwLsBkqwF/gh4RTd2LLBqLnNNchjwN8AXgJOBc4HXJblwoOxFwE1dL1uAd3X7LgP+FrgfWNPtf1NVPdbVv3zgGC8FbquqvXPpS089BoOWopcDH6uqj1XVt6tqKzAFvLAb/zbwnCRPr6o9VbV9xPN9pqr+qjvX/wC/DFxVVdPdN96rgcu620yXAX9bVZ/sxn6762cufhBYWVXXVNXjVbUL+GPg8oGaf+jm/S1gM/D93faz6AfRb1TVI1X1zara/zxlE/CyLnigH1qb5/HnoKcI75dqKToV+Jkklwxs+y7gE1X1SJKfpX8V8b4knwbeUFVfHuF8u4ec/yNJBr/hfws4nv435/+r7/r5+hzPcypw0ozbXMuATw2s/8fA8qPAEV0gnQLcX1VPzDxoVd2Z5BHgx5PsAU6nf7UhDWUwaCmY+SB4N7C5qn5paHHVrcCtSZ5O/3bTHwM/NuQ4o5z/F6vq0zMLu2+83zewfiT920n7PQIcObB+wozj3ltVZ8yjx93A6iTLh4UD/auGl9MPlluq6pvzOIeeIryVpKXgq8CzB9b/BLgkyYVJliU5onuIu6r7PYQXdc8aHgMepv/T/P7jrErytBH7eQ/wO0lOBeiebazrxm4BfirJj3bnuYb2v7O7gRcmWZHkBOB1A2OfBf4ryRuTPL2b23OS/OAcevossAe4Lskzuj+THxkY3wz8NP1w+OA85qynEINBS8HvAm/uPoF0ZVXtpv9A+LeAvfR/Wv4N+n+fDwPeADwA7KP/gPhXu+N8HNgO/EeSB0fo5w/o34r5+yT/DfwTcDZA9zzjNcCf0f9G/RAwPbDvZvoPl+8D/h64ef9A99zgEuB5wL30f5/ivcBRszU0sO/p9B/WTwM/OzA+DXyO/tXPp4YdQ9ov/o96pIWV5D7g1QOfcppUH+8HHqiqN0+yDx36fMYgPQUkWQO8GHj+ZDvRUuCtJOlJLsnbgC8Cv1dV9066Hx36vJUkSWp4xSBJaizJZwzPetazas2aNZNuQ5KWlG3btj1YVbP+G1lLMhjWrFnD1NTUpNuQpCUlyf1zqfNWkiSpYTBIkhoGgySpYTBIkhoGgySpYTBIkhoGgySpYTBIkhoGgySpYTBIkhoGgySpYTBIkhoGgySpYTBIkhoGgySpYTBIkhoGgySpYTBIkhoGgySpYTBIkhoGgySpYTBIkhoGgySpYTBIkhoGgySpMZZgSHJRknuS7EyyYcj44Ulu7sbvTLJmxvjqJA8nuXIc/UiS5m/kYEiyDHg3cDGwFnhpkrUzyl4FPFRVpwM3AtfPGL8R+LtRe5EkjW4cVwxnATuraldVPQ7cBKybUbMO2NQt3wKcmyQASS4FdgHbx9CLJGlE4wiGk4HdA+vT3bahNVX1BPAN4NgkzwDeCLx1tpMkuSLJVJKpvXv3jqFtSdIw4wiGDNlWc6x5K3BjVT0820mqamNV9aqqt3Llynm0KUmai+VjOMY0cMrA+irggQPUTCdZDhwF7APOBi5LcgNwNPDtJN+sqneNoS9J0jyMIxjuAs5Ichrw78DlwMtm1GwB1gOfAS4DPl5VBfzY/oIkVwMPGwqSNFkjB0NVPZHktcCtwDLg/VW1Pck1wFRVbQHeB2xOspP+lcLlo55XkrQw0v/BfWnp9Xo1NTU16TYkaUlJsq2qerPV+ZvPkqSGwSBJahgMkqSGwSBJahgMkqSGwSBJahgMkqSGwSBJahgMkqSGwSBJahgMkqSGwSBJahgMkqSGwSBJahgMkqSGwSBJahgMkqSGwSBJahgMkqSGwSBJahgMkqSGwSBJahgMkqSGwSBJahgMkqSGwSBJahgMkqSGwSBJahgMkqSGwSBJaowlGJJclOSeJDuTbBgyfniSm7vxO5Os6bafn2Rbkn/u3n9yHP1IkuZv5GBIsgx4N3AxsBZ4aZK1M8peBTxUVacDNwLXd9sfBC6pqucC64HNo/YjSRrNOK4YzgJ2VtWuqnocuAlYN6NmHbCpW74FODdJqurzVfVAt307cESSw8fQkyRpnsYRDCcDuwfWp7ttQ2uq6gngG8CxM2peAny+qh4bQ0+SpHlaPoZjZMi2OpiaJGfSv710wQFPklwBXAGwevXqg+9SkjQn47himAZOGVhfBTxwoJoky4GjgH3d+irgI8DPV9VXDnSSqtpYVb2q6q1cuXIMbUuShhlHMNwFnJHktCRPAy4Htsyo2UL/4TLAZcDHq6qSHA18FHhTVX16DL1IkkY0cjB0zwxeC9wKfAn4UFVtT3JNkhd1Ze8Djk2yE3g9sP8jra8FTgd+O8nd3eu4UXuSJM1fqmY+Djj09Xq9mpqamnQbkrSkJNlWVb3Z6vzNZ0lSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSYyzBkOSiJPck2Zlkw5Dxw5Pc3I3fmWTNwNibuu33JLlwHP1IkuZv5GBIsgx4N3AxsBZ4aZK1M8peBTxUVacDNwLXd/uuBS4HzgQuAv6wO54kaULGccVwFrCzqnZV1ePATcC6GTXrgE3d8i3AuUnSbb+pqh6rqnuBnd3xJEkTMo5gOBnYPbA+3W0bWlNVTwDfAI6d474AJLkiyVSSqb17946hbUnSMOMIhgzZVnOsmcu+/Y1VG6uqV1W9lStXHmSLkqS5GkcwTAOnDKyvAh44UE2S5cBRwL457itJWkTjCIa7gDOSnJbkafQfJm+ZUbMFWN8tXwZ8vKqq235596ml04AzgM+OoSdJ0jwtH/UAVfVEktcCtwLLgPdX1fYk1wBTVbUFeB+wOclO+lcKl3f7bk/yIeBfgCeA11TVt0btSZI0f+n/4L609Hq9mpqamnQbkrSkJNlWVb3Z6vzNZ0lSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDUMBklSw2CQJDVGCoYkK5JsTbKjez/mAHXru5odSdZ3245M8tEkX06yPcl1o/QiSRqPUa8YNgC3V9UZwO3deiPJCuAtwNnAWcBbBgLk96vqe4HnAz+S5OIR+5EkjWjUYFgHbOqWNwGXDqm5ENhaVfuq6iFgK3BRVT1aVZ8AqKrHgc8Bq0bsR5I0olGD4fiq2gPQvR83pOZkYPfA+nS37f8kORq4hP5VhyRpgpbPVpDkNuCEIUNXzfEcGbKtBo6/HPhz4J1Vtes79HEFcAXA6tWr53hqSdLBmjUYquq8A40l+WqSE6tqT5ITga8NKZsGzhlYXwXcMbC+EdhRVe+YpY+NXS29Xq++U60kaf5GvZW0BVjfLa8H/npIza3ABUmO6R46X9BtI8m1wFHA60bsQ5I0JqMGw3XA+Ul2AOd36yTpJXkvQFXtA94G3NW9rqmqfUlW0b8dtRb4XJK7k7x6xH4kSSNK1dK7K9Pr9WpqamrSbUjSkpJkW1X1ZqvzN58lSQ2DQZLUMBgkSQ2DQZLUMBgkSQ2DQZLUMBgkSQ2DQZLUMBgkSQ2DQZLUMBgkSQ2DQZLUMBgkSQ2DQZLUMBgkSQ2DQZLUMBgkSQ2DQZLUMBgkSQ2DQZLUMBgkSQ2DQZLUMBgkSQ2DQZLUMBgkSQ2DQZLUMBgkSQ2DQZLUMBgkSQ2DQZLUMBgkSY2RgiHJiiRbk+zo3o85QN36rmZHkvVDxrck+eIovUiSxmPUK4YNwO1VdQZwe7feSLICeAtwNnAW8JbBAEnyYuDhEfuQJI3JqMGwDtjULW8CLh1ScyGwtar2VdVDwFbgIoAkzwReD1w7Yh+SpDEZNRiOr6o9AN37cUNqTgZ2D6xPd9sA3ga8HXh0thMluSLJVJKpvXv3jta1JOmAls9WkOQ24IQhQ1fN8RwZsq2SPA84vap+Pcma2Q5SVRuBjQC9Xq/meG5J0kGaNRiq6rwDjSX5apITq2pPkhOBrw0pmwbOGVhfBdwBvAD4gST3dX0cl+SOqjoHSdLEjHoraQuw/1NG64G/HlJzK3BBkmO6h84XALdW1R9V1UlVtQb4UeBfDQVJmrxRg+E64PwkO4Dzu3WS9JK8F6Cq9tF/lnBX97qm2yZJOgSlaundru/1ejU1NTXpNiRpSUmyrap6s9X5m8+SpIbBIElqGAySpIbBIElqGAySpIbBIElqGAySpIbBIElqGAySpIbBIElqGAySpIbBIElqGAySpIbBIElqGAySpIbBIElqGAySpIbBIElqGAySpIbBIElqGAySpIbBIElqGAySpIbBIElqpKom3cNBS7IXuH/SfRykZwEPTrqJReacnxqc89JxalWtnK1oSQbDUpRkqqp6k+5jMTnnpwbn/OTjrSRJUsNgkCQ1DIbFs3HSDUyAc35qcM5PMj5jkCQ1vGKQJDUMBklSw2AYoyQrkmxNsqN7P+YAdeu7mh1J1g8Z35Lkiwvf8ehGmXOSI5N8NMmXk2xPct3idn9wklyU5J4kO5NsGDJ+eJKbu/E7k6wZGHtTt/2eJBcuZt+jmO+ck5yfZFuSf+7ef3Kxe5+PUb7G3fjqJA8nuXKxel4QVeVrTC/gBmBDt7wBuH5IzQpgV/d+TLd8zMD4i4E/A7446fks9JyBI4Gf6GqeBnwKuHjSczrAPJcBXwGe3fX6BWDtjJpfBd7TLV8O3Nwtr+3qDwdO646zbNJzWuA5Px84qVt+DvDvk57PQs53YPzDwF8AV056PqO8vGIYr3XApm55E3DpkJoLga1Vta+qHgK2AhcBJHkm8Hrg2kXodVzmPeeqerSqPgFQVY8DnwNWLULP83EWsLOqdnW93kR/7oMG/yxuAc5Nkm77TVX1WFXdC+zsjneom/ecq+rzVfVAt307cESSwxel6/kb5WtMkkvp/9CzfZH6XTAGw3gdX1V7ALr344bUnAzsHlif7rYBvA14O/DoQjY5ZqPOGYAkRwOXALcvUJ+jmnUOgzVV9QTwDeDYOe57KBplzoNeAny+qh5boD7HZd7zTfIM4I3AWxehzwW3fNINLDVJbgNOGDJ01VwPMWRbJXkecHpV/frM+5aTtlBzHjj+cuDPgXdW1a6D73BRfMc5zFIzl30PRaPMuT+YnAlcD1wwxr4WyijzfStwY1U93F1ALGkGw0GqqvMONJbkq0lOrKo9SU4EvjakbBo4Z2B9FXAH8ALgB5LcR//rclySO6rqHCZsAee830ZgR1W9YwztLpRp4JSB9VXAAweome7C7ihg3xz3PRSNMmeSrAI+Avx8VX1l4dsd2SjzPRu4LMkNwNHAt5N8s6retfBtL4BJP+R4Mr2A36N9EHvDkJoVwL30H74e0y2vmFGzhqXz8HmkOdN/nvJh4LBJz2WWeS6nf//4NP7/weSZM2peQ/tg8kPd8pm0D593sTQePo8y56O7+pdMeh6LMd8ZNVezxB8+T7yBJ9OL/r3V24Ed3fv+b3494L0Ddb9I/wHkTuAXhhxnKQXDvOdM/yeyAr4E3N29Xj3pOX2Hub4Q+Ff6n1y5qtt2DfCibvkI+p9I2Ql8Fnj2wL5XdfvdwyH6yatxzhl4M/DIwNf1buC4Sc9nIb/GA8dY8sHgP4khSWr4qSRJUsNgkCQ1DAZJUsNgkCQ1DAZJUsNgkCQ1DAZJUuN/AVVypOQSGf2+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 1\n"
     ]
    }
   ],
   "source": [
    "def plot_distribution(dataset):\n",
    "    train = dataset.splits['train']\n",
    "    test = dataset.splits['test']\n",
    "    print('train: {} test:{} percent: {} total: {}'.format(\n",
    "        len(train),\n",
    "        len(test),\n",
    "        len(train)/len(train+test),\n",
    "        len(train) + len(test),\n",
    "    ))\n",
    "    def split_to_freq(split, T=None):\n",
    "        anns = [dataset.answer_maps[i] for i in split]\n",
    "        _cntr = defaultdict(int)\n",
    "        for ann in anns:\n",
    "            _cntr[ann['answer']]+=1\n",
    "        tmp = sorted(_cntr.items(), key=lambda kv: kv[1], reverse=True)[:T]\n",
    "        print(len(tmp))\n",
    "        return [x[1] for x in tmp]\n",
    "\n",
    "    def plot_freq(title, Y):\n",
    "        X = range(len(Y))\n",
    "        plt.bar(X, Y)\n",
    "        plt.title(title)\n",
    "        plt.show()\n",
    "\n",
    "    train_freq = split_to_freq(train, T=50)#len(train))\n",
    "    test_freq = split_to_freq(test, T=50)#(train))\n",
    "    plot_freq(\"train frequency\", train_freq)\n",
    "    plot_freq(\"test frequency\", test_freq)\n",
    "    print('1', len(train_freq))\n",
    "\n",
    "plot_distribution(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/1 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "batch_size: 5 shuffle: True\n",
      "\n",
      "minibatch:\n",
      "idx tensor([0])\n",
      "v:  torch.Size([1, 3, 224, 224])\n",
      "a:  tensor([0])\n",
      "q:  tensor([[1, 7, 8, 5, 4, 6, 2]])\n",
      "len:  tensor([7])\n",
      "{'image_id': 262148, 'question': 'Where is he looking?', 'question_id': 262148000, 'tokens': ['<start>', 'where', 'is', 'he', 'looking', '?', '<end>'], 'encoding': [1, 7, 8, 5, 4, 6, 2], 'image_path': 'data/val2014/COCO_val2014_000000262148.jpg'}\n",
      "0 tensor(7)\n",
      "\tcheck: <start> where is he looking ? <end> 7\n",
      "\tanswer: <start> where is he looking ? <end>\n",
      "\tcheck: down\n",
      "\tanswer: down\n",
      "<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=640x512 at 0x2AAB1682E8D0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import tqdm\n",
    "train_loader = dataset.build_data_loader(train=True, args={'batch_size': 5})\n",
    "# print(len(train_loader))\n",
    "for minibatch in tqdm.tqdm(train_loader):\n",
    "    idxs, v,q,a,q_len = minibatch\n",
    "    q = VQADataSet.batchify_questions(q)\n",
    "    batch_size = len(q_len)\n",
    "    print(\"\\nminibatch:\")\n",
    "    print('idx', idxs)\n",
    "    print('v: ', v.shape)\n",
    "    print('a: ', a)\n",
    "    print('q: ', q)\n",
    "    print('len: ', q_len)\n",
    "    for i in range(batch_size):\n",
    "        idx_i = idxs[i]\n",
    "        q_i = q[i]\n",
    "        a_i = a[i].item()\n",
    "        v_i = v[i].numpy()\n",
    "        q_len_i = q_len[i]\n",
    "        qj, aj = dataset.get(idx_i, 'train')\n",
    "        print(qj)\n",
    "        qj_check = dataset.decode_question(q_i.tolist())\n",
    "        qj_ans = \" \".join(qj['tokens'])\n",
    "        aj_check = str(dataset.decode_answer(a_i))\n",
    "        aj_ans = str(aj['answer'])\n",
    "        print(idx_i.item(), q_len_i)\n",
    "        print(\"\\tcheck: \" + qj_check + \" \" + str(len(qj['tokens'])))\n",
    "        print(\"\\tanswer: \" + qj_ans)\n",
    "        print(\"\\tcheck: \" + aj_check)\n",
    "        print(\"\\tanswer: \" + aj_ans)\n",
    "        assert(aj_ans == aj_check)\n",
    "        assert(qj_ans in qj_check)\n",
    "        img_v = v_i\n",
    "        img_v = Image.open(qj['image_path'])\n",
    "        print(img_v)\n",
    "#         img_v = img_v.reshape(224, 224, 3)\n",
    "#         imshow(img_v[0,:,:], interpolation='nearest')\n",
    "        break\n",
    "    break\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 1)\n"
     ]
    }
   ],
   "source": [
    "print(dataset.size())"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "x= 'data/val2014/COCO_val2014_000000262274.jpg'\n",
    "\n",
    "print(os.path.exists(x))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://arxiv.org/pdf/1803.07724.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # pp(dataset.answer_maps)\n",
    "# from random import shuffle\n",
    "# from collections import defaultdict\n",
    "# cntr = defaultdict(int)\n",
    "# dist = defaultdict(list)\n",
    "# for i, ann in enumerate(dataset.answer_maps):\n",
    "#     ans = ann['answer']\n",
    "#     cntr[ans]+=1\n",
    "#     dist[ans].append(i)\n",
    "#     split = {'train': [], 'test': []}\n",
    "#     z_cnt = 0\n",
    "#     for ans, idxes in dist.items():\n",
    "#         shuffle(idxes)\n",
    "#         c = int(len(idxes)*0.20)\n",
    "#         split['train'] += idxes[c:]\n",
    "#         split['test'] += idxes[:c]\n",
    "#     split['test'].sort()\n",
    "#     print(split['test'][:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad>\n",
      "<start>\n",
      "<end>\n",
      "<unk>\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n"
     ]
    }
   ],
   "source": [
    "print(dataset.itoq[0])\n",
    "print(dataset.itoq[1])\n",
    "print(dataset.itoq[2])\n",
    "print(dataset.itoq[3])\n",
    "\n",
    "print(dataset.qtoi['<pad>'])\n",
    "print(dataset.qtoi['<start>'])\n",
    "print(dataset.qtoi['<end>'])\n",
    "print(dataset.qtoi['<unk>'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}