Single Observation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gridworld import gameEnv\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import deque\n",
    "import pickle\n",
    "from skimage.color import rgb2gray\n",
    "import random\n",
    "import torch\n",
    "import torch.nn as nn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Define Environment object </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADKdJREFUeJzt3V+sHPV5xvHvUxtCQtqAgVouhh5XQSBUCUOtFERUtYBbQiLoRYRAURVVSNykLTSREmivIvUikaokXFSRECRFFeVPCDTIikipQ1RVqhyOgSZgQ2wIBFuA3RRKSqW2Tt5ezDg9ce2cOT67e87w+36k1e7M7Gp+o/GzMzue876pKiS15RdWegCSZs/gSw0y+FKDDL7UIIMvNcjgSw0y+FKDlhX8JFcmeS7J3iS3TGpQkqYrx3sDT5I1wPeArcA+4HHg+qraNbnhSZqGtcv47PuAvVX1AkCSe4FrgGMG//TTT6+5ubllrPLtb+fOnSs9BI1cVWWx9ywn+GcCLy+Y3gf85s/7wNzcHPPz88tY5dtfsug+k5Zt6hf3ktyYZD7J/MGDB6e9OkkDLCf4+4GzFkxv7Of9jKq6vaq2VNWWM844YxmrkzQpywn+48A5STYlORG4Dnh4MsOSNE3H/Ru/qg4l+SPgG8Aa4EtV9czERiZpapZzcY+q+jrw9QmNRdKMeOee1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1KBFg5/kS0kOJHl6wbx1SR5Nsqd/PnW6w5Q0SUOO+H8NXHnEvFuA7VV1DrC9n5Y0EosGv6r+Efi3I2ZfA9zVv74L+P0Jj0vSFB3vb/z1VfVK//pVYP2ExiNpBpZ9ca+6rpvH7LxpJx1p9Tne4L+WZANA/3zgWG+0k460+hxv8B8GPtq//ijwtckMR9IsDPnvvHuAfwbOTbIvyQ3AZ4CtSfYAV/TTkkZi0U46VXX9MRZdPuGxSJoR79yTGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGjSk9NZZSR5LsivJM0lu6ufbTUcaqSFH/EPAJ6rqfOBi4GNJzsduOtJoDemk80pVPdG//hGwGzgTu+lIo7Wk3/hJ5oALgR0M7KZjQw1p9Rkc/CTvBr4K3FxVby5c9vO66dhQQ1p9BgU/yQl0ob+7qh7sZw/upiNpdRlyVT/AncDuqvrcgkV205FGatGGGsClwB8A303yVD/vz+i659zfd9Z5Cbh2OkOUNGlDOun8E5BjLLabjjRC3rknNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0a8vf4mqmjVjDTTx3rL8S1FB7xpQYZfKlBQ2runZTk20n+pe+k8+l+/qYkO5LsTXJfkhOnP1xJkzDkiP9fwGVVdQGwGbgyycXAZ4HPV9V7gdeBG6Y3TEmTNKSTTlXVf/STJ/SPAi4DHujn20lHGpGhdfXX9BV2DwCPAs8Db1TVof4t++jaah3ts3bSkVaZQcGvqh9X1WZgI/A+4LyhK7CTjrT6LOmqflW9ATwGXAKckuTwfQAbgf0THpukKRlyVf+MJKf0r98JbKXrmPsY8OH+bXbSkUZkyJ17G4C7kqyh+6K4v6q2JdkF3JvkL4An6dpsSRqBIZ10vkPXGvvI+S/Q/d6XNDLeuSc1yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81aHDw+xLbTybZ1k/bSUcaqaUc8W+iK7J5mJ10pJEa2lBjI/BB4I5+OthJRxqtoUf8LwCfBH7ST5+GnXSk0RpSV/9DwIGq2nk8K7CTjrT6DKmrfylwdZKrgJOAXwJuo++k0x/17aQjjciQbrm3VtXGqpoDrgO+WVUfwU460mgt5//xPwV8PMleut/8dtKRRmLIqf5PVdW3gG/1r+2kI42Ud+5JDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81aFAhjiQvAj8CfgwcqqotSdYB9wFzwIvAtVX1+nSGKWmSlnLE/52q2lxVW/rpW4DtVXUOsL2fljQCyznVv4aukQbYUEMalaHBL+Dvk+xMcmM/b31VvdK/fhVYP/HRSZqKocU2319V+5P8MvBokmcXLqyqSlJH+2D/RXEjwNlnn72swUqajEFH/Kra3z8fAB6iq677WpINAP3zgWN81k460iozpIXWyUl+8fBr4HeBp4GH6RppgA01pFEZcqq/Hnioa5DLWuBvq+qRJI8D9ye5AXgJuHZ6w5Q0SYsGv2+cccFR5v8QuHwag5I0Xd65JzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzVo6F/naWay0gNQAzziSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UoEHBT3JKkgeSPJtkd5JLkqxL8miSPf3zqdMerKTJGHrEvw14pKrOoyvDtRs76UijNaTK7nuA3wLuBKiq/66qN7CTjjRaQ474m4CDwJeTPJnkjr7Mtp10pJEaEvy1wEXAF6vqQuAtjjitr6qia7P1/yS5Mcl8kvmDBw8ud7ySJmBI8PcB+6pqRz/9AN0XgZ10pJFaNPhV9SrwcpJz+1mXA7uwk440WkP/LPePgbuTnAi8APwh3ZeGnXSkERoU/Kp6CthylEV20pFGyDv3pAYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYNqat/bpKnFjzeTHKznXSk8RpSbPO5qtpcVZuB3wD+E3gIO+lIo7XUU/3Lgeer6iXspCON1lKDfx1wT//aTjrSSA0Ofl9a+2rgK0cus5OONC5LOeJ/AHiiql7rp+2kI43UUoJ/Pf93mg920pFGa1Dw++64W4EHF8z+DLA1yR7gin5a0ggM7aTzFnDaEfN+iJ10pFHyzj2pQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQUNLb/1pkmeSPJ3kniQnJdmUZEeSvUnu66vwShqBIS20zgT+BNhSVb8OrKGrr/9Z4PNV9V7gdeCGaQ5U0uQMPdVfC7wzyVrgXcArwGXAA/1yO+lIIzKkd95+4C+BH9AF/t+BncAbVXWof9s+4MxpDVLSZA051T+Vrk/eJuBXgJOBK4euwE460uoz5FT/CuD7VXWwqv6Hrrb+pcAp/ak/wEZg/9E+bCcdafUZEvwfABcneVeS0NXS3wU8Bny4f4+ddKQRGfIbfwfdRbwngO/2n7kd+BTw8SR76Zpt3DnFcUqaoHSNbmdjy5YtNT8/P7P1jVF3UiUdv6pa9B+Rd+5JDTL4UoMMvtQggy81aKYX95IcBN4C/nVmK52+03F7Vqu307bAsO351apa9IaZmQYfIMl8VW2Z6UqnyO1Zvd5O2wKT3R5P9aUGGXypQSsR/NtXYJ3T5PasXm+nbYEJbs/Mf+NLWnme6ksNmmnwk1yZ5Lm+Tt8ts1z3ciU5K8ljSXb19Qdv6uevS/Jokj3986krPdalSLImyZNJtvXTo62lmOSUJA8keTbJ7iSXjHn/TLPW5cyCn2QN8FfAB4DzgeuTnD+r9U/AIeATVXU+cDHwsX78twDbq+ocYHs/PSY3AbsXTI+5luJtwCNVdR5wAd12jXL/TL3WZVXN5AFcAnxjwfStwK2zWv8UtudrwFbgOWBDP28D8NxKj20J27CRLgyXAduA0N0gsvZo+2w1P4D3AN+nv261YP4o9w9dKbuXgXV0NS+3Ab83qf0zy1P9wxty2Gjr9CWZAy4EdgDrq+qVftGrwPoVGtbx+ALwSeAn/fRpjLeW4ibgIPDl/qfLHUlOZqT7p6Zc69KLe0uU5N3AV4Gbq+rNhcuq+xoexX+TJPkQcKCqdq70WCZkLXAR8MWqupDu1vCfOa0f2f5ZVq3Lxcwy+PuBsxZMH7NO32qV5AS60N9dVQ/2s19LsqFfvgE4sFLjW6JLgauTvAjcS3e6fxsDaymuQvuAfdVVjIKuatRFjHf/LKvW5WJmGfzHgXP6q5In0l2oeHiG61+Wvt7gncDuqvrcgkUP09UchBHVHqyqW6tqY1XN0e2Lb1bVRxhpLcWqehV4Ocm5/azDtSFHuX+Ydq3LGV+wuAr4HvA88OcrfQFliWN/P91p4neAp/rHVXS/i7cDe4B/ANat9FiPY9t+G9jWv/414NvAXuArwDtWenxL2I7NwHy/j/4OOHXM+wf4NPAs8DTwN8A7JrV/vHNPapAX96QGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxr0v95G4v3/3GYuAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "env = gameEnv(partial=True,size=9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7efdaecc4898>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADMZJREFUeJzt3V+MHfV5xvHvUxtCQtqAgVouRrWrIBCqhKFWCiKqWsAtIRH0IkKgqIoqJG7SFppICbRXkXqRSFUSLqpICJKiivInBBpkRaTUIaoqVQ5roAnYEBtigi3AJoWSUqmtk7cXM6aLZePZ3XPO7vD7fqTVOTNzDvMbxs+ZObOz75uqQlJbfmm5ByBp9gy+1CCDLzXI4EsNMvhSgwy+1CCDLzVoScFPckWSZ5PsSXLzpAYlabqy2Bt4kqwCfgRsAfYBjwHXVdXOyQ1P0jSsXsJ7PwTsqarnAZLcA1wNHDP4p59+em3YsGEJq5T0Tvbu3curr76a471uKcE/E3hx3vQ+4Lff6Q0bNmxgbm5uCauU9E42b9486HVTv7iX5IYkc0nmDh48OO3VSRpgKcHfD5w1b3p9P+9tquq2qtpcVZvPOOOMJaxO0qQsJfiPAWcn2ZjkROBa4KHJDEvSNC36O35VHUryJ8B3gFXA16rq6YmNTNLULOXiHlX1beDbExqLpBnxzj2pQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQccNfpKvJTmQ5Kl589YkeSTJ7v7x1OkOU9IkDTni/y1wxRHzbga2VdXZwLZ+WtJIHDf4VfXPwL8fMftq4M7++Z3AH054XJKmaLHf8ddW1Uv985eBtRMaj6QZWPLFveq6bh6z86addKSVZ7HBfyXJOoD+8cCxXmgnHWnlWWzwHwI+2T//JPCtyQxH0iwM+XXe3cC/Auck2ZfkeuALwJYku4HL+2lJI3HcTjpVdd0xFl024bFImhHv3JMaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaNKT01llJHk2yM8nTSW7s59tNRxqpIUf8Q8Bnquo84CLgU0nOw2460mgN6aTzUlU93j//GbALOBO76UijtaDv+Ek2ABcA2xnYTceGGtLKMzj4Sd4PfBO4qaremL/snbrp2FBDWnkGBT/JCXShv6uqHuhnD+6mI2llGXJVP8AdwK6q+tK8RXbTkUbquA01gEuAPwJ+mOTJft5f0HXPua/vrPMCcM10hihp0oZ00vkXIMdYbDcdaYS8c09qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGjTk7/EnZseOHXR1PXRsR61gpl75v2ciPOJLDTL4UoOG1Nw7Kcn3k/xb30nn8/38jUm2J9mT5N4kJ05/uJImYcgR/7+BS6vqfGATcEWSi4AvAl+uqg8CrwHXT2+YkiZpSCedqqr/7CdP6H8KuBS4v59vJx1pRIbW1V/VV9g9ADwCPAe8XlWH+pfso2urdbT3vtVJZxIDlrR0g4JfVT+vqk3AeuBDwLlDVzC/k84ixyhpwhZ0Vb+qXgceBS4GTkly+D6A9cD+CY9N0pQMuap/RpJT+ufvBbbQdcx9FPh4/zI76UgjMuTOvXXAnUlW0X1Q3FdVW5PsBO5J8lfAE3RttiSNwJBOOj+ga4195Pzn6b7vSxoZ79yTGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGjQ4+H2J7SeSbO2n7aQjjdRCjvg30hXZPMxOOtJIDW2osR74KHB7Px3spCON1tAj/leAzwK/6KdPw0460mgNqav/MeBAVe1YzArspCOtPEPq6l8CXJXkSuAk4FeAW+k76fRHfTvpSCMypFvuLVW1vqo2ANcC362qT2AnHWm0lvJ7/M8Bn06yh+47v510pJEYcqr/lqr6HvC9/rmddKSR8s49qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBg0qxJFkL/Az4OfAoaranGQNcC+wAdgLXFNVr01nmJImaSFH/N+rqk3zquXeDGyrqrOBbf20pBFYyqn+1XSNNMCGGtKoDA1+Af+YZEeSG/p5a6vqpf75y8DaiY9O0lQMLbb54aran+RXgUeSPDN/YVVVkjraG/sPihuOtkzS8hh0xK+q/f3jAeBBuuq6ryRZB9A/HjjGe+2kI60wQ1ponZzklw8/B34feAp4iK6RBthQQxqVIaf6a4EHuwa5rAb+vqoeTvIYcF+S64EXgGumN0xJk3Tc4PeNM84/yvyfApdNY1CSpss796QGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGDf3rPM1MlnsAK9xR/whUC+QRX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBg0KfpJTktyf5Jkku5JcnGRNkkeS7O4fT532YCVNxtAj/q3Aw1V1Ll0Zrl3YSUcarSFVdj8A/A5wB0BV/U9VvY6ddKTRGnLE3wgcBL6e5Ikkt/dltu2kI43UkOCvBi4EvlpVFwBvcsRpfVUVx7iJOskNSeaSzC11sJImY0jw9wH7qmp7P30/3QeBnXSkkTpu8KvqZeDFJOf0sy4DdmInHWm0hv5Z7p8CdyU5EXge+GO6Dw076UgjNCj4VfUkcLRTdTvpSCPknXtSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSg4bU1T8nyZPzft5IcpOddKTxGlJs89mq2lRVm4DfAv4LeBA76UijtdBT/cuA56rqBeykI43WQoN/LXB3/9xOOtJIDQ5+X1r7KuAbRy6zk440Lgs54n8EeLyqXumn7aQjjdRCgn8d/3+aD3bSkUZrUPD77rhbgAfmzf4CsCXJbuDyflrSCAztpPMmcNoR836KnXSkUfLOPalBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBQ0tv/XmSp5M8leTuJCcl2Zhke5I9Se7tq/BKGoEhLbTOBP4M2FxVvwmsoquv/0Xgy1X1QeA14PppDlTS5Aw91V8NvDfJauB9wEvApcD9/XI76UgjMqR33n7gr4Gf0AX+P4AdwOtVdah/2T7gzGkNUtJkDTnVP5WuT95G4NeAk4Erhq7ATjrSyjOkvPblwI+r6iBAkgeAS4BTkqzuj/rrgf1He3NV3Qbc1r/3qG22JM3WkO/4PwEuSvK+JKGrpb8TeBT4eP8aO+lIIzLkO/52uot4jwM/7N9zG/A54NNJ9tA127hjiuOUNEHpGt3OaGWe6muJZvnvdYw2b97M3Nxcjvc679yTGmTwpQYZfKlBBl9q0KA22RP0KvBm//hucTpuz8x0v1EebEVvyyIM2Z5fH/IfmulVfYAkc1W1eaYrnSK3Z+V6N20LTHZ7PNWXGmTwpQYtR/BvW4Z1TpPbs3K9m7YFJrg9M/+OL2n5eaovNWimwU9yRZJn+zp9N89y3UuV5KwkjybZ2dcfvLGfvybJI0l294+nLvdYFyLJqiRPJNnaT4+2lmKSU5Lcn+SZJLuSXDzm/TPNWpczC36SVcDfAB8BzgOuS3LerNY/AYeAz1TVecBFwKf68d8MbKuqs4Ft/fSY3Ajsmjc95lqKtwIPV9W5wPl02zXK/TP1WpdVNZMf4GLgO/OmbwFumdX6p7A93wK2AM8C6/p564Bnl3tsC9iG9XRhuBTYCoTuBpHVR9tnK/kH+ADwY/rrVvPmj3L/0JWyexFYQ3ej3VbgDya1f2Z5qn94Qw4bbZ2+JBuAC4DtwNqqeqlf9DKwdpmGtRhfAT4L/KKfPo3x1lLcCBwEvt5/dbk9ycmMdP/UlGtdenFvgZK8H/gmcFNVvTF/WXUfw6P4NUmSjwEHqmrHco9lQlYDFwJfraoL6G4Nf9tp/cj2z5JqXR7PLIO/Hzhr3vQx6/StVElOoAv9XVX1QD/7lSTr+uXrgAPLNb4FugS4Ksle4B660/1b6Wsp9q8Z0z7aB+yrrmIUdFWjLmS8++etWpdV9b/A22pd9q9Z9P6ZZfAfA87ur0qeSHeh4qEZrn9J+nqDdwC7qupL8xY9RFdzEEZUe7Cqbqmq9VW1gW5ffLeqPsFIaylW1cvAi0nO6Wcdrg05yv3DtGtdzviCxZXAj4DngL9c7gsoCxz7h+lOE38APNn/XEn3vXgbsBv4J2DNco91Edv2u8DW/vlvAN8H9gDfAN6z3ONbwHZsAub6ffQPwKlj3j/A54FngKeAvwPeM6n94517UoO8uCc1yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtSg/wPLWM6YXGWLRAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "prev_state = env.reset()\n",
    "plt.imshow(prev_state)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 align='center'> Training Q Network </h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Hyper-parameters </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "FREEZE_INTERVAL = 20000 # steps\n",
    "MEMORY_SIZE = 60000 \n",
    "OUTPUT_SIZE = 4\n",
    "TOTAL_EPISODES = 10000\n",
    "MAX_STEPS = 50\n",
    "INITIAL_EPSILON = 1.0\n",
    "FINAL_EPSILON = 0.1\n",
    "GAMMA = 0.99\n",
    "INPUT_IMAGE_DIM = 84\n",
    "PERFORMANCE_SAVE_INTERVAL = 500 # episodes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Save Dictionay Function </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def save_obj(obj, name ):\n",
    "    with open('data/'+ name + '.pkl', 'wb') as f:\n",
    "        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Experience Replay</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Memory():\n",
    "    \n",
    "    def __init__(self,memsize):\n",
    "        self.memsize = memsize\n",
    "        self.memory = deque(maxlen=self.memsize)\n",
    "    \n",
    "    def add_sample(self,sample):\n",
    "        self.memory.append(sample)\n",
    "    \n",
    "    def get_batch(self,size):\n",
    "        return random.sample(self.memory,k=size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Preprocess Images </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_image(image):\n",
    "    image = rgb2gray(image) # this automatically scales the color for block between 0 - 1\n",
    "    return np.copy(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fbfa7338048>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADLVJREFUeJzt3X+oX/V9x/Hna4nW1m7VuC1kRpaMiiIDo4ZOsYxNzZbaovujiFJGGUL+6TZdC61ufxX2RwujrX+MQtB2Mpw/anWVUOyyNGUMRur1x1pNtIk21gQ1WnV2Dralfe+Pc9Jds2Q5N/f7/d57/DwfcPl+zznfb87ncHzd8+Me3+9UFZLa8gtLPQBJs2fwpQYZfKlBBl9qkMGXGmTwpQYZfKlBiwp+ks1JnkmyL8ktkxqUpOnKyT7Ak2QF8ANgE3AAeAS4oap2T254kqZh5SK++wFgX1U9B5DkHuBa4LjBT+JjglqUSy65ZKmHsKzt37+fV199NSf63GKCfzbwwrzpA8BvLeLfk05obm5uqYewrG3cuHHQ5xYT/EGSbAG2THs9koZbTPAPAufMm17bz3ubqtoKbAVP9aXlYjF39R8Bzk2yPsmpwPXAQ5MZlqRpOukjflUdTvLHwLeAFcBXquqpiY1M0tQs6hq/qr4JfHNCY5E0Iz65JzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXohMFP8pUkh5I8OW/eqiTbk+ztX8+c7jAlTdKQI/7fAJuPmncLsKOqzgV29NOSRuKEwa+qfwJeO2r2tcCd/fs7gT+Y8LgkTdHJXuOvrqoX+/cvAasnNB5JM7DoTjpVVf9foww76UjLz8ke8V9Osgagfz10vA9W1daq2lhVw5p6SZq6kw3+Q8DH+/cfB74xmeFImoUhf867G/gX4LwkB5LcCHwO2JRkL3BVPy1pJE54jV9VNxxn0ZUTHoukGfHJPalBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBQ0pvnZNkZ5LdSZ5KclM/32460kgNOeIfBj5VVRcAlwKfSHIBdtORRmtIJ50Xq+qx/v1PgD3A2dhNRxqtBTXUSLIOuAjYxcBuOjbUkJafwTf3krwX+Dpwc1W9OX9ZVRVwzG46NtSQlp9BwU9yCl3o76qqB/rZg7vpSFpehtzVD3AHsKeqvjBvkd10pJEaco1/OfCHwPeTPNHP+3O67jn39Z11ngeum84QJU3akE46/wzkOIvtpiONkE/uSQ0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDFlRzT9O3atWqpR6CGuARX2qQwZcaNKTm3mlJvpvkX/tOOp/t569PsivJviT3Jjl1+sOVNAlDjvj/CVxRVRcCG4DNSS4FPg98sareD7wO3Di9YUqapCGddKqq/r2fPKX/KeAK4P5+vp10pBEZWld/RV9h9xCwHXgWeKOqDvcfOUDXVutY392SZC7J3CQGLGnxBgW/qn5aVRuAtcAHgPOHrsBOOtLys6C7+lX1BrATuAw4I8mR5wDWAgcnPDZJUzLkrv6vJDmjf/9uYBNdx9ydwEf7j9lJRxqRIU/urQHuTLKC7hfFfVW1Lclu4J4kfwk8TtdmS9IIDOmk8z261thHz3+O7npf0sj45J7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UoMHB70tsP55kWz9tJx1ppBZyxL+JrsjmEXbSkUZqaEONtcCHgdv76WAnHWm0hh7xvwR8GvhZP30WdtKRRmtIXf2PAIeq6tGTWYGddKTlZ0hd/cuBa5JcDZwG/BJwG30nnf6obycdaUSGdMu9tarWVtU64Hrg21X1MeykI43WYv6O/xngk0n20V3z20lHGokhp/o/V1XfAb7Tv7eTjjRSPrknNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UoEGFOJLsB34C/BQ4XFUbk6wC7gXWAfuB66rq9ekMU9IkLeSI/7tVtWFetdxbgB1VdS6wo5+WNAKLOdW/lq6RBthQQxqVocEv4B+SPJpkSz9vdVW92L9/CVg98dFJmoqhxTY/WFUHk/wqsD3J0/MXVlUlqWN9sf9FseVYyyQtjUFH/Ko62L8eAh6kq677cpI1AP3roeN810460jIzpIXW6Ul+8ch74PeAJ4GH6BppgA01pFEZcqq/Gniwa5DLSuDvqurhJI8A9yW5EXgeuG56w5Q0SScMft8448JjzP8xcOU0BiVpunxyT2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2rQ0P87TzPy2muvLfUQ1ACP+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNWhQ8JOckeT+JE8n2ZPksiSrkmxPsrd/PXPag5U0GUOP+LcBD1fV+XRluPZgJx1ptIZU2X0f8NvAHQBV9V9V9QZ20pFGa8gRfz3wCvDVJI8nub0vs20nHWmkhgR/JXAx8OWqugh4i6NO66uq6Nps/R9JtiSZSzK32MFKmowhwT8AHKiqXf30/XS/COykI43UCYNfVS8BLyQ5r591JbAbO+lIozX0f8v9E+CuJKcCzwF/RPdLw0460ggNCn5VPQEc61TdTjrSCPnkntQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtSgIXX1z0vyxLyfN5PcbCcdabyGFNt8pqo2VNUG4BLgP4AHsZOONFoLPdW/Eni2qp7HTjrSaC00+NcDd/fv7aQjjdTg4Pelta8Bvnb0MjvpSOOykCP+h4DHqurlftpOOtJILST4N/C/p/lgJx1ptAYFv++Ouwl4YN7szwGbkuwFruqnJY3A0E46bwFnHTXvx9hJRxoln9yTGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGjS09NafJXkqyZNJ7k5yWpL1SXYl2Zfk3r4Kr6QRGNJC62zgT4GNVfWbwAq6+vqfB75YVe8HXgdunOZAJU3O0FP9lcC7k6wE3gO8CFwB3N8vt5OONCJDeucdBP4K+BFd4P8NeBR4o6oO9x87AJw9rUFKmqwhp/pn0vXJWw/8GnA6sHnoCuykIy0/Q8prXwX8sKpeAUjyAHA5cEaSlf1Rfy1w8FhfrqqtwNb+u8dssyVptoZc4/8IuDTJe5KErpb+bmAn8NH+M3bSkUZkyDX+LrqbeI8B3++/sxX4DPDJJPvomm3cMcVxSpqgdI1uZ7QyT/W1SLP873WMNm7cyNzcXE70OZ/ckxpk8KUGGXypQQZfatCgNtkT9CrwVv/6TvHLuD0z0/1FebBlvS0nYcj2/PqQf2imd/UBksxV1caZrnSK3J7l6520LTDZ7fFUX2qQwZcatBTB37oE65wmt2f5eidtC0xwe2Z+jS9p6XmqLzVopsFPsjnJM32dvltmue7FSnJOkp1Jdvf1B2/q569Ksj3J3v71zKUe60IkWZHk8STb+unR1lJMckaS+5M8nWRPksvGvH+mWetyZsFPsgL4a+BDwAXADUkumNX6J+Aw8KmqugC4FPhEP/5bgB1VdS6wo58ek5uAPfOmx1xL8Tbg4ao6H7iQbrtGuX+mXuuyqmbyA1wGfGve9K3ArbNa/xS25xvAJuAZYE0/bw3wzFKPbQHbsJYuDFcA24DQPSCy8lj7bDn/AO8Dfkh/32re/FHuH7pSdi8Aq+getNsG/P6k9s8sT/WPbMgRo63Tl2QdcBGwC1hdVS/2i14CVi/RsE7Gl4BPAz/rp89ivLUU1wOvAF/tL11uT3I6I90/NeVal97cW6Ak7wW+DtxcVW/OX1bdr+FR/JkkyUeAQ1X16FKPZUJWAhcDX66qi+geDX/baf3I9s+ial2eyCyDfxA4Z970cev0LVdJTqEL/V1V9UA/++Uka/rla4BDSzW+BbocuCbJfuAeutP92+hrKfafGdM+OgAcqK5iFHRVoy5mvPvn57Uuq+q/gbfVuuw/c9L7Z5bBfwQ4t78reSrdjYqHZrj+RenrDd4B7KmqL8xb9BBdzUEYUe3Bqrq1qtZW1Tq6ffHtqvoYI62lWFUvAS8kOa+fdaQ25Cj3D9OudTnjGxZXAz8AngX+YqlvoCxw7B+kO038HvBE/3M13XXxDmAv8I/AqqUe60ls2+8A2/r3vwF8F9gHfA1411KPbwHbsQGY6/fR3wNnjnn/AJ8FngaeBP4WeNek9o9P7kkN8uae1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSg/4H2rG5brDnLUQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "processed_prev_state = preprocess_image(prev_state)\n",
    "plt.imshow(processed_prev_state,cmap='gray')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Build Model </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Network(\n",
      "  (conv_layer1): Conv2d(1, 32, kernel_size=(8, 8), stride=(4, 4))\n",
      "  (conv_layer2): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2))\n",
      "  (conv_layer3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1))\n",
      "  (fc1): Linear(in_features=3136, out_features=512, bias=True)\n",
      "  (fc2): Linear(in_features=512, out_features=4, bias=True)\n",
      "  (relu): ReLU()\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "import torch\n",
    "\n",
    "class Network(nn.Module):\n",
    "    \n",
    "    def __init__(self,image_input_size,out_size):\n",
    "        super(Network,self).__init__()\n",
    "        self.image_input_size = image_input_size\n",
    "        self.out_size = out_size\n",
    "\n",
    "        self.conv_layer1 = nn.Conv2d(in_channels=1,out_channels=32,kernel_size=8,stride=4) # GRAY - 1\n",
    "        self.conv_layer2 = nn.Conv2d(in_channels=32,out_channels=64,kernel_size=4,stride=2)\n",
    "        self.conv_layer3 = nn.Conv2d(in_channels=64,out_channels=64,kernel_size=3,stride=1)\n",
    "        self.fc1 = nn.Linear(in_features=7*7*64,out_features=512)\n",
    "        self.fc2 = nn.Linear(in_features=512,out_features=OUTPUT_SIZE)\n",
    "        self.relu = nn.ReLU()\n",
    "\n",
    "    def forward(self,x,bsize):\n",
    "        x = x.view(bsize,1,self.image_input_size,self.image_input_size) # (N,Cin,H,W) batch size, input channel, height , width\n",
    "        conv_out = self.conv_layer1(x)\n",
    "        conv_out = self.relu(conv_out)\n",
    "        conv_out = self.conv_layer2(conv_out)\n",
    "        conv_out = self.relu(conv_out)\n",
    "        conv_out = self.conv_layer3(conv_out)\n",
    "        conv_out = self.relu(conv_out)\n",
    "        out = self.fc1(conv_out.view(bsize,7*7*64))\n",
    "        out = self.relu(out)\n",
    "        out = self.fc2(out)\n",
    "        return out\n",
    "\n",
    "main_model = Network(image_input_size=INPUT_IMAGE_DIM,out_size=OUTPUT_SIZE)\n",
    "print(main_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Deep Q Learning with target freeze </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mem = Memory(memsize=MEMORY_SIZE)\n",
    "main_model = Network(image_input_size=INPUT_IMAGE_DIM,out_size=OUTPUT_SIZE).float().cuda() # Primary Network\n",
    "target_model = Network(image_input_size=INPUT_IMAGE_DIM,out_size=OUTPUT_SIZE).float().cuda() # Target Network\n",
    "\n",
    "target_model.load_state_dict(main_model.state_dict())\n",
    "criterion = nn.SmoothL1Loss()\n",
    "optimizer = torch.optim.Adam(main_model.parameters())\n",
    "\n",
    "# filling memory with transitions\n",
    "for i in range(0,MEMORY_SIZE):\n",
    "    \n",
    "    prev_state = env.reset()\n",
    "    processed_prev_state = preprocess_image(prev_state)\n",
    "    step_count = 0\n",
    "    game_over = False\n",
    "    \n",
    "    while (game_over == False) and (step_count < MAX_STEPS):\n",
    "        \n",
    "        step_count +=1\n",
    "        action = np.random.randint(0,4)\n",
    "        next_state,reward, game_over = env.step(action)\n",
    "        processed_next_state = preprocess_image(next_state)\n",
    "        mem.add_sample((processed_prev_state,action,reward,processed_next_state,game_over))\n",
    "    \n",
    "        prev_state = next_state\n",
    "        processed_prev_state = processed_next_state\n",
    "\n",
    "print('Populated %d Samples'%(len(mem.memory)))\n",
    "\n",
    "# Algorithm Starts\n",
    "total_steps = 0\n",
    "epsilon = INITIAL_EPSILON\n",
    "loss_stat = []\n",
    "total_reward_stat = []\n",
    "\n",
    "for episode in range(0,int(MEMORY_SIZE/MAX_STEPS)):\n",
    "    \n",
    "    prev_state = env.reset()\n",
    "    processed_prev_state = preprocess_image(prev_state)\n",
    "    game_over = False\n",
    "    step_count = 0\n",
    "    total_reward = 0\n",
    "    \n",
    "    while (game_over == False) and (step_count < MAX_STEPS):\n",
    "        \n",
    "        step_count +=1\n",
    "        total_steps +=1\n",
    "        \n",
    "        if np.random.rand() <= epsilon:\n",
    "            action = np.random.randint(0,4)\n",
    "        else:\n",
    "            with torch.no_grad():\n",
    "                torch_x = torch.from_numpy(processed_prev_state).float().cuda()\n",
    "\n",
    "                model_out = main_model.forward(torch_x,bsize=1)\n",
    "                action = int(torch.argmax(model_out.view(OUTPUT_SIZE),dim=0))\n",
    "                \n",
    "        next_state, reward, game_over = env.step(action)\n",
    "        processed_next_state = preprocess_image(next_state)\n",
    "        total_reward += reward\n",
    "        \n",
    "        mem.add_sample((processed_prev_state,action,reward,processed_next_state,game_over))\n",
    "        \n",
    "        prev_state = next_state\n",
    "        processed_prev_state = processed_next_state\n",
    "        \n",
    "        if (total_steps % FREEZE_INTERVAL) == 0:\n",
    "            target_model.load_state_dict(main_model.state_dict())\n",
    "        \n",
    "        batch = mem.get_batch(size=BATCH_SIZE)\n",
    "        current_states = []\n",
    "        next_states = []\n",
    "        acts = []\n",
    "        rewards = []\n",
    "        game_status = []\n",
    "        \n",
    "        for element in batch:\n",
    "            current_states.append(element[0])\n",
    "            acts.append(element[1])\n",
    "            rewards.append(element[2])\n",
    "            next_states.append(element[3])\n",
    "            game_status.append(element[4])\n",
    "            \n",
    "        current_states = np.array(current_states)\n",
    "        next_states =  np.array(next_states)\n",
    "        rewards = np.array(rewards)\n",
    "        game_status = [not b for b in game_status]\n",
    "        game_status_bool = np.array(game_status,dtype='float') # FALSE 1, TRUE 0\n",
    "        torch_acts = torch.tensor(acts)\n",
    "        \n",
    "        Q_next = target_model.forward(torch.from_numpy(next_states).float().cuda(),bsize=BATCH_SIZE)\n",
    "        Q_s = main_model.forward(torch.from_numpy(current_states).float().cuda(),bsize=BATCH_SIZE)\n",
    "        Q_max_next, _ = Q_next.detach().max(dim=1)\n",
    "        Q_max_next = Q_max_next.double()\n",
    "        Q_max_next = torch.from_numpy(game_status_bool).cuda()*Q_max_next\n",
    "        \n",
    "        target_values = (rewards + (GAMMA * Q_max_next))\n",
    "        Q_s_a = Q_s.gather(dim=1,index=torch_acts.cuda().unsqueeze(dim=1)).squeeze(dim=1)\n",
    "    \n",
    "        loss = criterion(Q_s_a,target_values.float().cuda())\n",
    "        \n",
    "        # save performance measure\n",
    "        loss_stat.append(loss.item())\n",
    "        \n",
    "        # make previous grad zero\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # back - propogate \n",
    "        loss.backward()\n",
    "        \n",
    "        # update params\n",
    "        optimizer.step()\n",
    "    \n",
    "    # save performance measure\n",
    "    total_reward_stat.append(total_reward)\n",
    "    \n",
    "    if epsilon > FINAL_EPSILON:\n",
    "        epsilon -= (INITIAL_EPSILON - FINAL_EPSILON)/TOTAL_EPISODES\n",
    "    \n",
    "    if (episode + 1)% PERFORMANCE_SAVE_INTERVAL == 0:\n",
    "        perf = {}\n",
    "        perf['loss'] = loss_stat\n",
    "        perf['total_reward'] = total_reward_stat\n",
    "        save_obj(name='SINGLE_OBSERV_NINE',obj=perf)\n",
    "    \n",
    "    #print('Completed episode : ',episode+1,' Epsilon : ',epsilon,' Reward : ',total_reward,'Loss : ',loss.item(),'Steps : ',step_count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Save Primary Network Weights </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "torch.save(main_model.state_dict(),'SINGLE_OBSERV_NINE_WEIGHTS.torch')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 align='center'> Testing Policy </h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Load Primary Network Weights </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = torch.load('data/SINGLE_OBSERV_NINE_WEIGHTS.torch', map_location='cpu')\n",
    "main_model.load_state_dict(weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Algorithm Starts\n",
    "epsilon = INITIAL_EPSILON\n",
    "FINAL_EPSILON = 0.01\n",
    "total_reward_stat = []\n",
    "\n",
    "for episode in range(0,TOTAL_EPISODES):\n",
    "    \n",
    "    prev_state = env.reset()\n",
    "    processed_prev_state = preprocess_image(prev_state)\n",
    "    game_over = False\n",
    "    step_count = 0\n",
    "    total_reward = 0\n",
    "    \n",
    "    while (game_over == False) and (step_count < MAX_STEPS):\n",
    "        \n",
    "        step_count +=1\n",
    "        \n",
    "        if np.random.rand() <= epsilon:\n",
    "            action = np.random.randint(0,4)\n",
    "        else:\n",
    "            with torch.no_grad():\n",
    "                torch_x = torch.from_numpy(processed_prev_state).float().cuda()\n",
    "\n",
    "                model_out = main_model.forward(torch_x,bsize=1)\n",
    "                action = int(torch.argmax(model_out.view(OUTPUT_SIZE),dim=0))\n",
    "                \n",
    "        next_state, reward, game_over = env.step(action)\n",
    "        processed_next_state = preprocess_image(next_state)\n",
    "        total_reward += reward\n",
    "        \n",
    "        prev_state = next_state\n",
    "        processed_prev_state = processed_next_state\n",
    "    \n",
    "    # save performance measure\n",
    "    total_reward_stat.append(total_reward)\n",
    "    \n",
    "    if epsilon > FINAL_EPSILON:\n",
    "        epsilon -= (INITIAL_EPSILON - FINAL_EPSILON)/TOTAL_EPISODES\n",
    "    \n",
    "    if (episode + 1)% PERFORMANCE_SAVE_INTERVAL == 0:\n",
    "        perf = {}\n",
    "        perf['total_reward'] = total_reward_stat\n",
    "        save_obj(name='SINGLE_OBSERV_NINE',obj=perf)\n",
    "    \n",
    "    print('Completed episode : ',episode+1,' Epsilon : ',epsilon,' Reward : ',total_reward,'Steps : ',step_count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 align='center'> Create Policy GIF </h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Collect Frames Of an Episode Using Trained Network </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADN5JREFUeJzt3X/oXfV9x/Hna4nW1m7VqAuZ0SWlosjA6IJTLGNTs1lXdH8UUcooQ/CfbtO10Or2RynsjxZGW/8YBdF2Mpw/anWVUOxcaimDkRp/rNVEm2hjTVATO52dg21p3/vjnLBvs8Scb7733u89fp4PuNx7zrk353M4vL7n3HNP3u9UFZLa8kvLPQBJs2fwpQYZfKlBBl9qkMGXGmTwpQYZfKlBSwp+kiuSPJdkV5KbJzUoSdOVY72BJ8kK4IfAJmAP8BhwXVVtn9zwJE3DyiV89kJgV1W9AJDkHuBq4IjBP/XUU2vdunVLWKWkt7N7925ee+21HO19Swn+6cBLC6b3AL/1dh9Yt24d27ZtW8IqJb2djRs3Dnrf1C/uJbkhybYk2/bv3z/t1UkaYCnB3wucsWB6bT/vF1TVbVW1sao2nnbaaUtYnaRJWUrwHwPOSrI+yfHAtcBDkxmWpGk65u/4VXUgyZ8A3wJWAF+pqmcmNjJJU7OUi3tU1TeBb05oLJJmxDv3pAYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBS/pvufMgOWpdQWniJtZcfpna1HvElxpk8KUGHTX4Sb6SZF+SpxfMW5XkkSQ7++eTpztMSZM05Ij/t8AVh8y7GdhSVWcBW/ppSSNx1OBX1XeBfztk9tXAnf3rO4E/nPC4JE3RsX7HX11VL/evXwFWT2g8kmZgyRf3quu6ecTfJOykI82fYw3+q0nWAPTP+470RjvpSPPnWIP/EPCx/vXHgG9MZjiSZmHIz3l3A/8CnJ1kT5Lrgc8Bm5LsBC7vpyWNxFFv2a2q646w6LIJj0XSjHjnntQggy81yOBLDTL4UoMMvtQggy81aPQVeObbpKurWG1oXkxqTyxP/R2P+FKTDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UoCGlt85I8miS7UmeSXJjP99uOtJIDTniHwA+WVXnAhcBH09yLnbTkUZrSCedl6vqif71T4EdwOnYTUcarUV9x0+yDjgf2MrAbjo21JDmz+DgJ3kv8HXgpqp6c+Gyt+umY0MNaf4MCn6S4+hCf1dVPdDPHtxNR9J8GXJVP8AdwI6q+sKCRXbTkUZqSAWeS4A/An6Q5Kl+3l/Qdc+5r++s8yJwzXSGKGnShnTS+WeOXGnIbjrSCHnnntQgi21OlcUxNZ884ksNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1KAhNfdOSPK9JP/ad9L5bD9/fZKtSXYluTfJ8dMfrqRJGHLE/y/g0qo6D9gAXJHkIuDzwBer6gPA68D10xumpEka0kmnquo/+snj+kcBlwL39/PtpCONyNC6+iv6Crv7gEeA54E3qupA/5Y9dG21DvdZO+lIc2ZQ8KvqZ1W1AVgLXAicM3QFdtKZoJrgQ01b1FX9qnoDeBS4GDgpycFinWuBvRMem6QpGXJV/7QkJ/Wv3w1souuY+yjwkf5tdtKRRmRIee01wJ1JVtD9obivqjYn2Q7ck+SvgCfp2mxJGoEhnXS+T9ca+9D5L9B935c0Mt65JzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNGvL/8TUvMrl/atLVtyY4NM2AR3ypQQZfatDg4Pcltp9MsrmftpOONFKLOeLfSFdk8yA76UgjNbShxlrgD4Db++lgJx1ptIYe8b8EfAr4eT99CnbSkUZrSF39DwP7qurxY1mBnXSk+TPkd/xLgKuSXAmcAPwKcCt9J53+qG8nHWlEhnTLvaWq1lbVOuBa4NtV9VHspCON1lJ+x/808Ikku+i+89tJRxqJRd2yW1XfAb7Tv7aTjjRS3rknNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIGvuNSoW3WuaR3ypQQZfapDBlxpk8KUGGXypQQZfapDBlxo06Hf8JLuBnwI/Aw5U1cYkq4B7gXXAbuCaqnp9OsOUNEmLOeL/blVtqKqN/fTNwJaqOgvY0k9LGoGlnOpfTddIA2yoIY3K0OAX8I9JHk9yQz9vdVW93L9+BVg98dFJmoqh9+p/sKr2JvlV4JEkzy5cWFWVHP7u7/4PxQ0AZ5555pIGK2kyBh3xq2pv/7wPeJCuuu6rSdYA9M/7jvBZO+lIc2ZIC60Tk/zywdfA7wFPAw/RNdIAG2pIozLkVH818GDXIJeVwN9X1cNJHgPuS3I98CJwzfSGKWmSjhr8vnHGeYeZ/xPgsmkMStJ0eeee1CCDLzXI0lutslRW0zziSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDBgU/yUlJ7k/ybJIdSS5OsirJI0l29s8nT3uwkiZj6BH/VuDhqjqHrgzXDuykI43WkCq77wN+G7gDoKr+u6rewE460mgNOeKvB/YDX03yZJLb+zLbdtKRRmpI8FcCFwBfrqrzgbc45LS+qoquzdb/k+SGJNuSbNu/f/9SxytpAoYEfw+wp6q29tP30/0hsJPOmNWEH62M7R3iqMGvqleAl5Kc3c+6DNiOnXSk0RpaZfdPgbuSHA+8APwx3R8NO+lIIzQo+FX1FLDxMIvspCONkHfuSQ0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0aUlf/7CRPLXi8meQmO+mMXCb8aGVs7xBDim0+V1UbqmoD8JvAfwIPYicdabQWe6p/GfB8Vb2InXSk0Vps8K8F7u5f20lHGqnBwe9La18FfO3QZXbSkcZlMUf8DwFPVNWr/bSddKSRWkzwr+P/TvPBTjrSaA0Kft8ddxPwwILZnwM2JdkJXN5PSxqBoZ103gJOOWTeT7CTjjRK3rknNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNWjQnXvzrPuPgZIWwyO+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNGlp668+TPJPk6SR3JzkhyfokW5PsSnJvX4VX0ggMaaF1OvBnwMaq+g1gBV19/c8DX6yqDwCvA9dPc6CSJmfoqf5K4N1JVgLvAV4GLgXu75fbSUcakSG98/YCfw38mC7w/w48DrxRVQf6t+0BTp/WICVN1pBT/ZPp+uStB34NOBG4YugK7KQjzZ8hp/qXAz+qqv1V9T90tfUvAU7qT/0B1gJ7D/dhO+lI82dI8H8MXJTkPUlCV0t/O/Ao8JH+PXbSkUZkyHf8rXQX8Z4AftB/5jbg08Ankuyia7ZxxxTHKWmChnbS+QzwmUNmvwBcOPERSZo679yTGmTwpQYZfKlBBl9qUGZZrDLJfuAt4LWZrXT6TsXtmVfvpG2BYdvz61V11BtmZhp8gCTbqmrjTFc6RW7P/HonbQtMdns81ZcaZPClBi1H8G9bhnVOk9szv95J2wIT3J6Zf8eXtPw81ZcaNNPgJ7kiyXN9nb6bZ7nupUpyRpJHk2zv6w/e2M9fleSRJDv755OXe6yLkWRFkieTbO6nR1tLMclJSe5P8mySHUkuHvP+mWaty5kFP8kK4G+ADwHnAtclOXdW65+AA8Anq+pc4CLg4/34bwa2VNVZwJZ+ekxuBHYsmB5zLcVbgYer6hzgPLrtGuX+mXqty6qayQO4GPjWgulbgFtmtf4pbM83gE3Ac8Caft4a4LnlHtsitmEtXRguBTYDobtBZOXh9tk8P4D3AT+iv261YP4o9w9dKbuXgFV0/4t2M/D7k9o/szzVP7ghB422Tl+SdcD5wFZgdVW93C96BVi9TMM6Fl8CPgX8vJ8+hfHWUlwP7Ae+2n91uT3JiYx0/9SUa116cW+RkrwX+DpwU1W9uXBZdX+GR/EzSZIPA/uq6vHlHsuErAQuAL5cVefT3Rr+C6f1I9s/S6p1eTSzDP5e4IwF00es0zevkhxHF/q7quqBfvarSdb0y9cA+5ZrfIt0CXBVkt3APXSn+7cysJbiHNoD7KmuYhR0VaMuYLz7Z0m1Lo9mlsF/DDirvyp5PN2FiodmuP4l6esN3gHsqKovLFj0EF3NQRhR7cGquqWq1lbVOrp98e2q+igjraVYVa8ALyU5u591sDbkKPcP0651OeMLFlcCPwSeB/5yuS+gLHLsH6Q7Tfw+8FT/uJLue/EWYCfwT8Cq5R7rMWzb7wCb+9fvB74H7AK+Brxruce3iO3YAGzr99E/ACePef8AnwWeBZ4G/g5416T2j3fuSQ3y4p7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKD/hfLSyKxO352BgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "full_env = gameEnv(partial=False,size=9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Reward : 0\n"
     ]
    }
   ],
   "source": [
    "frames = []\n",
    "local_frames = []\n",
    "\n",
    "random.seed(110)\n",
    "np.random.seed(110)\n",
    "\n",
    "for episode in range(0,1):\n",
    "    \n",
    "    prev_state = env.reset()\n",
    "    random.seed(110)\n",
    "    np.random.seed(110)\n",
    "    full_env_prev = full_env.reset()\n",
    "    \n",
    "    processed_prev_state = preprocess_image(prev_state)\n",
    "    \n",
    "    frames.append(full_env_prev)\n",
    "    local_frames.append(prev_state)\n",
    "    \n",
    "    game_over = False\n",
    "    step_count = 0\n",
    "    total_reward = 0\n",
    "    \n",
    "    while (game_over == False) and (step_count < MAX_STEPS):\n",
    "        \n",
    "        step_count +=1\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            torch_x = torch.from_numpy(processed_prev_state).float()\n",
    "            model_out = main_model.forward(torch_x,bsize=1)\n",
    "            action = int(torch.argmax(model_out.view(OUTPUT_SIZE),dim=0))\n",
    "                \n",
    "        next_state, reward, game_over = env.step(action)\n",
    "        full_env_next,r,g = full_env.step(action)\n",
    "        frames.append(full_env_next)\n",
    "        local_frames.append(next_state)\n",
    "        processed_next_state = preprocess_image(next_state)\n",
    "        total_reward += reward\n",
    "        \n",
    "        prev_state = next_state\n",
    "        processed_prev_state = processed_next_state\n",
    "        full_env_prev = full_env_next\n",
    "\n",
    "print('Total Reward : %d'%(total_reward))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7efdab6f5390>"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADNhJREFUeJzt3X/oXfV9x/Hna4nW1m41URcyo/umVBQZGF1wimVsajbriu6PIkoZZQj+0226Ftq4/VEK+6OF0dY/RkG0nQznj1pdJRQ7l1rKYKTGH2s10SbaWBPUxE5n52Bb2vf+OCfs2ywx55vvvff7PX6eD7jce86993s+h8PrnnPPPd/3O1WFpLb80lIPQNLsGXypQQZfapDBlxpk8KUGGXypQQZfatCigp/kyiTPJdmdZPOkBiVpunK8F/AkWQH8ENgE7AUeA66vqh2TG56kaVi5iPdeBOyuqhcAktwDXAMcNfinnXZazc3NLWKRkt7Onj17eO2113Ks1y0m+GcAL82b3gv81tu9YW5uju3bty9ikZLezsaNGwe9buon95LcmGR7ku0HDhyY9uIkDbCY4O8Dzpw3va6f9wuq6raq2lhVG08//fRFLE7SpCwm+I8BZydZn+RE4DrgockMS9I0Hfd3/Ko6mORPgG8BK4CvVNUzExuZpKlZzMk9quqbwDcnNBZJM+KVe1KDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UoEX9W+5ykByzrqA0cRNrLr9Eberd40sNMvhSg44Z/CRfSbI/ydPz5q1O8kiSXf39qukOU9IkDdnj/y1w5WHzNgNbq+psYGs/LWkkjhn8qvou8G+Hzb4GuLN/fCfwhxMel6QpOt7v+Guq6uX+8SvAmgmNR9IMLPrkXnVdN4/6m4SddKTl53iD/2qStQD9/f6jvdBOOtLyc7zBfwj4WP/4Y8A3JjMcSbMw5Oe8u4F/Ac5JsjfJDcDngE1JdgFX9NOSRuKYl+xW1fVHeeryCY9F0ox45Z7UIIMvNcjgSw0y+FKDDL7UIIMvNWj0FXikpTCpuk9LU3/HPb7UJIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNWhI6a0zkzyaZEeSZ5Lc1M+3m440UkP2+AeBT1bVecDFwMeTnIfddKTRGtJJ5+WqeqJ//FNgJ3AGdtORRmtB3/GTzAEXANsY2E3HhhrS8jM4+EneC3wduLmq3pz/3Nt107GhhrT8DAp+khPoQn9XVT3Qzx7cTUfS8jLkrH6AO4CdVfWFeU/ZTUcaqSEVeC4F/gj4QZKn+nl/Qdc9576+s86LwLXTGaKkSRvSSeefOXqlIbvpSCPklXtSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSg4bU3DspyfeS/GvfSeez/fz1SbYl2Z3k3iQnTn+4kiZhyB7/v4DLqup8YANwZZKLgc8DX6yqDwCvAzdMb5iSJmlIJ52qqv/oJ0/obwVcBtzfz7eTjjQiQ+vqr+gr7O4HHgGeB96oqoP9S/bStdU60nvtpCMtM4OCX1U/q6oNwDrgIuDcoQtou5NOTfY2yT+npi3orH5VvQE8ClwCnJLkUHnudcC+CY9N0pQMOat/epJT+sfvBjbRdcx9FPhI/zI76UgjMqSTzlrgziQr6D4o7quqLUl2APck+SvgSbo2W5JGYEgnne/TtcY+fP4LdN/3JY2MV+5JDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoOG/D++jluW7Z+bdPWtCa+ppsw9vtQggy81aHDw+xLbTybZ0k/bSUcaqYXs8W+iK7J5iJ10pJEa2lBjHfAHwO39dLCTjjRaQ/f4XwI+Bfy8nz4VO+lIozWkrv6Hgf1V9fjxLKDtTjrS8jTkd/xLgauTXAWcBPwKcCt9J51+r28nHWlEhnTLvaWq1lXVHHAd8O2q+ih20pFGazG/438a+ESS3XTf+e2kI43Egi7ZrarvAN/pH9tJRxopr9yTGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkDX3GhWL7jXNPb7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0a9Dt+kj3AT4GfAQeramOS1cC9wBywB7i2ql6fzjAlTdJC9vi/W1UbqmpjP70Z2FpVZwNb+2lJI7CYQ/1r6BppgA01pFEZGvwC/jHJ40lu7OetqaqX+8evAGsmPjpJUzH0Wv0PVtW+JL8KPJLk2flPVlUlR776u/+guBHgrLPOWtRgJU3GoD1+Ve3r7/cDD9JV1301yVqA/n7/Ud5rJx1pmRnSQuvkJL986DHwe8DTwEN0jTTAhhrSqAw51F8DPNg1yGUl8PdV9XCSx4D7ktwAvAhcO71hSpqkYwa/b5xx/hHm/wS4fBqDkjRdXrknNcjgSw2y9FarLJXVNPf4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtSgQcFPckqS+5M8m2RnkkuSrE7ySJJd/f2qaQ9W0mQM3ePfCjxcVefSleHaiZ10pNEaUmX3fcBvA3cAVNV/V9Ub2ElHGq0he/z1wAHgq0meTHJ7X2bbTjrSSA0J/krgQuDLVXUB8BaHHdZXVdG12fp/ktyYZHuS7QcOHFjseCVNwJDg7wX2VtW2fvp+ug8CO+mMWU341srY3iGOGfyqegV4Kck5/azLgR3YSUcaraFVdv8UuCvJicALwB/TfWjYSUcaoUHBr6qngI1HeMpOOtIIeeWe1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1KAhdfXPSfLUvNubSW62k87IZcK3Vsb2DjGk2OZzVbWhqjYAvwn8J/AgdtKRRmuhh/qXA89X1YvYSUcarYUG/zrg7v6xnXSkkRoc/L609tXA1w5/zk460rgsZI//IeCJqnq1n7aTjjRSCwn+9fzfYT7YSUcarUHB77vjbgIemDf7c8CmJLuAK/ppSSMwtJPOW8Cph837CXbSkUbJK/ekBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBg26cm856/4xUNJCuMeXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBQ0tv/XmSZ5I8neTuJCclWZ9kW5LdSe7tq/BKGoEhLbTOAP4M2FhVvwGsoKuv/3ngi1X1AeB14IZpDlTS5Aw91F8JvDvJSuA9wMvAZcD9/fN20pFGZEjvvH3AXwM/pgv8vwOPA29U1cH+ZXuBM6Y1SEmTNeRQfxVdn7z1wK8BJwNXDl2AnXSk5WfIof4VwI+q6kBV/Q9dbf1LgVP6Q3+AdcC+I73ZTjrS8jMk+D8GLk7yniShq6W/A3gU+Ej/GjvpSCMy5Dv+NrqTeE8AP+jfcxvwaeATSXbTNdu4Y4rjlDRBQzvpfAb4zGGzXwAumviIJE2dV+5JDTL4UoMMvtQggy81KLMsVpnkAPAW8NrMFjp9p+H6LFfvpHWBYevz61V1zAtmZhp8gCTbq2rjTBc6Ra7P8vVOWheY7Pp4qC81yOBLDVqK4N+2BMucJtdn+XonrQtMcH1m/h1f0tLzUF9q0EyDn+TKJM/1dfo2z3LZi5XkzCSPJtnR1x+8qZ+/OskjSXb196uWeqwLkWRFkieTbOmnR1tLMckpSe5P8mySnUkuGfP2mWaty5kFP8kK4G+ADwHnAdcnOW9Wy5+Ag8Anq+o84GLg4/34NwNbq+psYGs/PSY3ATvnTY+5luKtwMNVdS5wPt16jXL7TL3WZVXN5AZcAnxr3vQtwC2zWv4U1ucbwCbgOWBtP28t8NxSj20B67COLgyXAVuA0F0gsvJI22w534D3AT+iP281b/4otw9dKbuXgNV0/0W7Bfj9SW2fWR7qH1qRQ0Zbpy/JHHABsA1YU1Uv90+9AqxZomEdjy8BnwJ+3k+fynhrKa4HDgBf7b+63J7kZEa6fWrKtS49ubdASd4LfB24uarenP9cdR/Do/iZJMmHgf1V9fhSj2VCVgIXAl+uqgvoLg3/hcP6kW2fRdW6PJZZBn8fcOa86aPW6VuukpxAF/q7quqBfvarSdb2z68F9i/V+BboUuDqJHuAe+gO929lYC3FZWgvsLe6ilHQVY26kPFun0XVujyWWQb/MeDs/qzkiXQnKh6a4fIXpa83eAews6q+MO+ph+hqDsKIag9W1S1Vta6q5ui2xber6qOMtJZiVb0CvJTknH7WodqQo9w+TLvW5YxPWFwF/BB4HvjLpT6BssCxf5DuMPH7wFP97Sq678VbgV3APwGrl3qsx7FuvwNs6R+/H/gesBv4GvCupR7fAtZjA7C930b/AKwa8/YBPgs8CzwN/B3wrkltH6/ckxrkyT2pQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUG/S8PvTWbhFz4zgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(frames[14])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image, ImageDraw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "box = [[(15, 8), (37, 30)],\n",
    "       [(15, 16), (37, 38)],\n",
    "       [(15, 23), (37, 45)],\n",
    "       [(15, 30), (37, 52)]\n",
    "      ]\n",
    "box_index = [0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3\n",
    "            , 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2]\n",
    "\n",
    "for idx, img in enumerate(frames):\n",
    "    image = Image.fromarray(img)\n",
    "    drawer = ImageDraw.Draw(image)\n",
    "    drawer.rectangle(box[box_index[idx]], outline=(255, 255, 0))\n",
    "    frames[idx] = np.array(image)\n",
    "    #break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "51"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(box_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "51"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(frames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7efdab684c88>"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADPpJREFUeJzt3V+sHPV5xvHvUxtCQtqAgVouhh5XQSBUCUMtCiKqWsAtoRH0IkKgqIoqJG7SFppICbQXUaReJFKVhIsqEoKkqKL8CSENsiJS6hBVlSoH86cJ2BAbYoItwCaFklKprZO3FzNWT1wbz/HZ3XPm/L4fabU7s7ua33j87MzOmX3fVBWS2vILSz0ASbNn8KUGGXypQQZfapDBlxpk8KUGGXypQYsKfpKrkjyfZHeSWyc1KEnTleO9gCfJKuAHwGZgL/A4cENV7Zjc8CRNw+pFvPdiYHdVvQiQ5D7gWuCowT/99NNrbm5uEYuU9E727NnD66+/nmO9bjHBPxN4ed70XuA33+kNc3NzbN++fRGLlPRONm3aNOh1Uz+5l+SmJNuTbD9w4MC0FydpgMUEfx9w1rzp9f28n1NVd1TVpqradMYZZyxicZImZTHBfxw4J8mGJCcC1wMPT2ZYkqbpuL/jV9XBJH8MfAtYBXy5qp6d2MgkTc1iTu5RVd8EvjmhsUiaEa/ckxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGLepnuctBcsy6goO01i18Qv9szZrYf5cl+o/nHl9q0Oj3+NOwUveGrR3V6OiOucdP8uUk+5M8M2/emiSPJtnV35863WFKmqQhh/p/A1x12Lxbga1VdQ6wtZ+WNBLHDH5V/RPwb4fNvha4u398N/AHEx6XpCk63pN7a6vqlf7xq8DaCY1H0gws+qx+dV03j3rayE460vJzvMF/Lck6gP5+/9FeaCcdafk53uA/DHy0f/xR4BuTGY6kWRjy57x7gX8Bzk2yN8mNwGeBzUl2AVf205JG4pgX8FTVDUd56ooJj0XSjHjJrtQggy81yOBLDTL4UoMMvtQggy81yN/jT9WkfwC/QgsFjNCktsRSlUhwjy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDRpSeuusJI8l2ZHk2SQ39/PtpiON1JA9/kHgE1V1PnAJ8LEk52M3HWm0hnTSeaWqnuwf/wTYCZyJ3XSk0VrQd/wkc8CFwDYGdtOxoYa0/AwOfpL3Al8Dbqmqt+Y/907ddGyoIS0/g4Kf5AS60N9TVQ/1swd305G0vAw5qx/gLmBnVX1+3lN205FGakgFnsuAPwS+n+Tpft6f03XPeaDvrPMScN10hihp0oZ00vlnjl5pyG460gh55Z7UIIttTpXFMbU8uceXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBQ2runZTku0n+te+k85l+/oYk25LsTnJ/khOnP1xJkzBkj/9fwOVVdQGwEbgqySXA54AvVNX7gTeAG6c3TEmTNKSTTlXVf/STJ/S3Ai4HHuznr6hOOlUr8yYdMrSu/qq+wu5+4FHgBeDNqjrYv2QvXVutI73XTjrSMjMo+FX106raCKwHLgbOG7qAsXTSSUZwY3I3tW1BZ/Wr6k3gMeBS4JQkh4p1rgf2TXhskqZkyFn9M5Kc0j9+N7CZrmPuY8CH+5fZSUcakSHltdcBdydZRfdB8UBVbUmyA7gvyV8CT9G12ZI0AkM66XyPrjX24fNfpPu+L2lkvHJPapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxo05Pf4Wi4mWDNr0rU3Lec1Lu7xpQYZfKlBg4Pfl9h+KsmWftpOOtJILWSPfzNdkc1D7KQjjdTQhhrrgd8H7uynwwrupCOtdEP3+F8EPgn8rJ8+DTvpSKM1pK7+h4D9VfXE8SxgLJ10pJYM+Tv+ZcA1Sa4GTgJ+CbidvpNOv9e3k440IkO65d5WVeurag64Hvh2VX0EO+lIo7WYv+N/Cvh4kt103/ntpCONxIIu2a2q7wDf6R/bSUcaKa/ckxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapA19xoVi+41zT2+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNGvR3/CR7gJ8APwUOVtWmJGuA+4E5YA9wXVW9MZ1hSpqkhezxf6eqNlbVpn76VmBrVZ0DbO2nJY3AYg71r6VrpAE21JBGZWjwC/iHJE8kuamft7aqXukfvwqsnfjoJE3F0Gv1P1BV+5L8MvBokufmP1lVlRz56u/+g+ImgLPPPntRg5U0GYP2+FW1r7/fD3ydrrrua0nWAfT3+4/yXjvpSMvMkBZaJyf5xUOPgd8FngEepmukATbUkEZlyKH+WuDrXYNcVgN/V1WPJHkceCDJjcBLwHXTG6akSTpm8PvGGRccYf6PgSumMShJ0+WVe1KDDL7UIEtvtcpSWU1zjy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDRoU/CSnJHkwyXNJdia5NMmaJI8m2dXfnzrtwUqajKF7/NuBR6rqPLoyXDuxk440WkOq7L4P+C3gLoCq+u+qehM76UijNWSPvwE4AHwlyVNJ7uzLbNtJRxqpIcFfDVwEfKmqLgTe5rDD+qoqujZb/0+Sm5JsT7L9wIEDix2vpAkYEvy9wN6q2tZPP0j3QWAnnTGrCd9aGdsKcczgV9WrwMtJzu1nXQHswE460mgNrbL7J8A9SU4EXgT+iO5Dw0460ggNCn5VPQ1sOsJTdtKRRsgr96QGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGDamrf26Sp+fd3kpyi510Ri4TvrUythViSLHN56tqY1VtBH4D+E/g69hJRxqthR7qXwG8UFUvYScdabQWGvzrgXv7x3bSkUZqcPD70trXAF89/Dk76UjjspA9/geBJ6vqtX7aTjrSSC0k+Dfwf4f5YCcdabQGBb/vjrsZeGje7M8Cm5PsAq7spyWNwNBOOm8Dpx0278fYSUcaJa/ckxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxo06Mq95az7YaCkhXCPLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSg4aW3vqzJM8meSbJvUlOSrIhybYku5Pc31fhlTQCQ1ponQn8KbCpqn4dWEVXX/9zwBeq6v3AG8CN0xyopMkZeqi/Gnh3ktXAe4BXgMuBB/vn7aQjjciQ3nn7gL8CfkQX+H8HngDerKqD/cv2AmdOa5CSJmvIof6pdH3yNgC/ApwMXDV0AXbSkZafIYf6VwI/rKoDVfU/dLX1LwNO6Q/9AdYD+470ZjvpSMvPkOD/CLgkyXuShK6W/g7gMeDD/WvspCONyJDv+NvoTuI9CXy/f88dwKeAjyfZTdds464pjlPSBA3tpPNp4NOHzX4RuHjiI5I0dV65JzXI4EsNMvhSgwy+1KDMslhlkgPA28DrM1vo9J2O67NcraR1gWHr86tVdcwLZmYafIAk26tq00wXOkWuz/K1ktYFJrs+HupLDTL4UoOWIvh3LMEyp8n1Wb5W0rrABNdn5t/xJS09D/WlBs00+EmuSvJ8X6fv1lkue7GSnJXksSQ7+vqDN/fz1yR5NMmu/v7UpR7rQiRZleSpJFv66dHWUkxySpIHkzyXZGeSS8e8faZZ63JmwU+yCvhr4IPA+cANSc6f1fIn4CDwiao6H7gE+Fg//luBrVV1DrC1nx6Tm4Gd86bHXEvxduCRqjoPuIBuvUa5faZe67KqZnIDLgW+NW/6NuC2WS1/CuvzDWAz8Dywrp+3Dnh+qce2gHVYTxeGy4EtQOguEFl9pG22nG/A+4Af0p+3mjd/lNuHrpTdy8Aaul/RbgF+b1LbZ5aH+odW5JDR1ulLMgdcCGwD1lbVK/1TrwJrl2hYx+OLwCeBn/XTpzHeWoobgAPAV/qvLncmOZmRbp+acq1LT+4tUJL3Al8Dbqmqt+Y/V93H8Cj+TJLkQ8D+qnpiqccyIauBi4AvVdWFdJeG/9xh/ci2z6JqXR7LLIO/Dzhr3vRR6/QtV0lOoAv9PVX1UD/7tSTr+ufXAfuXanwLdBlwTZI9wH10h/u3M7CW4jK0F9hbXcUo6KpGXcR4t8+ial0eyyyD/zhwTn9W8kS6ExUPz3D5i9LXG7wL2FlVn5/31MN0NQdhRLUHq+q2qlpfVXN02+LbVfURRlpLsapeBV5Ocm4/61BtyFFuH6Zd63LGJyyuBn4AvAD8xVKfQFng2D9Ad5j4PeDp/nY13ffircAu4B+BNUs91uNYt98GtvSPfw34LrAb+CrwrqUe3wLWYyOwvd9Gfw+cOubtA3wGeA54Bvhb4F2T2j5euSc1yJN7UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDfpfFCx/v72HUn0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(frames[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Frames to GIF </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mayank/miniconda3/envs/rdqn/lib/python3.5/site-packages/ipykernel_launcher.py:7: DeprecationWarning: `imresize` is deprecated!\n",
      "`imresize` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n",
      "Use ``skimage.transform.resize`` instead.\n",
      "  import sys\n",
      "/home/mayank/miniconda3/envs/rdqn/lib/python3.5/site-packages/ipykernel_launcher.py:8: DeprecationWarning: `imresize` is deprecated!\n",
      "`imresize` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n",
      "Use ``skimage.transform.resize`` instead.\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "import imageio\n",
    "from scipy.misc import imresize\n",
    "resized_frames = []\n",
    "resized_local_frames = []\n",
    "\n",
    "for i in range(0,len(frames)):\n",
    "    resized_frames.append(imresize(frames[i],(256,256)))\n",
    "    resized_local_frames.append(imresize(local_frames[i],(256,256)))\n",
    "\n",
    "imageio.mimsave('data/GIFs/SINGLE_SIZE_9_frames.gif',resized_frames,fps=4)\n",
    "imageio.mimsave('data/GIFs/SINGEL_SIZE_9_local.gif',resized_local_frames,fps=4)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}