MDP_Size_9.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gridworld import gameEnv\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import deque\n",
    "import pickle\n",
    "from skimage.color import rgb2gray\n",
    "import random"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Define Environment Object </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADONJREFUeJzt3V+sHPV5xvHvUxtCQtqAgVouhh5XQSBUCUMtCiKqWsAtIRH0IkKgqIoqJG7SFppIiWkvoki9SKQqCRdVJARJUUX5EwKNZUWk1CGqKlUO5k8TsCE2xARbBpsUSkqltk7eXuy4PXFtzhyf3T07/n0/0urszOye+Y1Hz5nZ2fH7pqqQ1JZfWO4BSJo+gy81yOBLDTL4UoMMvtQggy81yOBLDVpS8JNck+SFJLuTbBrXoCRNVo73Bp4kK4AfABuBvcATwE1VtWN8w5M0CSuX8N5Lgd1V9RJAkvuB64FjBv/MM8+subm5JaxS0jvZs2cPr7/+ehZ63VKCfzbwyrzpvcBvvtMb5ubm2L59+xJWKemdbNiwodfrJn5xL8ktSbYn2X7w4MFJr05SD0sJ/j7gnHnTa7t5P6eq7qyqDVW14ayzzlrC6iSNy1KC/wRwXpJ1SU4GbgQ2j2dYkibpuD/jV9WhJH8EfAtYAXylqp4b28gkTcxSLu5RVd8EvjmmsUiaEu/ckxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGLem/5c6CZMG6gtLMWq429R7xpQYZfKlBCwY/yVeSHEjy7Lx5q5I8lmRX9/P0yQ5T0jj1OeL/NXDNEfM2AVur6jxgazctaSAWDH5V/SPwr0fMvh64p3t+D/D7Yx6XpAk63s/4q6tqf/f8VWD1mMYjaQqWfHGvRt9HHPM7CTvpSLPneIP/WpI1AN3PA8d6oZ10pNlzvMHfDHyse/4x4BvjGY6kaejzdd59wD8D5yfZm+Rm4HPAxiS7gKu7aUkDseAtu1V10zEWXTXmsUiaEu/ckxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxrUp/TWOUkeT7IjyXNJbu3m201HGqg+R/xDwCer6kLgMuDjSS7EbjrSYPXppLO/qp7qnv8E2Amcjd10pMFa1Gf8JHPAxcA2enbTsaGGNHt6Bz/Je4GvA7dV1Vvzl71TNx0bakizp1fwk5zEKPT3VtXD3eze3XQkzZY+V/UD3A3srKovzFtkNx1poBZsqAFcAfwB8P0kz3Tz/oxR95wHu846LwM3TGaIksatTyedfwJyjMV205EGyDv3pAb1OdXXCeioX8EsQcb5C491fjlDxv3vN20e8aUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfalCfmnunJPlukn/pOul8tpu/Lsm2JLuTPJDk5MkPV9I49Dni/ydwZVVdBKwHrklyGfB54ItV9X7gDeDmyQ1T0jj16aRTVfXv3eRJ3aOAK4GHuvl20pEGpG9d/RVdhd0DwGPAi8CbVXWoe8leRm21jvZeO+lIM6ZX8Kvqp1W1HlgLXApc0HcFdtKZTRnzY7y/bPYNfVMXdVW/qt4EHgcuB05LcrhY51pg35jHJmlC+lzVPyvJad3zdwMbGXXMfRz4SPcyO+lIA9KnvPYa4J4kKxj9oXiwqrYk2QHcn+QvgKcZtdmSNAB9Oul8j1Fr7CPnv8To876kgfHOPalBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBvYPfldh+OsmWbtpOOtJALeaIfyujIpuH2UlHGqi+DTXWAh8C7uqmg510pMHqe8T/EvAp4Gfd9BnYSUcarD519T8MHKiqJ49nBXbSkWZPn7r6VwDXJbkWOAX4JeAOuk463VHfTjrSgPTplnt7Va2tqjngRuDbVfVR7KQjDdZSvsf/NPCJJLsZfea3k440EH1O9f9XVX0H+E733E460kB5557UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDFnUDjxapxvz7Mubfp2Z5xJcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUG9vsdPsgf4CfBT4FBVbUiyCngAmAP2ADdU1RuTGaakcVrMEf93qmp9VW3opjcBW6vqPGBrNy1pAJZyqn89o0YaYEMNaVD6Br+Av0/yZJJbunmrq2p/9/xVYPXYRydpIvreq/+BqtqX5JeBx5I8P39hVVWSo96Z3v2huAXg3HPPXdJgJY1HryN+Ve3rfh4AHmFUXfe1JGsAup8HjvFeO+lIM6ZPC61Tk/zi4efA7wLPApsZNdIAG2pIg9LnVH818MioQS4rgb+tqkeTPAE8mORm4GXghskNU9I4LRj8rnHGRUeZ/2PgqkkMStJkeeee1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1KBewU9yWpKHkjyfZGeSy5OsSvJYkl3dz9MnPVhJ49H3iH8H8GhVXcCoDNdO7KQjDVafKrvvA34LuBugqv6rqt7ETjrSYPU54q8DDgJfTfJ0kru6Mtt20pEGqk/wVwKXAF+uqouBtznitL6qilGbrf8nyS1JtifZfvDgwaWOV9IY9An+XmBvVW3rph9i9IfATjoLyZgfs6zG+NDELRj8qnoVeCXJ+d2sq4Ad2ElHGqy+TTP/GLg3ycnAS8AfMvqjYScdaYB6Bb+qngE2HGWRnXSkAfLOPalBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBferqn5/kmXmPt5LcduJ10hlntcgGq0a2UlT0BNGn2OYLVbW+qtYDvwH8B/AIdtKRBmuxp/pXAS9W1cvYSUcarMUG/0bgvu65nXSkgeod/K609nXA145cZicdaVgWc8T/IPBUVb3WTdtJRxqoxQT/Jv7vNB/spCMNVq/gd91xNwIPz5v9OWBjkl3A1d20pAHo20nnbeCMI+b9GDvpSIPknXtSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSg3rduTfLRv8xcFbN8tjUMo/4UoMMvtQggy81yOBLDTL4UoMMvtQggy81qG/prT9N8lySZ5Pcl+SUJOuSbEuyO8kDXRVeSQPQp4XW2cCfABuq6teBFYzq638e+GJVvR94A7h5kgOVND59T/VXAu9OshJ4D7AfuBJ4qFtuJx1pQPr0ztsH/CXwI0aB/zfgSeDNqjrUvWwvcPakBilpvPqc6p/OqE/eOuBXgFOBa/quwE460uzpc6p/NfDDqjpYVf/NqLb+FcBp3ak/wFpg39HebCcdafb0Cf6PgMuSvCdJGNXS3wE8Dnyke42ddKQB6fMZfxuji3hPAd/v3nMn8GngE0l2M2q2cfcExylpjPp20vkM8JkjZr8EXDr2EUmaOO/ckxpk8KUGGXypQQZfalCmWawyyUHgbeD1qa108s7E7ZlVJ9K2QL/t+dWqWvCGmakGHyDJ9qraMNWVTpDbM7tOpG2B8W6Pp/pSgwy+1KDlCP6dy7DOSXJ7ZteJtC0wxu2Z+md8ScvPU32pQVMNfpJrkrzQ1enbNM11L1WSc5I8nmRHV3/w1m7+qiSPJdnV/Tx9uce6GElWJHk6yZZuerC1FJOcluShJM8n2Znk8iHvn0nWupxa8JOsAP4K+CBwIXBTkguntf4xOAR8sqouBC4DPt6NfxOwtarOA7Z200NyK7Bz3vSQayneATxaVRcAFzHarkHun4nXuqyqqTyAy4FvzZu+Hbh9WuufwPZ8A9gIvACs6eatAV5Y7rEtYhvWMgrDlcAWIIxuEFl5tH02yw/gfcAP6a5bzZs/yP3DqJTdK8AqRv+Ldgvwe+PaP9M81T+8IYcNtk5fkjngYmAbsLqq9neLXgVWL9OwjseXgE8BP+umz2C4tRTXAQeBr3YfXe5KcioD3T814VqXXtxbpCTvBb4O3FZVb81fVqM/w4P4miTJh4EDVfXkco9lTFYClwBfrqqLGd0a/nOn9QPbP0uqdbmQaQZ/H3DOvOlj1umbVUlOYhT6e6vq4W72a0nWdMvXAAeWa3yLdAVwXZI9wP2MTvfvoGctxRm0F9hbo4pRMKoadQnD3T9LqnW5kGkG/wngvO6q5MmMLlRsnuL6l6SrN3g3sLOqvjBv0WZGNQdhQLUHq+r2qlpbVXOM9sW3q+qjDLSWYlW9CryS5Pxu1uHakIPcP0y61uWUL1hcC/wAeBH48+W+gLLIsX+A0Wni94Bnuse1jD4XbwV2Af8ArFrusR7Htv02sKV7/mvAd4HdwNeAdy33+BaxHeuB7d0++jvg9CHvH+CzwPPAs8DfAO8a1/7xzj2pQV7ckxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfatD/ADPvEPBps6hQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "env = gameEnv(partial=False,size=9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f5c7ad46b70>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADPlJREFUeJzt3V+sHPV5xvHvUxtCQtqAgVouhh5XQSBUCUMtCiKqWsAtIRH0IkKgqIoqJG7SFppICbQXUaReJFKVhIsqEoKkqKL8CYEGWREpdYiiSJWD+dMEbIgNMcEWYJNCSanU1snbix23B9fGc3xmz9nx7/uRVrszu3vmN2f0nJmdnfO+qSokteWXlnsAkpaewZcaZPClBhl8qUEGX2qQwZcaZPClBi0q+EmuSPJckp1Jbh5qUJKmK0d7AU+SFcCPgI3AbuAx4Lqq2jbc8CRNw8pFvPdCYGdVvQCQ5B7gauCwwT/11FNrbm5uEYuU9E527drFa6+9liO9bjHBPx14ad70buC33+kNc3NzbN26dRGLlPRONmzY0Ot1Uz+5l+SGJFuTbN23b9+0Fyeph8UEfw9wxrzptd28t6mq26pqQ1VtOO200xaxOElDWUzwHwPOSrIuyfHAtcBDwwxL0jQd9Wf8qtqf5E+AbwErgK9U1TODjUzS1Czm5B5V9U3gmwONRdIS8co9qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2rQov4tdxYkR6wr2M80uoUPNDQdu5arTb17fKlBBl9q0BGDn+QrSfYmeXrevFVJHkmyo7s/ebrDlDSkPnv8vwWuOGjezcDmqjoL2NxNSxqJIwa/qr4L/OtBs68G7uwe3wn84cDjkjRFR/sZf3VVvdw9fgVYPdB4JC2BRZ/cq8n3EYf9TsJOOtLsOdrgv5pkDUB3v/dwL7STjjR7jjb4DwEf6x5/DPjGMMORtBT6fJ13N/DPwNlJdie5HvgcsDHJDuDyblrSSBzxkt2quu4wT1028FgkLRGv3JMaZPClBhl8qUEGX2qQwZcaZPClBo2+As9grJajhrjHlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUG9Sm9dUaSR5NsS/JMkhu7+XbTkUaqzx5/P/DJqjoXuAj4eJJzsZuONFp9Oum8XFVPdI9/BmwHTsduOtJoLegzfpI54HxgCz276dhQQ5o9vYOf5L3A14GbqurN+c+9UzcdG2pIs6dX8JMcxyT0d1XVA93s3t10JM2WPmf1A9wBbK+qL8x7ym460kj1qcBzCfBHwA+TPNXN+wsm3XPu6zrrvAhcM50hShpan0463+PwhanspiONkFfuSQ2y2OaoHPKLk6PUUHXRIX9tB4z81+ceX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBvWpuXdCku8n+Zeuk85nu/nrkmxJsjPJvUmOn/5wJQ2hzx7/P4FLq+o8YD1wRZKLgM8DX6yq9wOvA9dPb5iShtSnk05V1b93k8d1twIuBe7v5ttJRxqRvnX1V3QVdvcCjwDPA29U1f7uJbuZtNU61HvtpCPNmF7Br6qfV9V6YC1wIXBO3wXYSWdIGfDWkCF/bcfIr29BZ/Wr6g3gUeBi4KQkB4p1rgX2DDw2SVPS56z+aUlO6h6/G9jIpGPuo8BHupfZSUcakT7ltdcAdyZZweQPxX1VtSnJNuCeJH8FPMmkzZakEejTSecHTFpjHzz/BSaf9yWNjFfuSQ0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw3qHfyuxPaTSTZ103bSkUZqIXv8G5kU2TzATjrSSPVtqLEW+BBwezcd7KQjjVbfPf6XgE8Bv+imT8FOOtJo9amr/2Fgb1U9fjQLsJOONHv61NW/BLgqyZXACcCvALfSddLp9vp20pFGpE+33Fuqam1VzQHXAt+uqo9iJx1ptBbzPf6ngU8k2cnkM7+ddKSR6HOo/7+q6jvAd7rHdtKRRsor96QGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxrUqxBHkl3Az4CfA/urakOSVcC9wBywC7imql6fzjAlDWkhe/zfq6r1VbWhm74Z2FxVZwGbu2lJI7CYQ/2rmTTSABtqSKPSN/gF/GOSx5Pc0M1bXVUvd49fAVYPPjpJU9G32OYHqmpPkl8FHkny7Pwnq6qS1KHe2P2huAHgzDPPXNRgJQ2j1x6/qvZ093uBB5lU1301yRqA7n7vYd5rJx1pxvRpoXVikl8+8Bj4feBp4CEmjTTAhhrSqPQ51F8NPDhpkMtK4O+r6uEkjwH3JbkeeBG4ZnrDlDSkIwa/a5xx3iHm/xS4bBqDkjRdXrknNcjgSw1aUO+8mXTILxGP4sdkmJ8z3xR+pDQI9/hSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1CCDLzXI4EsNMvhSgwy+1KBewU9yUpL7kzybZHuSi5OsSvJIkh3d/cnTHqykYfTd498KPFxV5zApw7UdO+lIo9Wnyu77gN8B7gCoqv+qqjewk440Wn32+OuAfcBXkzyZ5PauzLaddKSR6hP8lcAFwJer6nzgLQ46rK+q4jBFsJLckGRrkq379u1b7HglDaBP8HcDu6tqSzd9P5M/BLPRSSfD3Ab6MW+76dhVA92WyxGDX1WvAC8lObubdRmwDTvpSKPVt8runwJ3JTkeeAH4YyZ/NOykI41Qr+BX1VPAhkM8ZScdaYS8ck9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qUJ+6+mcneWre7c0kN9lJp4ehKjIud2VG/T9jL8jap9jmc1W1vqrWA78F/AfwIHbSkUZroYf6lwHPV9WL2ElHGq2FBv9a4O7usZ10pJHqHfyutPZVwNcOfs5OOtK4LGSP/0Hgiap6tZuejU46khZsIcG/jv87zAc76Uij1Sv4XXfcjcAD82Z/DtiYZAdweTctaQT6dtJ5CzjloHk/xU460ih55Z7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UoF5X7s2yyT8GNqKhVdV0uceXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBfUtv/XmSZ5I8neTuJCckWZdkS5KdSe7tqvBKGoE+LbROB/4M2FBVvwmsYFJf//PAF6vq/cDrwPXTHKik4fQ91F8JvDvJSuA9wMvApcD93fN20pFGpE/vvD3AXwM/YRL4fwMeB96oqv3dy3YDp09rkJKG1edQ/2QmffLWAb8GnAhc0XcBdtKRZk+fQ/3LgR9X1b6q+m8mtfUvAU7qDv0B1gJ7DvVmO+lIs6dP8H8CXJTkPUnCpJb+NuBR4CPda+ykI41In8/4W5icxHsC+GH3ntuATwOfSLKTSbONO6Y4TkkD6ttJ5zPAZw6a/QJw4eAjkjR1XrknNcjgSw0y+FKDDL7UoCxlscok+4C3gNeWbKHTdyquz6w6ltYF+q3Pr1fVES+YWdLgAyTZWlUblnShU+T6zK5jaV1g2PXxUF9qkMGXGrQcwb9tGZY5Ta7P7DqW1gUGXJ8l/4wvafl5qC81aEmDn+SKJM91dfpuXsplL1aSM5I8mmRbV3/wxm7+qiSPJNnR3Z+83GNdiCQrkjyZZFM3PdpaiklOSnJ/kmeTbE9y8Zi3zzRrXS5Z8JOsAP4G+CBwLnBdknOXavkD2A98sqrOBS4CPt6N/2Zgc1WdBWzupsfkRmD7vOkx11K8FXi4qs4BzmOyXqPcPlOvdVlVS3IDLga+NW/6FuCWpVr+FNbnG8BG4DlgTTdvDfDcco9tAeuwlkkYLgU2AWFygcjKQ22zWb4B7wN+THfeat78UW4fJqXsXgJWMfkv2k3AHwy1fZbyUP/Aihww2jp9SeaA84EtwOqqerl76hVg9TIN62h8CfgU8Itu+hTGW0txHbAP+Gr30eX2JCcy0u1TU6516cm9BUryXuDrwE1V9eb852ryZ3gUX5Mk+TCwt6oeX+6xDGQlcAHw5ao6n8ml4W87rB/Z9llUrcsjWcrg7wHOmDd92Dp9syrJcUxCf1dVPdDNfjXJmu75NcDe5RrfAl0CXJVkF3APk8P9W+lZS3EG7QZ216RiFEyqRl3AeLfPompdHslSBv8x4KzurOTxTE5UPLSEy1+Urt7gHcD2qvrCvKceYlJzEEZUe7CqbqmqtVU1x2RbfLuqPspIaylW1SvAS0nO7mYdqA05yu3DtGtdLvEJiyuBHwHPA3+53CdQFjj2DzA5TPwB8FR3u5LJ5+LNwA7gn4BVyz3Wo1i33wU2dY9/A/g+sBP4GvCu5R7fAtZjPbC120b/AJw85u0DfBZ4Fnga+DvgXUNtH6/ckxrkyT2pQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUG/Q++8g/3YgFKrwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "prev_state = env.reset()\n",
    "plt.imshow(prev_state)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 align='center'> Training Q Network </h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Hyper-parameters </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 64\n",
    "FREEZE_INTERVAL = 20000 # steps\n",
    "MEMORY_SIZE = 60000 \n",
    "OUTPUT_SIZE = 4\n",
    "TOTAL_EPISODES = 10000\n",
    "MAX_STEPS = 50\n",
    "INITIAL_EPSILON = 1.0\n",
    "FINAL_EPSILON = 0.01\n",
    "GAMMA = 0.99\n",
    "INPUT_IMAGE_DIM = 84\n",
    "PERFORMANCE_SAVE_INTERVAL = 500 # episodes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Save Dictionay Function </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def save_obj(obj, name ):\n",
    "    with open('data/'+ name + '.pkl', 'wb') as f:\n",
    "        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Experience Replay </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Memory():\n",
    "    \n",
    "    def __init__(self,memsize):\n",
    "        self.memsize = memsize\n",
    "        self.memory = deque(maxlen=self.memsize)\n",
    "    \n",
    "    def add_sample(self,sample):\n",
    "        self.memory.append(sample)\n",
    "    \n",
    "    def get_batch(self,size):\n",
    "        return random.sample(self.memory,k=size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Preprocess Images </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_image(image):\n",
    "    image = rgb2gray(image) # this automatically scales the color for block between 0 - 1\n",
    "    return np.copy(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f7547615be0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADVNJREFUeJzt3X+oX/V9x/Hna4nW1m71VxYyo4ulosjA6C6ZYhmbmi1xRfdHEUVGGYL/dJtdC61usFLYHy2Mtv4xCkHbueH8UVtXCY2dSy1jMKLxx1o1WqONNUGTWHXtHGxL+94f3yO7zRLvubnne+89fp4PuHy/53x/nM/h8Pqe8z3fc9/vVBWS2vILSz0ASYvP4EsNMvhSgwy+1CCDLzXI4EsNMvhSgxYU/CSbkjybZHeSm4YalKTpyrFewJNkBfB9YCOwF3gEuLaqnh5ueJKmYeUCXrsB2F1VLwAkuQu4Cjhq8E877bRat27dAhYp6e3s2bOHV199NXM9byHBPx14adb0XuA33u4F69atY+fOnQtYpKS3MzMz0+t5Uz+5l+SGJDuT7Dx48OC0Fyeph4UEfx9wxqzptd28n1NVW6pqpqpmVq1atYDFSRrKQoL/CHB2krOSHA9cA9w/zLAkTdMxf8evqkNJ/gj4FrAC+HJVPTXYyCRNzUJO7lFV3wS+OdBYJC0Sr9yTGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYt6N9yl4NkzrqC0rK1VG3q3eNLDTL4UoPmDH6SLyc5kOTJWfNOSfJgkue625OnO0xJQ+qzx/8bYNNh824CtlfV2cD2blrSSMwZ/Kr6Z+C1w2ZfBdze3b8d+P2BxyVpio71O/7qqnq5u/8KsHqg8UhaBAs+uVeT3yOO+puEnXSk5edYg78/yRqA7vbA0Z5oJx1p+TnW4N8PfKS7/xHgG8MMR9Ji6PNz3p3AvwLnJNmb5Hrgs8DGJM8Bl3fTkkZizkt2q+raozx02cBjkbRIvHJPapDBlxpk8KUGGXypQQZfapDBlxo0+go8LdmwYcNg7/Xwww8P9l4aH/f4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtSgPqW3zkjyUJKnkzyV5MZuvt10pJHqs8c/BHyiqs4DLgI+muQ87KYjjVafTjovV9Vj3f2fALuA07GbjjRa8/qOn2QdcAGwg57ddGyoIS0/vYOf5L3A14CPVdWPZz/2dt10bKghLT+9gp/kOCahv6Oqvt7N7t1NR9Ly0uesfoDbgF1V9flZD9lNRxqpPhV4LgH+APhekie6eX/GpHvOPV1nnReBq6czRElD69NJ51+AHOVhu+lII+SVe1KDRl9sc9u2bYO8z+bNmwd5n2myQKaG4h5fapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUGGXypQQZfapDBlxpk8KUG9am5d0KSh5P8W9dJ5zPd/LOS7EiyO8ndSY6f/nAlDaHPHv+/gEur6nxgPbApyUXA54AvVNUHgNeB66c3TElD6tNJp6rqP7rJ47q/Ai4F7u3m20lHGpG+dfVXdBV2DwAPAs8Db1TVoe4pe5m01TrSa+2kIy0zvWruVdVPgfVJTgLuA87tu4Cq2gJsAZiZmTlit52FGEOtvKFs2LBhsPeyfl/b5nVWv6reAB4CLgZOSvLWB8daYN/AY5M0JX3O6q/q9vQkeTewkUnH3IeAD3dPs5OONCJ9DvXXALcnWcHkg+Keqtqa5GngriR/CTzOpM2WpBHo00nnu0xaYx8+/wVguC+dkhaNV+5JDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoMMvtQggy81yOBLDTL4UoN6ld7S8mC5LA3FPb7UIIMvNah38LsS248n2dpN20lHGqn57PFvZFJk8y120pFGqm9DjbXA7wG3dtPBTjrSaPXd438R+CTws276VOykI41Wn7r6HwIOVNWjx7KAqtpSVTNVNbNq1apjeQtJA+vzO/4lwJVJrgBOAH4JuIWuk06317eTjjQifbrl3lxVa6tqHXAN8O2qug476UijtZDf8T8FfDzJbibf+e2kI43EvC7ZrarvAN/p7ttJRxopr9yTGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkDX3NIht27YN9l6bN28e7L10ZO7xpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qUK/f8ZPsAX4C/BQ4VFUzSU4B7gbWAXuAq6vq9ekMU9KQ5rPH/+2qWl9VM930TcD2qjob2N5NSxqBhRzqX8WkkQbYUEMalb7BL+Afkzya5IZu3uqqerm7/wqwevDRSZqKvtfqf7Cq9iX5ZeDBJM/MfrCqKkkd6YXdB8UNAGeeeeaCBitpGL32+FW1r7s9ANzHpLru/iRrALrbA0d5rZ10pGWmTwutE5P84lv3gd8BngTuZ9JIA2yoIY1Kn0P91cB9kwa5rAT+vqoeSPIIcE+S64EXgaunN0xJQ5oz+F3jjPOPMP9HwGXTGJSk6fLKPalBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlBvYKf5KQk9yZ5JsmuJBcnOSXJg0me625PnvZgJQ2j7x7/FuCBqjqXSRmuXdhJRxqtPlV23wf8JnAbQFX9d1W9gZ10pNHqs8c/CzgIfCXJ40lu7cps20lHGqk+wV8JXAh8qaouAN7ksMP6qiombbb+nyQ3JNmZZOfBgwcXOl5JA+hTV38vsLeqdnTT9zIJ/v4ka6rq5bk66QBbAGZmZo744aDxu+6665Z6CJqHOff4VfUK8FKSc7pZlwFPYycdabT6Ns38Y+COJMcDLwB/yORDw0460gj1Cn5VPQHMHOEhO+lII+SVe1KDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKDDL7UIIMvNcjgSw0y+FKD5qzA09Xau3vWrPcDfwH8bTd/HbAHuLqqXh9+iG9v27Ztg7zP5s2bB3mfVr322mtLPQTNQ59im89W1fqqWg/8OvCfwH3YSUcarfke6l8GPF9VL2InHWm05hv8a4A7u/t20pFGqnfwu9LaVwJfPfwxO+lI4zKfPf5m4LGq2t9N7+866DBXJ52qmqmqmVWrVi1stJIGMZ/gX8v/HeaDnXSk0eoV/K477kbg67NmfxbYmOQ54PJuWtII9O2k8yZw6mHzfoSddKRR8so9qUEGX2qQwZcaZPClBhl8qUEGX2qQwZcaZPClBhl8qUG9rtxbzjZt2jTI+0z+wVBqg3t8qUEGX2qQwZcaZPClBhl8qUEGX2qQwZca1Lf01p8meSrJk0nuTHJCkrOS7EiyO8ndXRVeSSMwZ/CTnA78CTBTVb8GrGBSX/9zwBeq6gPA68D10xyopOH0PdRfCbw7yUrgPcDLwKXAvd3jdtKRRqRP77x9wF8BP2QS+H8HHgXeqKpD3dP2AqdPa5CShtXnUP9kJn3yzgJ+BTgR6H2BvJ10pOWnz6H+5cAPqupgVf0Pk9r6lwAndYf+AGuBfUd6sZ10pOWnT/B/CFyU5D1JwqSW/tPAQ8CHu+fYSUcakT7f8XcwOYn3GPC97jVbgE8BH0+ym0mzjdumOE5JA+rbSefTwKcPm/0CsGHwEUmaOq/ckxpk8KUGGXypQQZfalAWs8hkkoPAm8Cri7bQ6TsN12e5eietC/Rbn1+tqjkvmFnU4AMk2VlVM4u60ClyfZavd9K6wLDr46G+1CCDLzVoKYK/ZQmWOU2uz/L1TloXGHB9Fv07vqSl56G+1KBFDX6STUme7er03bSYy16oJGckeSjJ0139wRu7+ackeTDJc93tyUs91vlIsiLJ40m2dtOjraWY5KQk9yZ5JsmuJBePeftMs9blogU/yQrgr4HNwHnAtUnOW6zlD+AQ8ImqOg+4CPhoN/6bgO1VdTawvZsekxuBXbOmx1xL8Rbggao6FzifyXqNcvtMvdZlVS3KH3Ax8K1Z0zcDNy/W8qewPt8ANgLPAmu6eWuAZ5d6bPNYh7VMwnApsBUIkwtEVh5pmy3nP+B9wA/ozlvNmj/K7cOklN1LwClM/ot2K/C7Q22fxTzUf2tF3jLaOn1J1gEXADuA1VX1cvfQK8DqJRrWsfgi8EngZ930qYy3luJZwEHgK91Xl1uTnMhIt09NudalJ/fmKcl7ga8BH6uqH89+rCYfw6P4mSTJh4ADVfXoUo9lICuBC4EvVdUFTC4N/7nD+pFtnwXVupzLYgZ/H3DGrOmj1ulbrpIcxyT0d1TV17vZ+5Os6R5fAxxYqvHN0yXAlUn2AHcxOdy/hZ61FJehvcDemlSMgknVqAsZ7/ZZUK3LuSxm8B8Bzu7OSh7P5ETF/Yu4/AXp6g3eBuyqqs/Peuh+JjUHYUS1B6vq5qpaW1XrmGyLb1fVdYy0lmJVvQK8lOScbtZbtSFHuX2Ydq3LRT5hcQXwfeB54M+X+gTKPMf+QSaHid8Fnuj+rmDyvXg78BzwT8ApSz3WY1i33wK2dvffDzwM7Aa+Crxrqcc3j/VYD+zsttE/ACePefsAnwGeAZ4E/g5411Dbxyv3pAZ5ck9qkMGXGmTwpQYZfKlBBl9qkMGXGmTwpQYZfKlB/wsPrvc22jHaHAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "processed_prev_state = preprocess_image(prev_state)\n",
    "plt.imshow(processed_prev_state,cmap='gray')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Build Model </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Network(\n",
      "  (conv_layer1): Conv2d(1, 32, kernel_size=(8, 8), stride=(4, 4))\n",
      "  (conv_layer2): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2))\n",
      "  (conv_layer3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1))\n",
      "  (fc1): Linear(in_features=3136, out_features=512, bias=True)\n",
      "  (fc2): Linear(in_features=512, out_features=4, bias=True)\n",
      "  (relu): ReLU()\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "import torch\n",
    "\n",
    "class Network(nn.Module):\n",
    "    \n",
    "    def __init__(self,image_input_size,out_size):\n",
    "        super(Network,self).__init__()\n",
    "        self.image_input_size = image_input_size\n",
    "        self.out_size = out_size\n",
    "\n",
    "        self.conv_layer1 = nn.Conv2d(in_channels=1,out_channels=32,kernel_size=8,stride=4) # GRAY - 1\n",
    "        self.conv_layer2 = nn.Conv2d(in_channels=32,out_channels=64,kernel_size=4,stride=2)\n",
    "        self.conv_layer3 = nn.Conv2d(in_channels=64,out_channels=64,kernel_size=3,stride=1)\n",
    "        self.fc1 = nn.Linear(in_features=7*7*64,out_features=512)\n",
    "        self.fc2 = nn.Linear(in_features=512,out_features=OUTPUT_SIZE)\n",
    "        self.relu = nn.ReLU()\n",
    "\n",
    "    def forward(self,x,bsize):\n",
    "        x = x.view(bsize,1,self.image_input_size,self.image_input_size) # (N,Cin,H,W) batch size, input channel, height , width\n",
    "        conv_out = self.conv_layer1(x)\n",
    "        conv_out = self.relu(conv_out)\n",
    "        conv_out = self.conv_layer2(conv_out)\n",
    "        conv_out = self.relu(conv_out)\n",
    "        conv_out = self.conv_layer3(conv_out)\n",
    "        conv_out = self.relu(conv_out)\n",
    "        out = self.fc1(conv_out.view(bsize,7*7*64))\n",
    "        out = self.relu(out)\n",
    "        out = self.fc2(out)\n",
    "        return out\n",
    "\n",
    "main_model = Network(image_input_size=INPUT_IMAGE_DIM,out_size=OUTPUT_SIZE)\n",
    "print(main_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Deep Q Learning with Target Freeze </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populated 200 Samples\n"
     ]
    }
   ],
   "source": [
    "mem = Memory(memsize=MEMORY_SIZE)\n",
    "main_model = Network(image_input_size=INPUT_IMAGE_DIM,out_size=OUTPUT_SIZE).float().cuda()\n",
    "target_model = Network(image_input_size=INPUT_IMAGE_DIM,out_size=OUTPUT_SIZE).float().cuda()\n",
    "\n",
    "target_model.load_state_dict(main_model.state_dict())\n",
    "criterion = nn.SmoothL1Loss()\n",
    "optimizer = torch.optim.Adam(main_model.parameters())\n",
    "\n",
    "# filling memory with transitions\n",
    "for i in range(0,int(MEMORY_SIZE/MAX_STEPS)):\n",
    "    \n",
    "    prev_state = env.reset()\n",
    "    processed_prev_state = preprocess_image(prev_state)\n",
    "    step_count = 0\n",
    "    game_over = False\n",
    "    \n",
    "    while (game_over == False) and (step_count < MAX_STEPS):\n",
    "        \n",
    "        step_count +=1\n",
    "        action = np.random.randint(0,4)\n",
    "        next_state,reward, game_over = env.step(action)\n",
    "        processed_next_state = preprocess_image(next_state)\n",
    "        mem.add_sample((processed_prev_state,action,reward,processed_next_state,game_over))\n",
    "    \n",
    "        prev_state = next_state\n",
    "        processed_prev_state = processed_next_state\n",
    "\n",
    "print('Populated %d Samples'%(len(mem.memory)))\n",
    "\n",
    "# Algorithm Starts\n",
    "total_steps = 0\n",
    "epsilon = INITIAL_EPSILON\n",
    "loss_stat = []\n",
    "total_reward_stat = []\n",
    "\n",
    "for episode in range(0,TOTAL_EPISODES):\n",
    "    \n",
    "    prev_state = env.reset()\n",
    "    processed_prev_state = preprocess_image(prev_state)\n",
    "    game_over = False\n",
    "    step_count = 0\n",
    "    total_reward = 0\n",
    "    \n",
    "    while (game_over == False) and (step_count < MAX_STEPS):\n",
    "        \n",
    "        step_count +=1\n",
    "        total_steps +=1\n",
    "        \n",
    "        if np.random.rand() <= epsilon:\n",
    "            action = np.random.randint(0,4)\n",
    "        else:\n",
    "            with torch.no_grad():\n",
    "                torch_x = torch.from_numpy(processed_prev_state).float().cuda()\n",
    "\n",
    "                model_out = main_model.forward(torch_x,bsize=1)\n",
    "                action = int(torch.argmax(model_out.view(OUTPUT_SIZE),dim=0))\n",
    "                \n",
    "        next_state, reward, game_over = env.step(action)\n",
    "        processed_next_state = preprocess_image(next_state)\n",
    "        total_reward += reward\n",
    "        \n",
    "        mem.add_sample((processed_prev_state,action,reward,processed_next_state,game_over))\n",
    "        \n",
    "        prev_state = next_state\n",
    "        processed_prev_state = processed_next_state\n",
    "        \n",
    "        if (total_steps % FREEZE_INTERVAL) == 0:\n",
    "            target_model.load_state_dict(main_model.state_dict())\n",
    "        \n",
    "        batch = mem.get_batch(size=BATCH_SIZE)\n",
    "        current_states = []\n",
    "        next_states = []\n",
    "        acts = []\n",
    "        rewards = []\n",
    "        game_status = []\n",
    "        \n",
    "        for element in batch:\n",
    "            current_states.append(element[0])\n",
    "            acts.append(element[1])\n",
    "            rewards.append(element[2])\n",
    "            next_states.append(element[3])\n",
    "            game_status.append(element[4])\n",
    "            \n",
    "        current_states = np.array(current_states)\n",
    "        next_states =  np.array(next_states)\n",
    "        rewards = np.array(rewards)\n",
    "        game_status = [not b for b in game_status]\n",
    "        game_status_bool = np.array(game_status,dtype='float') # FALSE 1, TRUE 0\n",
    "        torch_acts = torch.tensor(acts)\n",
    "        \n",
    "        Q_next = target_model.forward(torch.from_numpy(next_states).float().cuda(),bsize=BATCH_SIZE)\n",
    "        Q_s = main_model.forward(torch.from_numpy(current_states).float().cuda(),bsize=BATCH_SIZE)\n",
    "        Q_max_next, _ = Q_next.detach().max(dim=1)\n",
    "        Q_max_next = Q_max_next.double()\n",
    "        Q_max_next = torch.from_numpy(game_status_bool).cuda()*Q_max_next\n",
    "        \n",
    "        target_values = (rewards + (GAMMA * Q_max_next)).cuda()\n",
    "        Q_s_a = Q_s.gather(dim=1,index=torch_acts.cuda().unsqueeze(dim=1)).squeeze(dim=1)\n",
    "    \n",
    "        loss = criterion(Q_s_a,target_values.float())\n",
    "        \n",
    "        # save performance measure\n",
    "        loss_stat.append(loss.item())\n",
    "        \n",
    "        # make previous grad zero\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # back - propogate \n",
    "        loss.backward()\n",
    "        \n",
    "        # update params\n",
    "        optimizer.step()\n",
    "    \n",
    "    # save performance measure\n",
    "    total_reward_stat.append(total_reward)\n",
    "    \n",
    "    if epsilon > FINAL_EPSILON:\n",
    "        epsilon -= (INITIAL_EPSILON - FINAL_EPSILON)/TOTAL_EPISODES\n",
    "    \n",
    "    if (episode + 1)% PERFORMANCE_SAVE_INTERVAL == 0:\n",
    "        perf = {}\n",
    "        perf['loss'] = loss_stat\n",
    "        perf['total_reward'] = total_reward_stat\n",
    "        save_obj(name='MDP_ENV_SIZE_NINE',obj=perf)\n",
    "    \n",
    "    #print('Completed episode : ',episode+1,' Epsilon : ',epsilon,' Reward : ',total_reward,'Loss : ',loss.item(),'Steps : ',step_count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Save Primary Network Weights </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "torch.save(main_model.state_dict(),'data/MDP_ENV_SIZE_NINE_WEIGHTS.torch')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 align='center'> Testing Policy </h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Load Primary Network Weights </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = torch.load('data/MDP_ENV_SIZE_NINE_WEIGHTS.torch', map_location='cpu')\n",
    "main_model.load_state_dict(weights)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Test Policy </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Algorithm Starts\n",
    "epsilon = INITIAL_EPSILON\n",
    "FINAL_EPSILON = 0.01\n",
    "total_reward_stat = []\n",
    "\n",
    "for episode in range(0,TOTAL_EPISODES):\n",
    "    \n",
    "    prev_state = env.reset()\n",
    "    processed_prev_state = preprocess_image(prev_state)\n",
    "    game_over = False\n",
    "    step_count = 0\n",
    "    total_reward = 0\n",
    "    \n",
    "    while (game_over == False) and (step_count < MAX_STEPS):\n",
    "        \n",
    "        step_count +=1\n",
    "        \n",
    "        if np.random.rand() <= epsilon:\n",
    "            action = np.random.randint(0,4)\n",
    "        else:\n",
    "            with torch.no_grad():\n",
    "                torch_x = torch.from_numpy(processed_prev_state).float().cuda()\n",
    "\n",
    "                model_out = main_model.forward(torch_x,bsize=1)\n",
    "                action = int(torch.argmax(model_out.view(OUTPUT_SIZE),dim=0))\n",
    "                \n",
    "        next_state, reward, game_over = env.step(action)\n",
    "        processed_next_state = preprocess_image(next_state)\n",
    "        total_reward += reward\n",
    "        \n",
    "        prev_state = next_state\n",
    "        processed_prev_state = processed_next_state\n",
    "    \n",
    "    # save performance measure\n",
    "    total_reward_stat.append(total_reward)\n",
    "    \n",
    "    if epsilon > FINAL_EPSILON:\n",
    "        epsilon -= (INITIAL_EPSILON - FINAL_EPSILON)/TOTAL_EPISODES\n",
    "    \n",
    "    if (episode + 1)% PERFORMANCE_SAVE_INTERVAL == 0:\n",
    "        perf = {}\n",
    "        perf['total_reward'] = total_reward_stat\n",
    "        save_obj(name='MDP_ENV_SIZE_NINE',obj=perf)\n",
    "    \n",
    "    print('Completed episode : ',episode+1,' Epsilon : ',epsilon,' Reward : ',total_reward,'Steps : ',step_count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 align='center'> Create Policy GIF </h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Collect Frames Of an Episode Using Trained Network</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Reward : 14\n"
     ]
    }
   ],
   "source": [
    "frames = []\n",
    "random.seed(110)\n",
    "np.random.seed(110)\n",
    "\n",
    "for episode in range(0,1):\n",
    "    \n",
    "    prev_state = env.reset()\n",
    "    processed_prev_state = preprocess_image(prev_state)\n",
    "    frames.append(prev_state)\n",
    "    game_over = False\n",
    "    step_count = 0\n",
    "    total_reward = 0\n",
    "    \n",
    "    while (game_over == False) and (step_count < MAX_STEPS):\n",
    "        \n",
    "        step_count +=1\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            torch_x = torch.from_numpy(processed_prev_state).float()\n",
    "            model_out = main_model.forward(torch_x,bsize=1)\n",
    "            action = int(torch.argmax(model_out.view(OUTPUT_SIZE),dim=0))\n",
    "        \n",
    "        next_state, reward, game_over = env.step(action)\n",
    "        frames.append(next_state)\n",
    "        processed_next_state = preprocess_image(next_state)\n",
    "        total_reward += reward\n",
    "        \n",
    "        prev_state = next_state\n",
    "        processed_prev_state = processed_next_state\n",
    "\n",
    "print('Total Reward : %d'%(total_reward)) # This should output same value which verifies seed is working correctly\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image, ImageDraw\n",
    "\n",
    "for idx, img in enumerate(frames):\n",
    "    image = Image.fromarray(img)\n",
    "    drawer = ImageDraw.Draw(image)\n",
    "    drawer.rectangle([(7,7),(76,76)], outline=(255, 255, 0))\n",
    "    #plt.imshow(np.array(image))\n",
    "    frames[idx] = np.array(image)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Frames to GIF </h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mayank/miniconda3/envs/rdqn/lib/python3.5/site-packages/ipykernel_launcher.py:5: DeprecationWarning: `imresize` is deprecated!\n",
      "`imresize` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.\n",
      "Use ``skimage.transform.resize`` instead.\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "import imageio\n",
    "from scipy.misc import imresize\n",
    "resized_frames = []\n",
    "for frame in frames:\n",
    "    resized_frames.append(imresize(frame,(256,256)))\n",
    "imageio.mimsave('data/GIFs/MDP_SIZE_9.gif',resized_frames,fps=4)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}