Actor Critic (Cartpole).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from util import to_onehot, moving_average, Agent\n",
    "import gym\n",
    "\n",
    "plt.style.use('fivethirtyeight')\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "\n",
    "SAMPLE_BATCH_SIZE = 32\n",
    "OBS_SPACE = 4\n",
    "ACTION_SPACE = env.action_space.n\n",
    "N_EPISODES = 2000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# scenario 1: single agent training\n",
    "for each s1, the agent learns q-values for s2. it doesnt learn policy - the agent's decision to select the action that leads to s2 with the greatest q-value is hardcoded as argmax.\n",
    "\n",
    "the original atari approach used a single DQN agent."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from agent import Agent\n",
    "agent = Agent(OBS_SPACE, ACTION_SPACE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0  / 19\n",
      "100  / 17\n",
      "200  / 60\n",
      "300  / 96\n",
      "400  / 67\n",
      "500  / 151\n",
      "600  / 144\n",
      "700  / 110\n",
      "800  / 187\n",
      "900  / 170\n",
      "1000  / 200\n",
      "1100  / 200\n",
      "1200  / 200\n",
      "1300  / 199\n",
      "1400  / 10\n",
      "1500  / 200\n",
      "1600  / 148\n",
      "1700  / 200\n",
      "1800  / 85\n",
      "1900  / 200\n"
     ]
    }
   ],
   "source": [
    "num_steps = []\n",
    "\n",
    "for episode in range(N_EPISODES):\n",
    "    s1 = env.reset()\n",
    "    done = False\n",
    "    steps = 0\n",
    "    \n",
    "    while not done:\n",
    "        \n",
    "        # propose an action\n",
    "        a = agent.act(s1.reshape(-1,4))\n",
    "        \n",
    "        # what are the consequences of taking that action?\n",
    "        s2, r, done, i = env.step(a)\n",
    "        \n",
    "        # store memories for experience replay (prevents catastrophic forgetting)\n",
    "        agent.remember(s1.reshape(-1,4), a, r, s2.reshape(-1,4), done)\n",
    "        \n",
    "        # move to next state\n",
    "        s1 = s2\n",
    "        steps += 1\n",
    "        \n",
    "    agent.replay(SAMPLE_BATCH_SIZE)\n",
    "    num_steps.append(steps)\n",
    "    \n",
    "    if episode % 100 == 0:\n",
    "        print episode, \" /\", steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x122d49890>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbIAAAEXCAYAAADIosFBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXeYJFW1wH9nctgJm3Nkh4VdwgLCI0fJKkFQUBHBhIqK\n+gRRRFB5BszPh4kgKiI5SFAQERCJS9xd2J1NbI6zMzs5nvdH9+x036rqrp7pnumeOb/vm2+6b1VX\nna7uvqfuiaKqGIZhGEaukjfUAhiGYRjGQDBFZhiGYeQ0psgMwzCMnMYUmWEYhpHTmCIzDMMwchpT\nZIZhGEZOY4rMMAzDyGlMkRlGliAivxcRjf51i8h6EfmDiEyN2edfInLjUMppGNmGKTLDyC6eASYD\nM4APAQcAdw2pRIaR5ZgiM4zsokNVN6vqBlV9GvgtcJiIVA61YIaRrZgiM4wsRUSmAOcA3dE/wzB8\nKBhqAQzDiONYEWkicpNZGh37sao2D6FMhpHVmCIzjOziBeBCoAT4APBu4KohlcgwshxTZIaRXbSq\n6oro48Uisgfwv8Anh1Amw8hqzEdmGNnNNcBFIvKuoRbEMLIVW5EZRhajqrUi8lfgOuDk6PAYEVno\n7LpLVVcNrnSGkR2YIjOM7Od64FkROTb6/KzoXyx/B04ZTKEMI1sQ6xBtGIZh5DLmIzMMwzByGlNk\nhmEYRk7TL0UmIseJyDHpFsYwDMMwUiWUIhORp0TkiOjjK4C/AH8Wka9nUjjDMAzDSEaoYA8R2QFM\nUNVuEVkBvA9oBJ5V1RkZltGXhoYGi1IxDMMY5lRVVUmyfcKG3+cBGq0yIKq6FEBERg9APsMwDMMY\nMGEV2b+BXxLpk3QfQFSpbc+QXIZhGIYRirDBHh8D6oE3gG9Fx/YCfp4BmQaV2traoRYhJUzezJFL\nsoLJm2lM3syRbllDrchUdQfwdWfs4bRKYhiGYRj9IGzU4pd7a7uJyKEislZEVovIYZkVzzAMwzAS\nE9a0+CVgdfTx94CfAN8FfpYJoQzDMAwjLGEVWZWqNohIBbA/8L+qehMwL8yLRWS6iDwpIktFZImI\nfDE6PkZEHheR2uj/0TGvuVJEVojIMhE5OfjohmEYxkgmrCJbJyKHA+cBT0fzySqB7pCv7wK+oqrz\ngUOBz4nIfOBrwBOqWgM8EX1OdNt5wAIiFb1vEJH8sG/KMHKZ+vYent7UzoqGTvpb1Luho4eWrp40\nS2YY2UnY8PuvAncDHcD7o2PvAV4M82JV3QRsij5uFJG3gKnAGcCx0d1uBf4FXBEd/4uqtgOro0nY\nhwDPhZTXMHKO7h7lY/+q46/vtO0eu2zfUVzzrqqUjnHJMzu5e1UrRfnwnhml3HTsmEyIaxi76exR\nCgREkuYuZ4RQKzJVfURVp6jqLFVdFB2+i0iFj5QQkVnAAcALwMSokgPYDEyMPp4KrIt52fromGEM\nWy59tj5OiQH8amkTbV3hV2XPb+3grlWtKNDeDfesbuXGt5rSLKlhROjqUT77zE6m/HEje9+xmSc3\ntCV/UQYI3Y9MRPYCziWifC6NPi9S1TdCn0xkFPAUcJ2q3isi9apaHbN9p6qOFpFfAs+r6p+i4zcB\nj6rq3b37xpaoyqX8CcPwY3WL8IFXSn23PXpIC+OKwh3nr1vy+XZtcdzYkaO7+emC9oGKOKi83SQ8\nW5fPAVU9HFhlJtJ0ct/mfH63tpCOHmFKSQ9fn9tBRQGML1KKUiwj/2xdHpctLYkb+9U+bbyremCf\nWU1Nze7HaStRJSLnAjcA9wAfAi4FRgHfB94d8hiF0dffpqr3Roe3iMhkVd0kIpOBrdHxDcD0mJdP\ni475EvumU6W2tnZArx9sTN7MMZSyHvvHjYD/TeWEabPZo8r7U42Vd01jF/evbuWnaxo9x+kuKqWm\nZkhKosYR9vrWNnRy0bNb6VIQ4G+njeO/JhYnfV26yaXvLiSXd0tLN2c/tp0lO7t2jzU05XPBa303\nUNccVMll+1Xsfr6xuZuyAqG62F/DPfB6I7Arbuwzi0u48ZjRnDOnrN+ypkpYH9m3gXer6usi8sHo\n2OtEIhiTIhHD6U3AW6r6k5hNDwIXElGIFwIPxIz/WUR+AkwBagjpjzOMXGN5fSfNCcyHqxu7fBVZ\nLxubu1l495bA7bUNXXzlufrdz2dV5HPxvHLKC7OzHeF1rzTSezkU+NqLDTz53glpO/7Khi6e3dLO\n4ROLmFtVmLbjZjvXLtoVp8T8uGbRLl7b0cm6pi4Wbe/cPf6/R1RzwZ7lnv0fXNPqe5x7V7cmVGTp\nJqwim0CkPBX03e4pQbeQXo4ALgDeFJHXomNfJ6LA7hSRjwPvAB8AUNUlInInsJRIxOPnVDVshKRh\n5BT3rPafDHo55/EdrP7QZEb73BWvbOjioHuDlRjA9rYebnq7OW5s0bZOfn9cdgaBPL0p3gz6asyE\nmgovbm3ne682UlWUx3cOrmT6qAJWNHRy5ANbaeuGsgLhqfeNp2aEKLOXtnWE2u9+H+X0+Wfr+UhN\nWVwwx5K6Tt6o8/9s6tsH1xwcVpEtIqKI/hAzdh7hoxb/TcRK4McJAa+5DrgupHyG0W+6e5TlDV28\n0ZDH9C6lpGBwI69eDjHBzP7zJnZ+bIonKuxbLzf065x/Xzc0Tvkw1KVhEmzvVj76zzo2t0aOdf+a\nVk6YWswTG/qUZEuX8r+Lm/jFESOjicfG5oGtBb76fANbW7t5alM77d1KW4LDJbIwZIKwtoUvAN8V\nkaeAchH5O/AdIhU/DCNnqWvr5oB7tnDY/Vv59JslTPrjRja1DO7if4MzwVy6YJTvfm/Ve81CD63t\nn0Jq7Va6erKrpZ+qcv1ru3y3rWlMbBJzWbStY7cS6yVWifVyz6rg1XBbN6xv6qI7w9epcxA+hx5V\nj3LZs6qAuZUF3BpyZX7j2808+E4bDR2Jldj5c8s4eHzI6KQ0EbZo8NvRKMX3AA8RCY1/SFUtrtfI\naW5Z1sLapvhf5a+XNHHtweFztwbKro74CffT88t578wSTn4kvkvS4fdv5Z0PT6YqGlr2p/Xen++P\nDq3iE3uPoq1L+ceGNjbHKOWrXmqIm4CaOpXq4qHJ+/Hjl4ubuO7VRt9th9y7hQdOGcdhIYM+VoVU\nfONK/O/ll+7s5MyXS9nx3Bb+a0IRd580loo0+hS7epRtbT1c/nwk5aK6SLhiYSUf36ucovz0fyat\njhIryYcXz564+/lDp47je6/u4tnN4cyPiSgvEH546OD9fiB81OJUoEVV74wZGy0iU1R1Y8akM4wM\ns2i794f7x9qWQVZk8ZNMZVEe00cVcMzkYp5y/EUzb9vE+XPLqG/v4dF18Xe9o4uFT+wdWc2VFAjv\nmRkfzn/96420xaxS5t+5mVffP5GJZUNfNOeHr+3ifwKUGEBHD9y+oiW0IlvREE6RHRCQ1/Cj1xvZ\n0RlRKC9s7eCula1cvJc32KE/bG/r5vRHtrMsRsb6DuXKFxv40euN/OM945ldGdbrEw53NVZeEK+U\nj5xUzMOnjt/9fEVDJ2f+fQfrQ5ojz51TyvjSPG5YEvHF5g1yYnTYW4z7iYTAxzKNaJNNw8hF1jd1\n8YiPaa4mQYRguunuUZqcSWZU1Ed33lz/qK/bV7TwqI+P64Cxic05brBIS5fyx9qWVMRNGx3dyt/W\ntXLJ03VU37IhoRLrZXtbeN+Za649e3Ypd584lgv3jL+mHT5mvc4e5V4nAOe3aUgqX9PYxdl/387c\n2zfHKbFYdrT3cM2i/vk9E9HU6SiywsSKZm5VIY+eNi7UsccW53Htu6r4n0OqefPciVy6j79pPJOE\n/cXuqapvxg6o6ptRc6Nh5BzdPcqxf93mu62nn/UN+0OjM8FUFAr5eZFJ5qxZpfxqSVNgZJjLnSeO\nTbj9lOklvF0fPyEvCXnsdKKqfOiJHfzDx2cVS3mBxK0k3Mk4Ee6+Z80u5d3TSsgTuHV5n/Ju9jnm\n/T5RpG/Xd1Hf3hOYTxWGi/5VFyoC84E16Q/E2dYar9iDTKqxTB9VwAfmlHKn40f8xRHVHDiuiPtW\nt1CQJ5wzp5Qp5fm7XzMUhD3rNhGZq6oregdEZC6wIzNiGUZmWVrfFXiHvyOFO/+Bsqsz/lwVMXfK\nJdHw8E8/vdMzmbhsvmAKBXmJ77KvPKCSv69riwsa2Z7Ia58hXt/RmVSJjS4W7nj3WE56uM9P2NwZ\n/nNp8ZjSJO5/L09tamfc7zdw4Lgi/u+oamqqCtnc6n9NZv15ExsvmExZQerK7K2dnSmlETR39qQ1\nz2+r850eH0KRAfz66NFcus8otrf1kCdw4LgiKqM+2n3GDK4fLBFhr9TNwD0i8h4RmS8i7yVSRPjG\nzIlmGJnjjhXBJrVVjd28tHXgTu8w+PnHYhERbjhqNH89ZRxXHVjpe4wJpXmhUgaK88UTau5O+IPB\nil3J/Vfvn13GKGciTyWk2638Xxa9Pu4xAboUXtzWwcH3bmWfOzfzzZf8IycB/ueV5CZQP+5YmZoJ\n1w1AGijbnAjO8aXh/KJ5Iuw3tojjp5Zw7JQSz/czWwgr1feBPwE/Al4Cfhh9/v0MyWUYGeO0R7bx\nyyWJfR4nPuxvdoSIWTId5sdnN7dz5ANb48YqfHwXBXnCUZOLOXu2fy3GshTy3tx9BzvfB/zNeb3s\nO6aQG46s5vv/VeVZPaViWnTfV+/7nlmReAJPFtzw6LrEK+Mg/rg8NUWW7s/FzSGbUJqdCqm/hA2/\n7wGuj/4ZRs5y67Jm/rMl3Gprc0s3k5yIvle2dfDRJ+vY0NzNRfPK+fFhVSm3rljT2MW9q1v59iLv\nnX+iO94xxXkI3nI6Yc1E4HXyr2mM5EnlJzFLBvHS1g4aOno4bkpx6GO4k3S+wHNnTmDP6vgKG6Mc\nWdc3d7OlpTtplOWOtm6WOqWYeqP0/FZkqbByVzcNHT27UyDC0NWj7HCSvCeU5rG1NdhU+symdt6V\nxlysH70Rv5IcXzL0karpJNSnISJfE5GDnbFDROTyzIhlGJnhzwEmxW/6mO2W1Xt9Gj94vZH1zd0o\ncPOyZp4NqRR7eXFrO0fcv9VXiQHMTRB2XV2cxwf38K7KPrl3+CixiaX5FMfMYW3d8EBAvbxk/N+S\nJk58eBvnPL6DsbduZNWuLt6u7+RXS5p4YkNbYFNQ15z5xX1HeZQYeEPEAebdsTmhTM9samf+nd59\nymKU4odrBlYDMGxofy8rfUypz585gd8dPZobjxnNzw6v9my/dtEuntuSvo4Fc5yVaEVR9uQPpoOw\ntxVfJFL3MJalwGXpFccwMseaxi5e8PF9zanI57J9vcrAz5Tllna6L0mdRJffvdUcaDb6rwlFcZXH\n/fi/I0fz99PG8ftjx/D9vdp56ewJfGCP8BNzaYFw/JT4thuP9LNc1TdejA8TP/CeLRx631aufLGB\n9z+2g5+/6W++9fqv/Keh4vzIKjQWwZtAHstv32qi3cc6GGtS/cXh1ZwzJ3JDcPqMEpZ+YBJvnjuR\nwyZ6V0AVhcJRk+LH/RRTIt72qcgypiSfc/co45w5ZXxsXjkf8LlB+es7ke/WsvpOLnqyjsPv28KX\n/rOzX52/6x1f7OFD0E0gk4RVZEWAe3vaAZT47GsYWcn3XvWugv57vwr+eup48vOEk8fHTzhufpdb\nHQFSj/oL8sF8fp9R/P308UxOYjbLzxP+a2IxZ84u5YRx3f0qeOv6R9xAgDCEKat0Z0CAg+sjC/Lx\niQg3HRMfnKJEbgaCcBuTQkQhxvrb8vOEG48ZQ/1FU7nthLFMKc9n+qgC7jtpHN84oIKzZpVy1qxS\n3j+pk4dOHedJmn59R2opC+7K/tN7exOrz5zlVWQNHYqq8umnd3LfmlaW1ndxy7IWrnwhtTyzHlXq\nHeU/fdQINC0SKRr8WWfsEuCV9IpjGJnjFSf8+bw9SrnqoEqmRnNgXP/3E06328l/9BaxqUsxVP8d\np3TSzFH5XH1QJVcf5B+RmAk+UhM/kT61qT3l4JUwBWiX+qxEIDg03o/jppaw35h4Zf2dV3axNSBE\n3o+qorykqQkQSXf46sJKbjluDLccN4avze1k/7FF7D82/vyPr09tBet+7/byMaOeNqOUTzqVQ5o7\nlfXN3bzmKM5bl7fwqafqAk23Lg0dSux9R2WhUNhPn2i2ElaRfQm4XEQWicidIrIIuIJIMWHDyHqW\n1HVS6/g2Ll8YrzzGFcVPDHevat09wa8KMCdtSWE1s7O9h40t8fs/9b4JfHm/ikGdWKp8/CO/WRq8\nyvFj8QASqd0k8GRRl0dP9prBXgvIyZroE4339QMGdpNw+KT48y9v6AqtSGsbOvmbY7qdV+3vBz1h\nWvx5XtneEZjreOeq1tB5aTscq4FfO6BcJ9Q7UtUlwJ5EohZfiv6fp6qu38wwshL3LvqwiUXMcQIr\n9qmI/8H3KDwfDeZ4PsDxnkprjKU74yeeuZUFA6oU0V/8Em2vXdQQ+g4f4PuvJc+nClJPro8pWQSg\nX43DoPw3v9ETpw3MAzK5LJ/5o+O/K0GK1OX7PqW39gpQZJXO57K2qTthS5tnN4cLBnGPMTaFKNdc\nIfQ7UtUmVf2Lql4f/W+V742cYYtzB+0qMYBDR3snjduiUY4NHf4TZ1OXJgw+iOUVp0DxgeOHpqGj\nX65aWzeeFWsiSkNUaFciNRVd3HJJeyQpkDunsoCzHB9Sc0DAQ4uz2rv1uDG7TccD4dAJ8aulV3eE\ni1b1a5o6JiD0fe/RhbiL0/c/Flw8yfXhBuEqMjeAZjgQtvr9MwR0g1bVo9MqkWFkANfP4NcvKV/g\ntuPH8OF/1u0eu622hb2qCkikqza3dIeqePCWk9u0MEmR30wRJOsvFjfxyyPDNZlsChk519KlnrYk\n7moqzAphjLOPX1K1+vTcOn1GeuLRFo4rhGV9z8OY9dY1pRbdmKrJrzukVdstuTYcFVnYd3QjcFPM\n38PAJOAfGZLLMNLGm3WdPOfke/n5iQBOnl7CDCei65sv7+InbwSb0sI24nzHmdiCfCWDwUf39Ibs\n/6m2JbTvK2xpK1exqHqr/SerxA5eP5rf+du64++2i/IIFeQRBjdy0fV7+XG/T35ess4KqaRSBK1K\nXTwrspFqWlTVW52/HwCnASeGeb2I3CwiW0VkcczYHSLyWvRvjYi8Fh2fJSKtMdt+3Z83ZgwfVjZ0\n8amn6/jEU3Vs6Uf35j8s8wYyBN39FuQJly/05nIlKhm0qSXchOLmpY0ewrp13wgIgLg9QQ3KWMIq\nst6cp+1t3Xx+cTHv+9v2uAi6wjxCBbqEKa0VVF8xHexVXRCXSA4EdrPuxb15AnyTn2OpTCFROexn\nsHMEmBYH8o42APuF3Pf3wCmxA6r6QVVdqKoLgXuAe2M2r+zdpqqXDEBGI8dRVT7+VB13rmzl7lWt\nzLtjc8pt73/3tleRJcrX+vDcMsam8GMPuyLzhJ2HWIlkioll+Wy+YIpnvDFEhfmWrp7QHQKaO5W7\nV7Uw9/bNPF+fzzNOB+KwysYN0X/dx0flRkMOtBxVLIV5QrGjcK97tTFhgMxrjk/06oMqOWJS4kTk\noMCXj+9VztecG6ywisxMi1FE5GLn71Ii5sXnw7xeVZ8G6vy2SaRQ3QeA20PKbIwgtrf1ePxbNyQp\n+OviN1cmCjAQEf5z5oTQx98YUpG5wRTpXDH0h5IC4YYj41cIYSbHN3d04hPD4Utzl/KT14PNsqNC\ntkQpdvxsj633Ruw1OI7MVFY3YfBLIg5ajbd1aVyqhQCfW5C8lFiQ//L4KcUscPLpwhZRHglRi2GN\n9Bc4z5uB/wA/TYMMRwFbVLU2Zmx21NTYAFylqs8kOkBtbW2izUkZ6OsHm5Ek77N1ebgFZJZs3kVt\nbXB1+liau6BL4/0OP9q7ndUrV/juHyvrPw+F45+Pf21xnvKeCV3cs7lvUlmxtSGpPH9aX0CkQE4E\nQdmxbjVtA3STDfS70LQ9H+hbJWytb6S2NnGbwee3xr/m+LFdXF3TQWsPfHNZMS839E34te+sZ11j\nMUHB+AXaGeo9rN8Sf/1G5avndW/Vx39Xirrb0zo3XDgxn8t3xq+ofvH8Wj4+w2sheNK5rhOKe1i7\nyv87F0t3ffzretG6DTR0CrHvb/uuZmpr49cHfu93w85ioO8zadm+idp+lLlKN4k+m5qampSOFbb6\n/XEpHTU1zid+NbYJmKGqO0TkIOB+EVmgqoEG6VTfdCy1tbUDev1gM9LkPeaPG3EDZp/dmR/6mJHy\nQH2tUqqKhE8cOsd3Xz9ZH6hq42dvNtHQ0cOC0YV8Zf8K1jd3c8+jfQ0fG/NKqamZmVCOxxZvAfom\nvL1HF7Jw72mh3kMQ6fgurCtrg7djFFdxedL30tXSCPT9HOdPqmbh3pEmi5M274CGvkCI6glTaHvb\n1xgT2V5WTE3N9KRyXjC+k/9d0/c5NnWL570vf6eVWMPPxMoyampmJD12EO713WOucvnb8dVdFneM\noqZmvOe1dzTuAvpWomPLikK9zzHTu7lh3VY2xyTaHzCukNMWTon0yFvS973TopK49xf0fWh1vnv7\nzplOzRBFzPaS7nksbPj9fGCHqm4RkVHAV4Ee4HpVTa3RTvxxC4CzgYN6x1S1HWiPPl4kIiuJJGO/\n3N/zGLlLkKkrbCsNN2F5/ujUcreOmVLCMU6RXbfM4OYQwR6u2evIJL6SwcJtsLg+RMj4+ub4fabF\nmNwqnM9keUMnidxubvmpINzPepyPeWyXY2pLdxPIPBFuOXY0F/1r5+6xZT5luLp6lB855tQjQhbp\nHVuSz1Pvm8A/NrTR1KmMLcnj5Okl5Il4EtndnLkg3O/eiK3sQWTF1GtM/xFwNHAo8JsBnv/dwNuq\nur53QETGi0h+9PEcoAZYNcDzGEPAfza3856XSph528bQ0XCxBFXTAP8WK364RXqnpSE5dlJZ/M9m\nc0t3wlqFPaqe3lMDLZuULmY77T3eaepOWhB4XVPwNXV9j36lvWaMyqeiUDhjVgnXvivcdXADY/xu\ncOrbXR9Z+ifs986MT8ze1tbjCZD5mU/V/4/7FAoOYmJZPh+uKefT80dxzpwyKqIKzNNoNGSwh7tf\nRRqDYLKFsBb6Waq6LBqYcTYwH2gFVod5sYjcDhwLjBOR9cC3VPUm4Dy8QR5HA98WkU4iq75LVDXY\nNmFkLd98qYEt7XmA8plndtLY0cMn9y4P1Yjy6U3tvO9v2wO3J2pKGIsbiDElSXX5MJQV5FFVJLur\nfXRpJChlQkD7+Pr2HmLnkopCGZLSVH6MKsxjUmneblNWt8Laxm72SJDvlOjmwG3y6fZrm1qWzxvn\nTkpZTr88sh5V8mK+S27pq3RU9HApyBNmV+SzurHvGqxp7GbfMZH3vaG5m+++4vWC1CSpXhKGMMrc\nRVVDdxvIZcL+mtpEpAI4BFirqtuJmP9Cpc2r6vmqOllVC1V1WlSJoaofU9VfO/veo6oLoqH3B6rq\nX1N4P0aWoKoscqofXP5CA09vSl7aR1UTKjGINFAMg2taTNfk5irEXyWIpHQLC6fS0XkwmO1Msm4p\nLZf1zoosNprPnWzd1dvofr73PBFPWSx3IndblaRj9e3H7IrgVec3X/K2WDl/blm/O3DH4iqgphCp\nEu3dxEWYFuXhqbQyHAj7rfoz8E/gViI5YQAHEnJFZow83CTMXi5/vj7pa/0aEbq4IflBbHAU2ZQ0\nTW6THEX20zebAnuTveg085yYhlVhOnHLdX3y6Z0Be0b8P7GmqjyJz0tKdrc/kBuJZNU9Bmvl4ZpP\nY1eCvc0wY7nKp/t4fygvEE93b9ec6pLJJPFsImxljy8B3wA+o6q/jA73EGnvYhgegvJrljV0sd9d\nm7lvdbDPLEzjwhe2doSq1p6pFZlfuxC/Sg7gNW+mkmw9GBzltEkRCGxT4lbUKC+QOFNxot5iAJfv\nn7gDdiKSmdZS6XM2EOY6Ztde60BzZ48nsOXCPcvS9p0TEc8qc3OSdjKufyydSeLZRCrV7x9T1Sdj\nnr+sqv/MjFhGruPWFYxlbVM3F/0ruGX7Jc8ErwhiGX/rxqQNId3mjumaVOb6dGYOqnbhrk6zxT/W\ni9vvS4G7VnlXFpB81VOWIMG5UJSDfIo1h8UT7NDpKrLBWX249RKf3NjOef/Ywas+N2Df+6+qtJ7b\nDdTwK56caHumlPtQk12/KGPY4FZ698OvmWNQuaeTphUzx4mw61J43KfCQy+PrI2fjAvEP2y7P3yk\nxlvcNcjM4/puDps4tDk8LsX54ulO/KBPwVvwKgt3YkykPAY6h3pNi/GyeJRshlYfe/oEwvxtXRtf\n+U+82fycOaUJFXt/CFNzMtH2oSyLlklMkRkZIUwV9T8s9yoyvzqKfzp+DHeeOM63MvjP3wwuf/Sh\nJ+KDXSuKJC7KbSBMKsv3FBd2FVYvbi+z6iEsFhzEJfPjyye5AR29uKsgV1kUJVjwTi8N37jTj2Q+\nsh3tiZVsugha1S9rcNv0pL/fnPue3GatLs2dg3NNhpqwtRaz75dnZCVNnT2c9fft3BdwRx/LWp/J\n8p1G79jMaJTYFQsrcG+y/7Olg7tWev1tXT2K279wV0BzzP4ywVnd3exTnBigwZlgq7LMtAgwzvH5\nuUm0vbjKY5QzMU4tz/dtupkncOG0cAE6QbgJwbErsH9uaPNEh2bKtCgi3HfS2KT7ua1f0oHr49qe\npHize+Ph1x18OJD0XUWTk5tFJDtKERhZzW21LTy5MVxofLd6zXFv+qzk9om2mRcRas+b7Nl+3ave\nvJ3lDV24QYRvfzD1/KVEuL6uXZ3q67NzV2rZuCKrKJS4aohNXUqXT2K0a6ry85FddVAleQLVRcKv\njxrNo6eNY/l5kzhpfOoteGJxVxOxSvUHr3lX5pmM0Dtuagl/PWVc4HYB9svAisztJRZ0w9GL+3mN\nGqmmRVXtBpYDyW9BjBHPFS9482gSsdYJCnFzmL51UGVcVFx1cR6ji+N/jGsa46tRtHT1cPj9W+P2\nOWJSkacc00A5wKdend+qz51sgpp6DiV5Ip5q8bt8JskwrWg+t2AUGz8yhbc+OJnz5pZx2MRixrnL\n436QyD9hOzAFAAAgAElEQVT0llPpZXSxUJHhSfuoycWcMcs/lXbPqoKMVNA4cFy8ckyqyEZAMjSE\n95HdBjwkIheKyAkicnzvXyaFM3KLzQGBGu6PL5ajH9wWp4TcEPZ3T/NOFIvOnugZWxtjkvzlYm9y\ncjoqK7j4Vb9wJ/obljR5UhGyLWqxF7eeoevbA6/PJWhiLCkQStM8abrnuuw/9XT1KGsauzw3EA+c\nPC5UBZmBclRAzczDJ2UmoMddzft9RrG4XaRHtI8M+AwwGrgGuBG4Kfp3Y2bEMoaa1bu6qL5lA9W3\nbODL/6kPlbP12Hr/9u/nJWnf/qmnIuH2v1zsNQ/N9VFAY0ryPZF/D8Qkov5huddnlinfgBtJGTtx\n/Lm2ma+/GL9CzRevXylbcCdJv1wyt2ZmeZqj8hLht/q7rbaFhXdviRubU5HPfoNU3X2vgCLUH0zy\nne8vrn/V9b+6eKMWs/MmaqCETYieHfDn3w/DyGl6VDngnr7J4eZlzfwrhN8rKI/qpOmJK5ndt6aV\nurZurnrJ6+sKuqt3c3lWR6sr1Lf3eGoBQubCjt2ovVhTzi3LvMEfVUV5g7JS6A+zK+OV8lKfFAq3\nw/OuEGWS0sUCH6Xx7UXe70zIWrppYe9q/5X+oSGr3aeKd0WWmmkxW2+iBkpo9SwiBSJytIicLyJH\nRVuwGMOQ1bu8iuCsx3bQkaAt8CvbOrjWZ1L508LWUK3Vf++zikqEm8vzx9rI63+91L/moV8ljnTg\nTgyxpkW/6iaZ9tsMBLfFzddejM+L8vv8QyzU08YZs0o9Y27IPUTSNQaLsT6+v0wqC9e/GpTy0Yvl\nkcUgInsBbxGpufgFIhXr3xaRvTMomzFE1O7yD5P2qyMHEd/Y8Q95OyRfeUAF80ZpqMn7Zz75YN85\nOLhG3Yk+vrP2bg0sb+U3CaaDRAEIbot5CF5hZgPuzYF7A+I3aX5uwSjPWKYoyBOefp+3iWUsswfR\nrNjL+2fHf7f+dELmFKnrX03mI3MLC490H9kNwG+B6ap6mKpOA34dHTeGGX4rMoCPP7XTY8pY09jF\nXnds9t1/XrSMk4iwV4AJphe/aL+L5gX3cPJLSn2nsYslPgmiz5wxIS1Rc34EJek2d/b4ttnwS+rO\nFtxmn/Xt8fL7mbEOHEDJqf6wb5JGnAdPGPyqKdcfWsVHasqYWpbPbceP4dgpoZqC9ItRBUJsml5L\nlya0lHhKVI1kHxmwEPiJxnv8fxYdN4YZfuaaXu6MST7e2NztcbTHMiOmvcdVB1ZSXiCE7WZx6YJR\nCQuc+m17bH2bJ8l68wVTkk5+A8E11fQ2Wdzm4y+sKBS+uM/grWBSxS3f1dqtcRGl7t3/AQmiUTOF\niHDytGD/01CsOMaU5PPLI0ez5IOTOH1mZlb+vYiIT3Rp8O/Vr8jzcCSsItsIHOOMHRUdN4YZQS1Y\noK8lSXeP8oVng4v75kl8Quh7ZpZSe/4k1n54MnefmDwl0W2V4ccUp1OzGyyyV3UBJRn+4brNNHsV\nqRvxN608n3UfmZKWvlSZIjJJBueSuS1x3Al1sPDzS/WS7tqG2Uh1Cn6yMHl/w4Gwn/rXgQdF5C8i\n8gMR+QvwYHTcGGbUJSh7c9eqVna29zD21o38Y0NwJOOY4jwKnEm7rCCPUYV5vHtaCRsv8FboiCVR\nh+JeTpuR+O73XYNg9prlhN9vau5VZPHXcP7o3IiNcpVTr8n3uS3tXPhknbPv0EyKiVYVwzXhNxY3\nBD/Rjaf5yGJQ1QeJNNJcDFRE/x+kqg+Eeb2I3CwiW0VkcczYNSKyQURei/6dFrPtShFZISLLROTk\nlN6RMSBUNWmdxC/9J3lzzLYkMdDJ7pyDwppj+WySQIPB6MQ83lkd9JoU3Rp46a4qkikqA8xWlz/v\nrdiSKb9jMhIpq+E6Ucfi9rN7PqAPHviVqBqeK9bAdyUiG2Me36yqy1X1u6r62ej/5Smc5/fAKT7j\nP1XVhdG/R6Lnmg+cByyIvuaGaL1HIyS7Onpo7Wcyzc/e9A9fj+X+EAWBf3Bo8j5MB4/397GUFUio\niX9OZUHCsPpLB8EfNd45f2+XaNe0OBhKNR1UOqanXZ1Ke7f61sA8bsrQlF9NZB4brqazWNx6i1e/\nvCuwYIH1I4NCEel1ZpwzkJOo6tNAXdIdI5wB/EVV21V1NbACOGQg5x8pqCrvf2w7M27bxOQ/buSJ\nDf6VNhLhlwuWKntU5nNmiHD3oAiqVJpfzqv2V4YTS/MS+lLSxQR3RRY1KW5rHR4rsl0dPZ56mBAx\nHZ/kkwIxGCRakR08yFGUQ4GfteO+1QGNUM1Hxm+AdSKyFigTkbV+fwM8/+dF5I2o6XF0dGwqsC5m\nn/XRMSMJd65q5YkYv9VlIUyAsawP6Op8aMiQ5p8eVs3lCyu4+8RxocJ8g+4O3VVBItwKH724yb2Z\nwm1/0mtS3OqU3ndbvmQrfoWDf+hTWf5Te5dT5NOuZTAIMo/99ujRLMxA65Rs47/3r/CMPbrOe9Pa\n3aOeYI/h6kMMdESo6lUi8htgJvAYcEGaz/0r4DtEOqt/B/gxcHF/DlRbWzsgQQb6+sEmSN5rXioh\n9t5kXVM3y5fXErYi0i3rCgDvRFDU1UKCrwoA9xzUyoz8FhgFXVugNiYqP0jertYi3+NOy28N/Zlc\nPAZuwpubVdTZTG1tcFRlEKl+F1ShUErp1MhFbu5S3ni7lrV1xUDfKqyjbhO1Pekv55T2725LIdB3\nE/Cdl+rY1O5VHGeUb6G2Njj1Ioh0yJvXkAfErwa/OqeDA3o2kO7LkY1zQ4nCHmUlrGzp+1yWbm2C\nKfHyNncBMb+Nkjxl5YoVgyhpYhJd25qampSOlXB2UtV1RFZl71XVp1I6chJUdfevQER+BzwUfboB\nmB6z67ToWCCpvulYamtrB/T6wSaRvJv+7b1M1dPneELEg+jaUQ/E1wccVSB887DJnPiwt3JHL5sv\nmBIY5p5I3v+b3s3c273J1PtNHUNNjfeuM4gfdjdxudM+ZvrYKmpqqkMfI5msiZjw6mY2xFT+r5w6\nm6alO4C+Fe4Bc2dQk+ZVYia+uwvaGmFTn3nZT4l966BK9t4zdSNJuuQtauyCJfFKdNaUCdTUBCfQ\n94dsnhsemNLFPnf1XYONnQVAe5y8W1q64fm+31dFUX7WvJ90X9uwUYtPpO2MUUQkNv76LCKRkBAJ\n6z9PRIpFZDZQA7yY7vMPNzb6FMqF4Ir0fviF8V5/WDXvCgjKAPjawop+52qNK8nnJ4d5lU13igX8\n/BT1YIaG+5kXczXYY1ZF8mhRt5TVYDN9lPfzHq4msyAmlcVfg4YO9dS9HCn+MUihaPBAEJHbgeeA\neSKyXkQ+DvxQRN4UkTeA44AvAajqEuBOYCnwN+Bz0eaeRgIeXuvv7L303+H9ZG7u08Xzyjl/bhki\nwqUBoe4D7a118V7lHDM5PvrNr45iIlxFAoPbhdlVUm/u6IyrgpEn3rqF2cpMHyURy/zRBZwwdWiC\nPHrJE/EEE+03Zvj7xmIpyBNi44x6FNz70JGSQwbJHB9pQlXP9xm+KcH+1wHXZU6i4YerhHpJtJpy\nWdUYH+zx0T377OuTAyIJZySZ+MJwxcIKXtneQWOncsr0Eham2CLeLxjlgEGMXnNLO33l+XrP9myu\n6BFLohXZ+XPL+Mlh1RmvlhKG6w+toqWrh6U7u/j03uWhEuiHG2UFebR19/3u3SlgpOSQwSApMmPo\nSFSlI36/btY5dQpjTXaTA/K19klDHcPDJxXz2jkT2dbWw7yqgpT7dRXkCcs+OIkD7tlCW7fyP4dU\nccTEwVNkM53Jv8cx8eSKWRGCV9gC/Pzw6iGLVHQZX5rPnSeOG2oxhpTyQqEuprhOS7dTuqrdVmSI\nyLfDHEBVr06fOEZ/Caq35t6VBTHHJ+giNtnYtclDpNbh9BRyvhIxtiR/QHlfE8vy2XjBlLTIkiof\n3bOcH/iEqPcSNtgmm5lVkZ81SsyI4PY9cxt6X/96/HcylfzMXCPRiiw2crAEeD/wEvAOMINIkvI9\nmRPNSIWglufJOsgmItYcNn90ISX5EJse9cR7J2Rtt+PBZGp5PidNK+ax9f61J3NpRRZEmCLOxuDi\nBm+0xqzI/rmhjVe2x1djmTOMP8PAX5iqXtT7R8SycL6qHqGqH1LVI4mUkTKyhKAVWVt38rqHfuVt\n/nu/+PD36uI8rj+0muoiYXxJHrefMIbJPqu0kcq08uBJIleqeiTiYwl6wxlDQ7lTrzS2IflXnvMG\neaXqe84lwqroU4EPO2MPArekVxyjv7hNEGNp6OihpCB4MvVrAPmNA715XBfsWc4Fe9qE5odbESOW\nCQnqQWYjH6kp40+1fX3nSvOF02cMbaSi4SXRiswvleaYIaqNORiE/YWtAD7njH0GWJlecYz+sjOB\nCTGZedFtmDixNM9Mhini1iiMZWKOrcj+e/+K3RXWJ5bm8co5E+37kIW4wRsxOfmevLq/njKOvGH8\nGYZdkX0CuE9ELidSZWMqkbIFZ2dKMCM13AilWFxF5bLLyTdJNCkb/iRKwJ5cllvXc1ZFAc+fNYHF\ndZ3sN7ZwUIovG6njKrLYFVmTY2VJR3RxNhNKkanqqyJSAxwKTAE2Ac+pqre3gzHoqGrCLrGJtoE3\nUGSoGibmMom6JU/MQV/i+NJ8jpuae3KPJNzC3L0rMlWlZYS0b+mlX7eK0bYsRSJiDpMsoKVL6Uyg\nq1I1LVYO48TJTBFUhV8gbSkKhhFLkI+sowdiF2QFwrBPnQg1Y4nIvsBy4Hf0VeQ4Brg5Q3IZKVCf\nxHSYTJG5psVEqwvDn33HFPq2n5lZkR+qpY1hpIonjyz6M252S1MN4xqLvYT9hf0KuFpV9wJ6zYlP\nAUdmRCojJRL5xyC5j8xVdIki8Ax/8kR8V2UHjYD+WMbQ4F2RRf57SlMVDP8bqbDvcAHwp+hjBVDV\nZiB5G2Aj49yzuiXh9mSKbpej6GxF1j+aOr03DGeE6JRtGP2hzM0j6+7riRe3n63IdrMGOCh2QEQO\nIRKWbwwhLV09/OSNpoT7JPeRmWkxHYzymTDeO9Pyr4zM4I1ajPwfaYEeEF6RfRN4WESuJRLkcSVw\nF3BVxiQzQvHs5g7PmOvXTRp+7wZ7mGmxX3zJqYZyxcIKy78yMoZ749TSE3nuht6PhF5tYRtrPgSc\nAown4hubCZytqo9lUDYjBMvqvRkQbu+rpOH3tiJLCydPL+Gje5YxriSPs2aV8sV9/Xu4GUY6CFyR\ndcX/nv0sBcONpHlkIpJPJDrxU6r62cyLZIRFVbnqpV2e8SMnFXPfmr5Gm6maFm1F1j8K84RfHDGa\nXxwx1JIYIwFvHlnUR+YxLQ7/G9Ok7zDanfkkoP9l1I2MsGi7fz76p+bHp/e9GrBfLxbsYRi5h7vS\nCopatGCPPn4KXCsiw7vOSY6xpM6roL6wzyjflhtdbrfHGMy0aBi5h+v7au0JWpGZIuvl88BXgUYR\nWScia3v/wrxYRG4Wka0isjhm7HoReVtE3hCR+0SkOjo+S0RaReS16N+vU35XIwQ/k+G176qk2kcR\nLW/oCjyOp9biCLiDM4xcJ6hosLsiGwmKLGzR4I8M8Dy/B34J/CFm7HHgSlXtEpEfAFcCV0S3rVTV\nhQM857CnzskPO2tWKSJCkU9FpF0BfjJV9bSAqQpod28YRvZQWiAI0cReoL1H6O5RT7DHSKgsE7Zo\n8FMDOYmqPi0is5yx2IjH54FzBnKOkYiryI6c3FdF4rgpxTy5sa9jsesH6+Wdpm5au/u2VRSKp/SN\nYRjZR54I5QUSF27f3KWexPyREH4fqMhE5Buqel308beD9lPVq9Mgx8XAHTHPZ4vIa0ADcJWqPpOG\ncww7trTGK7LYsHvXvOiaD3txG/DNriiw3CfDyBHKC72KzFOiagS4ChKtyKbFPJ6eKQFE5BtEepvd\nFh3aBMxQ1R0ichBwv4gsUFVvnHmU2traAckw0NcPNr3y/n1dWdx4Qf1GamsjX2JtLSL2412xfgu1\n3V4/2dv1eUBf9Yn8rra0X49cur65JCuYvJkm2+Ut0hJiQx2WrFjNlp2FxP72G7dvoVa6vS8eYhJd\n25qampSOFajIVPUzMY8vSumoIRGRjwHvAU5Q1d4aju1Ae/TxIhFZCewJvBx0nFTfdCy1tbUDev1g\n0yvv4rpOYGvctqMXzNkdcTi9vgG29JWuKqkeR01NfOUJgBVrW2Fx3e7n4yvLqKmZkXZ5c4FckhVM\n3kyTC/JWLd3Kura+6OXxU2eQt2UX0SkUgDnTJlMzI7tqfqb72obykYnInKBtqrqqPycWkVOAy4Fj\nVLUlZnw8UKeq3dHz1gD9Osdw5juLGjxjsdGGbuRhkGmxsdM1Qwx/x7BhDBdcf3Zzp9e0aMEefawg\nEhwTe9V6r1bSroEicjtwLDBORNYD3yISpVgMPB71yTyvqpcARwPfFpFOIknYl6hqne+BRzCL67xm\nwljfVqXrIwsI9vDmkA1/e7phDBfcVi7NXToi88jCRi3GzYoiMomIMgoVhKGq5/sM3+QzhqreA9wT\n5rgjmXdNKGTDmmC7t6vIgspUuQWFLRnaMHIHV0k1dyotnhXZ8Fdk/Zq1VHUzcBnwvfSKY4RFiP9y\nXrEw3v/lMS0GKjKr6mEYuYrrCmjo6BmRK7KBzFrzgLKkexkZYXtb/Grs0AnxnYg9pkWfpo9gisww\nchm300Vdew9NbkL0CFBkYYM9nqHPJwYRBbYACMwvMzLLjrb4L+vYkvgvtFvBPsi06C0YPPy/9IYx\nXHB/93XtPT6mxeF/cxo22ONG53kz8LqqZneSxTBmu6PIxpXEx9y4K6vwwR7D/0tvGMMFd0W2paWb\n2ADlfIGR8JMOG+xxa6YFMcLTo+opT+VZkYUMvzdFZhi5y2hHka1tinc5lBfKiKjUE9a0GMqEmKZy\nVUYS6tt7iCmPSGWhUJwf/2WtcBRSY4fSo0qeJDY5mmnRMHIH9wZ2sdPaadwIKQAe1rRYA7wfeAl4\nB5gBHEIkTL4tuk9wwysjrbhmRffLDJFuxWUFstterkBTp/r4ziz83jByFde02OT4x6aWJ03zHRaE\nVWQCnB/N8YoMiJwNnJup8lVGMDvaXf+Yv/KpLJQ4x++ujp64aEZVNdOiYeQwriJzcS0zw5Ww7/JU\n4H5n7EHgtPSKY4ShzlmRuXbyXjwBH04Ifmu3xjmGi/KgZASE6hrGcGFMwE1sLyMh9B7CK7IVwOec\nsc8AK9MrjhGGemcVVR2gyFwFt7w+vqyVG8no5p4ZhpHdFOZJwo7upsji+QTwZRFZLyIvROslfiU6\nbgwy9Y4CGh2ggCaVxdvHX9zWHve80YlkTPSDMAwjOwmyyACUjZDfdNjw+1dFpAY4FJhCpGfYc6ra\nmfiVRiZwm2EGfZHdPEh3BWYrMsPIfcaW5PFOk3/d1fKCkfGb7m+txaeBIhEpT7M8RgjWNcWbCINM\ni6c7PYi2tcZ/2d36i6bIDCP3SBTwYabFGERkX2A58Dv6qtYfA9ycIbmMBNQ2xCuyoBDbyWXxH+82\nJ0jEDf6oGCFmCMMYTiRUZCPkNx32FvxXwNWquhfQa058CjgyI1IZCXGresyr8rcQTyiNV3BbWx1F\nZisyw8h53Oa4sZTZiiyOBcCfoo8VQFWbgezqnz0CUPUqsiAf2bjS+PHtbd2oxuSVOT8AC/YwjNzj\n9JklgdtGSsf3sO9yDXBQ7ICIHEIkLN8YRLZ0SFyQRkl+sCIbVSCUxpSuauuOz/y3FZlh5D57VxcG\nbrMVWTzfBB4WkWuJBHlcCdwFXJUxyQxfGuPdY8yuKKAgz//LKiKeVdm2GPOiR5HZiswwcg6/EnW9\nWLBHDKr6EHAKMJ6Ib2wmcLaqPhbm9SJys4hsFZHFMWNjRORxEamN/h8ds+1KEVkhIstE5OSU3tEw\np6U7/os5KonyGV/iKrK+yEXXtm4rMsPIPdzfeCxzA/znw42kM5eI5IvIrcBSVf2sqp6uqpeo6qIU\nzvN7Ioowlq8BT6hqDfBE9DkiMh84j4hf7hTgBhEZGZUvQ9DipIska5rnmh1j/WJe0+LIuHszjOFE\neWFeoAnRDfgariRVZKraDZwE+De0CkE076zOGT4D6O1zditwZsz4X1S1XVVXE/HDHdLfcw831rXG\nf2RBVT16cU0La2Jsk55gD1uRGUZO4meZ+eyCkZPmG3bm+ilwrYgEexVTZ6Kqboo+3gxMjD6eCqyL\n2W99dMwAXtsV/5EdOC7xR+L2KfvWy7t2P250VmSWR2YYuYmbWgMwpnhkrMYgfBuXzwOTiNRb3EZM\n7zFVnTFQIVRVRaTf/cxqa2sHdP6Bvn4wWdEcH2o7pWMrtbWbA/ff1lBE7Mfc0qW73+/25hJi72V2\nblpHbUP628rl0vXNJVnB5M00uSNvmWdkgW5JODcMNYmubU1NTUrHCqvIPpLSUcOxRUQmq+omEZkM\nbI2ObwCmx+w3LToWSKpvOpba2toBvX6waXpxXdzzQ/ealbB53rk0869/18eNjZ+xB9XFebzz7/jL\nuu/c2UxJcyO+XLq+uSQrmLyZJqfk/bd3ijx14dwhECQc6b62YaMWnwr6G8C5HwQujD6+EHggZvw8\nESkWkdlEulO/OIDzDBvq23vY1hH/kSUzB35orvdO7eJ/1bG11VtktMKCPQxjWDC7YuSYFaGfRYNT\nRURuB54D5kVbwXwc+D5woojUAu+OPkdVlwB3AkuBvwGfiwacjHh+v6zZM5Ys/F7Eu/2fG9u5Y2WL\n91gjJOfEMIYb+46J95UfPbl4iCQZGgYlyUBVzw/YdELA/tcB12VOotzkmc3tnrE8H0XlUiDQ5bi+\n1jR67w38lJ5hGNnPjw+r4qSHtwOR3/sl80cNsUSDy8jIlhsmrHd6Di0cGy6I1FViAK9t70iHSIZh\nZAGHTCjmN/u2saZgPO+eWsLeo9MZYJ799Mu0KCKlIjKy1q5ZwDKnfct3D6kK9boP7uGt7bzFCdf9\nwByr/2wYucyBVT1csbCSg8YXDbUog07YfmQ/ihYJRkROJ5LcvFNE3ptJ4Yw+drR5TYEzR4Vz6H5l\nvwrP2Prm+OPVjJBSNoZhDD/Crsg+DPTWSbyaSDj++4D/yYRQhhd3BQUwfVQ45bNndWHSKCar6mEY\nRq4SdvYqU9UWERkLzFHVe1T1H0SKBxuDwL83xQd6BDXTDOKyfb2rsliqTJEZhpGjhJ0Nl4vIh4G5\nwOMAIjIOaM2UYEY8l7/QEPe8sTO10pfVCdqhgxUMNgwjdwmryD4L/BzoBC6Ojp0MhGrjYqSfVPNE\ngppv9mKmRcMwcpVQikxVXwIOd8ZuA27LhFBGPKre+PkZFamZFpMqMisYbBhGjhJ6NhSR44HzgSnA\nRiKtVp7IlGBGHz4Bi3ykxlt6KhHVSUyHtiIzDCNXCRt+/xXgL0TC7h8GdgB/jo4bGebSf+/0jM0I\nGbHYS7IVmQV7GIaRq4SdDb8MHK+qvSH4iMgfiQR+/DgTghkRVu3q4p7VA4+pKS8QCvMgKEbEepEZ\nhpGrpHIbvsJ5voqYvmRGZnglTaWkRCRwVVZeIBTkmSIzDCM3CavIrgFuEpGaaHmqPYHfAt8Skbze\nv4xJOYLZ2Ox1kO1R2b8WDaX5/sqqykLvDcPIYcKaFn8T/X8+kVVY78z34eg2iY6PrCY4g8CmFq8i\ne/S08f061q4Au6IFehiGkcuEVWSzMyqFEYhbE/GLszqYUNq/+4XuAENwZaEpMsMwcpeweWTvAETN\nhxNVdVNGpTJ2s3RnZ9zz/StTq+gRS3uAJrOqHoZh5DJhw++rReTPQBvRoA8ReZ+IfDeTwhmwvS1e\ncU0v7b8iu2heue94f1d4hmEY2UBYm9KvgQYiRYJ7w+ieAz6YCaGMSDWPa15uoKEjfhWVYvpYHF/2\naecCJK2MbxiGkc2EnRZPAKaoaqeIKICqbhORCQM5uYjMA+6IGZpDpE1MNfBJYFt0/Ouq+shAzpVL\ntHcrE/+w0XdbwQCsgBNK87loXhm3LGuJG582EO1oGIYxxISdwRqAccBu35iIzIh93h9UdRmwMHq8\nfGADcB9wEfBTVf3RQI6fq/x2aVPGjr3vmCIgXpFNKbMVmWEYuUtY0+KNwD0ichyQJyKHAbcSMTmm\nixOAlb2BJSOZaxbtytix51V7712mllvUomEYuYv4VVb37CQiwBeATxPxk60lkj/2cw1zgDCCiNwM\nvKKqvxSRa4isyhqAl4GvqGpcwcGGhobd562trU2HCFnB9g449UX/gsAXTuvk0lmdvtvCUt8JJ74Q\nf/ynD2vB4j0Mw8gWampqdj+uqqpK6lAJq8gmqermsOOpIiJFRCrqL1DVLSIyEdhOJMn6O8BkVb04\n9jWximwg1NbWxl20oea5Le2c+sh2z/gH5pRyw1GjWb1yxYDkVVUOu38rb9d3AbDfmEKePmNArs6E\nZNv1TUQuyQomb6YxeTNHKrKGUWShO0QDlT7jS4ExIY+RiFOJrMa2APT+BxCR3wEPpeEcOcGONm94\n/daPTqEooLxUqogIvzl6NFe/tIs8ge8cXJWW4xqGYQwVYRWZZxYVkUqg/0lN8ZwP3B5z7MkxSddn\nAYt9XzUMcRXZh+aWpU2J9bL/2CIeOGVcWo9pGIYxVCRUZCKyjoh5r1RE1jqbxxKjfPqLiJQDJxLx\nv/XyQxFZGD33GmfbsMZNgB5XYoEYhmEYiUi2IvsIkdXYI8AFMeMKbImGzw8IVW0mohRjxy4I2H3Y\n88PX4yMWx5siMwzDSEhCRaaqTwGIyDhVbUm0rzFw6tq6aXeK3S8cVzQ0whiGYeQIYW/3L4ma+hCR\nQ0VkrYisFpHDMyjbiGPxzi7P2CETTJEZhmEkIqwi+xKwOvr4e8BPgO8CP82EUCOV+vZ4/9i08nyK\n0xzoYRiGMdwIG7VYpaoNIlIB7A+8W1W7ReTHGZRtxOE2vjxykq3GDMMwkhFWka2LmhEXAE9HlVgl\n4DEjz5gAABNFSURBVG1fbPSbnc6KrMo6NxuGYSQlrCL7KnA3kRYu74+OvQd4MRNCjVR+9kZ8sWAz\nKxqGYSQnbIfoR4ApzvBd0T8jTexwVmT1HenKNzcMwxi+9LsRlaoOrHqtEYdfzctjJhcPgSSGYRi5\nhTlhsgTXPwbw3pmlQyCJYRhGbmGKLEt4ZnNH3POZo/LTXmPRMAxjOBKoyETk+pjHxw+OOCOXN3bE\nK7IFYwqHSBLDMIzcItGK7FMxj+/PtCAjnac3tcc9f5+ZFQ3DMEKRKNjjdRG5m0jPsWIR+bbfTqp6\ndUYkG2GsboxPyTtsoiVDG4ZhhCGRIjuHyKpsJpEK+NN99klLl+aRTn17T1z7lnyBqeX5QyiRYRhG\n7hCoyFR1K5F6iohIgapeNGhSjSDq2rqZc/vmuLHJZfkU5Fmgh2EYRhjCJkRfJCKjgfcCU4ENwEOq\nWpdJ4YY7m1q62fuOzZ7x6mILJjUMwwhLqBlTRA4DVgKXAPsR6di8Ijpu9IMeVU5/ZJvvtpmjzKxo\nGIYRlrCVPX4GfFZV/9I7ICIfBH4BHJwJwYY7S3d2sarRv+byp/YeNcjSGIZh5C5hbVh7Anc6Y3cD\ncwcqgIisEZE3ReQ1EXk5OjZGRB4Xkdro/9EDPU+2sb0tuHGARSwahmGEJ6wiqwXOc8bOJWJuTAfH\nqepCVX1X9PnXgCdUtQZ4Ivp8WHH3qlbf8c0XTLGKHoZhGCkQ1rR4GfCQiHwBeAeYBdQQaeWSCc4A\njo0+vhX4F3BFhs41qCza1sHj69u4d7W/IispMCVmGIaRCuJXdd13x4h573Qi7Vw2Ao+kI2pRRFYD\nDUSadP5GVX8rIvWqWh3dLsDO3ue9NDQ07Ba8trZ2oGJklIZOWNmSR3EeXPx6MT34K6sTxnbx/b07\nfLcZhmGMFGpqanY/rqqqSnp3H7qNi6ruBP7UP7EScqSqbhCRCcDjIvK2c14VkYTaNvZNp0ptbe2A\nXp+MxXWdvPehbbR2J79hOHvvcdTUlCfcJ9PypptckjeXZAWTN9OYvJkj3bL2ux9ZulDVDdH/W0Xk\nPuAQYIuITFbVTSIyGdg6pEKmwK6OHv5vSRM72nr4xN7lXPfKrlBK7NIFozh3TtkgSGgYhjG8GFJF\nJiLlQJ6qNkYfnwR8G3gQuBD4fvT/A0MnZWqc/uh23qyL9By98e3mUK85eVox3z2kKpNiGYZhDFtC\nKTIRyVNVb+fHgTMRuC/iBqMA+LOq/k1EXgLuFJGPEwku+UAGzp127lnVsluJpcI3DzIlZhiG0V+S\nKjIRyQeaRKRaVduT7Z8KqroK2N9nfAdwQjrPNRh8/KmdKb9mz6oC9rHeY4ZhGP0maR6ZqnYDy4Gx\nmRcnd+ns6V8jgB8dVp18J8MwDCOQsD6y24jkkf0cWE9M+xZV/WcmBMs1vvVyQ9J9rjmoki/uO4qX\ntnXw6No2jpxczNGTiwdBOsMwjOFLWEX2mej/a5xxBeakTZoc5Za3m7lhSeLAjopC4bL9KgA4ZEIx\nh0wwBWYYhpEOwrZxmZ1pQXKZLz1Xn3Sfr+5fMQiSGIZhjDxCN74SkUIROSpa9R4RKY+GzI9oGjq8\nwZwf3KPUMza+1FqzGIZhZIKw/cj2JRLw8TvgpujwMcDNGZIrZ3htu7ek1LcOquKoSfEV7A8YZ5GJ\nhmEYmSDsiuxXwNWquhfQmyj1FHBkRqTKIV7YGq/ITp9RwpTyfH5waDXTR+UjwGfml7NXtSkywzCM\nTBA22GMBfXUWFUBVm0XEa0MbQagq//NqY9zYcVMiQRzzRxfyxjkTaeuGUqtobxiGkTHCrsjWAAfF\nDojIIcCKdAuUSzz4Tptn7MBxfSZFETElZhiGkWHCrsi+CTwsIr8GikTkSuAS4JMZkyzLeWJDGxc+\nGd/FZnSxsN9YMyEahmEMJqFWZKr6EHAKMJ6Ib2wmcLaqPpZB2bKWjc3dnP+PHZ7xL+9XQUGercAM\nwzAGk1T6kb0KfDaDsuQMj69vw42636Myn0/tPWpoBDIMwxjBhA2/LxKRb4tIrYg0R/9/R0RKMi1g\nNrKmscsz9uAp4ynOt9WYYRjGYBN2RfYrYB7wBSJtVWYCXwemAhdnRrTsZVtb/HLsZ4dXM7XcEp4N\nwzCGgrCK7ExgD1XtrcW0VEReIBK1OOIV2biS0AVSDMMwjDQTdgbeDJQ5Y6XApvSKkxtsb+2Oe26K\nzDAMY+gIXJGJyPExT/8I/E1E/pdIG5fpwOeAP2RWvOyjR5XlDfE+solWR9EwDGPISGRavMln7OvO\n808DP+jvyUVkOhFlOJFIxZDfqurPReQaIjlq23rPq6qP9Pc86aShQ2ns7GuiWV4gzKowRWYYhjFU\nBCqyQWrd0gV8RVVfEZEKYJGIPB7d9lNV/dEgyJASz2xqj3s+tiQPEYtWNAzDGCpC55FlAlXdRNTP\npqqNIvIWkUjIrMVVZJPMrGgYhjGkhM0j219E/ikidSLSEf3rFBFvD5N+IiKzgAOAF6JDnxeRN0Tk\nZhEZna7zDJS/r4+vr7jX6CG9FzAMwxjxiKom30lkKXAPcAfQGrtNVVcOWAiRUURKX12nqveKyERg\nOxG/2XeAyaoaF+bf0NCwW/Da2tqBihCKug44+cX44M3bD2hlbnnya2gYhmGEo6amZvfjqqqqpL6b\nsIqsDhirYXZOEREpBB4C/q6qP/HZPgt4SFX3iR2PVWQDoba2Nu6iJeLZze2c/uj2uLGdH5syqD6y\nVOTNBnJJ3lySFUzeTGPyZo5UZA2jyMImQN0KfCjkvqGRiAa4CXgrVomJyOSY3c4CFqf73AAPv9PK\nwf8u41NP1xFGRz+yNt6seOasUgv0MAzDGGLCOni+DzwnIl8HtsRuUNXj/V8SiiOAC4A3ReS16NjX\ngfNFZCER0+IaImH+aeeg8ZHeYXeubOWBNa08e8YE9qgs8FVO3T3KX1a0xAs/qcizn2EYhjG4hFVk\ndwOrgftwfGQDQVX/DfgtaTKeM9bc2UNsx5X2bnjXvVsBeO/MEm45dkxcS5a7V7eyo72vNFWBwDlz\n3GInhmEYxmATVpEtJOIjS1uU4lDzjw3tvFnX6bvtr++08Zu3mvncgkhblutf28V1rzbG7XPqjBJG\nF1tpKsMwjKEm7Ez8DDA/k4IMNu+bWUKirivfeLGBho4edrR1e5QYwDGTizMonWEYhhGWsCuy1cBj\nInIfXh/Z1WmXahAQEa48oJKjCrdy7itltHZ7gz1m3uZfE7mqSDhrdmmmRTQMwzBCEHZFVgY8DBQR\nKRgc+5fTTCxWlp8/iS/uE7678+Onj2dsiVX0MAzDyAZCrchU9aJMCzKUVBTmce3BVSwcV8hF/9qZ\ncN8HTh7LntWFgySZYRiGkYxQikxE5gRtU9VV6RNnaDlzVik7D1O+/Fy97/YL9yzjaPONGYZhZBVh\nfWQriOR0xYZH9DqVho2NTUS4eK9yLtyzjCtfbOC3bzXv3nblARVcsbByCKUzDMMw/AhrWozzpYnI\nJOBbRKIZhx35ecIPD63mqgMr+eeGdqaU53HIBFuJGYZhZCP9Kt2uqptF5DJgOfDn9IqUPVQW5XGm\nRScahmFkNQPJ6J1HJJrRMAzDMIaMsMEez9DnE4OIAlsAfDsTQhmGYRhGWMKaFm90njcDr6vq4DQC\nMwzDMIwAwgZ73JppQQzDMAyjP4Q1LRYBHyNSPDiuBIaqfjT9YhmGYRhGOMJ2iL4d2B/4KxDXlEtV\nr82MaIlJV4dowzAMI3sJ0yE6rI/sFGC2qvqXvDAMwzCMISJs+P1awDKCDcMwjKwjrGnxK8C5wM/x\ntnH5Z2ZEMwzDMIzkhFVkqwM2qaoGFhQ2DMMwjEwTyrSoqrMD/nJWiYnIKSKyTERWiMjXhloeABGZ\nLiJPishSEVkiIl+Mjl8jIhtE5LXo32kxr7ky+h6WicjJQyDzGhF5MyrXy9GxMSLyuIjURv+PzgZ5\nRWRezDV8TUR2ichl2XR9ReRmEdkqIotjxlK+niJyUPRzWSEivxCRpA7zNMl6vYi8LSJviMh9IlId\nHZ8lIq0x1/jXgylrAnlT/uyHWN47YmRdIyKvRceH9PommLsG57urqiPuj0jF/pXAHCLNQl8H5meB\nXJOBA6OPK4jUspwPXAP8t8/+86OyFwOzo+8pf5BlXgOMc8Z+CHwt+vhrwA+yRV7nO7AZmJlN1xc4\nGjgQWDyQ6wm8CBxKpGPFo8CpgyTrSUBB9PEPYmSdFbufc5yMy5pA3pQ/+6GU19n+Y+DqbLi+BM9d\ng/LdHUitxVzmEGCFqq5S1Q7gL8AZQywTqrpJVV+JPm4E3gKmJnjJGcBfVLVdVVcTabdzSOYlTcoZ\nQG8S/a3AmTHj2SLvCcBKVX0nwT6DLq+qPg3U+cgR+nqKyGSgUlWf18jM8IeY12RUVlV9TFW7ok+f\nB6YlOsZgyRokbwKG9Nomkze6SvkAcHuiYwzidyFo7hqU7+5IVWRTgXUxz9eTWGEMOiIyCzgAeCE6\n9PmouebmmOV5NrwPBf4hIotE5FPRsYmquin6eDMwMfo4G+Tt5TziJ4Fsvb6Q+vWcGn3sjg82FxO5\no+5ldtTs9ZSIHBUdywZZU/nss0FegKOALRpfJjArrq8zdw3Kd3ekKrKsRkRGAfcAl6nqLuBXRMyg\nC4FNREwK2cKRqroQOBX4nIgcHbsxeleVVcnrEqlU8z7gruhQNl/fOLLxevohIt8AuoDbokObgBnR\n78qXgT+LSDZ0qs2Zz97hfOJvxLLi+vrMXbvJ5Hd3pCqyDcD0mOfTomNDjogUEvki3Kaq9wKo6v+3\nd28hVtVRHMe/v9LMxAxrAsdKNE26UElZEj5ECpWXInvIyEqQLg8hglCULxIFBSFJPSRdMAK7YOSF\neqmgJ7OL0k0qzAxKS8scUrIUXT3813GOB5V0Zs45W38fOMxh733+e83/bPaa/d97/mtbROyPiAPA\ni3QPb7X894iILflzO/BOxrYthwhqQxvbc/OWx5tuBtZHxDZo7/5Nx9qfWzh0SK+pcUuaDUwD7sqT\nFzmEtCPfr6PcE7mo1bEex3ff0ngBJPUDZgBv1pa1Q/8e7txFk47dkzWRfQaMkTQy/zqfCaxqcUy1\nce+XgW8jYlHd8mF1m90G1J5iWgXMlDRA0khgDOVGabPiHSRpcO095Ub/NxnXvbnZvcDKdoi3ziF/\nzbZr/9Y5pv7MoZy/JE3IY+qeus/0KUk3AQ8Dt0TE33XLOySdmu9HZaw/tjLWjOWYvvtWx5smA99F\nxMEhuFb375HOXTTr2O3tp1eq8gKmUJ6s2QQsaHU8GdNEyqX3V8AX+ZoCvAZ8nctXAcPqPrMgf4fv\n6aOnp44S7yjKk0dfAhtq/QicDXwIbAQ+AIa2Q7y5/0HADmBI3bK26V9Kgv0V2Ee5PzDnePoTuJpy\nUt4EPE/+z2gTYv2Bcu+jdvy+kNvensfIF8B6YHozYz1KvMf83bcy3ly+FHiwYduW9i9HPnc15dj9\nX/8QbWZm1q5O1qFFMzM7QTiRmZlZpTmRmZlZpTmRmZlZpTmRmZlZpTmRmbVQzhR+fS+3uVTSE73Z\nplk769fqAMxOZhFxaatjMKs6X5GZmVmlOZGZ9QJJnZLelvS7pM2S5ubyhZKWqxRE3CVpvaQr6j73\nk6TJ+f4aSZ+rFPzcJql+mrJbchiyS9JHki6uWzcu290l6U3g9IbYpuWs6F2S1ki6vG7dIyqFJXep\nFDic1IfdZNYnnMjMekjSKcBqylRdwym1zuapu+rtrZSZ9ocCy4AVOcFqo8XA4og4E7gQeCvbv4gy\nXdE8oAN4D1gt6bScK3QFZaqlobmf2+tiGwe8AjxAmS5oCbAq57gbCzwEjI+IwcCNlEKpZpXiRGbW\nc+OBjoh4PCL2RsSPlJnUZ+b6dRGxPCL2AYsoV0wTDtPOPmC0pHMiYndErM3ldwDvRsT72cYzwEDg\numynP/BsROyLiOWUSbFr7geWRMQnUWZ5fxX4Nz+3n1Kh9xJJ/SPip4jY1Gu9YtYkTmRmPTcC6Myh\nuy5JXcBjdBcRPFhAMEq5kF+AzsO0M4dSeuM7SZ9JmpbLO4GDlayzjZ8pV3+dwJY4dNLU+qrXI4D5\nDbGdD3RGxA+Uq7yFwHZJb0g6XFxmbc2JzKznfgY2R8RZda/BETEl1x+su5TDkOcBWxsbiYiNEXEn\ncC7wNLA8y+NspSSkWhvKNrdQZkcfnstqLmiI7cmG2M6IiNdzn8siYmK2H7lfs0pxIjPruU+BXfng\nxEBJp0q6TNL4XH+VpBlZEHEeZWhvbWMjkmZJ6sgrrq5cfIByr2yqpEl5b21+trEG+JhSiXmupP6S\nZtBdHBLKEOeDkq5VMUjSVEmDJY2VdIOkAcA/wJ7cn1mlOJGZ9VBE7KdURL4S2Az8AbwEDMlNVlLu\nc+0E7gZm5L2uRjcBGyTtpjz4MTMi9kTE98As4Llsezql3tTeiNhLqRY8G/gz91OrzktEfA7cR6nr\ntJNSL2x2rh4APJVt/ka5Eny0Z71h1nyuR2bWhyQtBEZHxKxWx2J2ovIVmZmZVZoTmZmZVZqHFs3M\nrNJ8RWZmZpXmRGZmZpXmRGZmZpXmRGZmZpXmRGZmZpXmRGZmZpX2H8MZC8TN8965AAAAAElFTkSu\nQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x122d06810>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(moving_average(num_steps, 50))\n",
    "plt.ylabel(\"number of steps required for success\")\n",
    "plt.xlabel(\"episodes\")\n",
    "plt.title(\"RL\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# approach 2: actor/critic model\n",
    "\n",
    "a critic network (aka value network) predicts a single value for a specific location.\n",
    "\n",
    "the value network will place higher values near the final winning position (and low values in the \"hole\" positions). it will place the higest value on the winning final position.\n",
    "\n",
    "the value network expresses the the most efficient path to the policy network. it is no different from a critic telling an actor how to act (and similarly, it is up to the actor to follow the critic's suggestions.)\n",
    "\n",
    "the actor (aka policy network) predicts the best action from the current state. the best action is quantified as (1) reward + (2) discounted next value - (3) predicted value. the actor will no longer select the < action/next-state > with the highest value; instead it will select the < action/next-state > that results in the greatest change in value.\n",
    "\n",
    "* http://www.rage.net/~greg/2016-07-05-ActorCritic-with-OpenAI-Gym.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### critic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from critic import Critic\n",
    "import math\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "critic = Critic(OBS_SPACE, ACTION_SPACE)\n",
    "gamma = 0.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0, 200)\n",
      "(500, 200)\n",
      "(1000, 200)\n",
      "(1500, 200)\n"
     ]
    }
   ],
   "source": [
    "for episode in range(N_EPISODES):\n",
    "    \n",
    "    s1 = env.reset()\n",
    "    done = False\n",
    "    steps = 0\n",
    "    \n",
    "    while not done:\n",
    "        \n",
    "        # ask critic to predict value for current state        \n",
    "        orig_val = critic.predict(s1.reshape(-1,4))\n",
    "        \n",
    "        # take an action\n",
    "        a = agent.act(s1.reshape(-1,4))\n",
    "        \n",
    "        # what are the consequences of taking that action?\n",
    "        s2, r, done, i = env.step(a)\n",
    "        \n",
    "        # get critic's prediction on next state\n",
    "        new_val = critic.predict(s2.reshape(-1,4))\n",
    "        \n",
    "        # determine target value\n",
    "        if not done:\n",
    "            target = (gamma * new_val)\n",
    "        else:\n",
    "            target = (gamma * r)  # max value is discounted reward at final location\n",
    "        \n",
    "        # positions near final location will propagate backwards\n",
    "        best_val = max((orig_val*gamma), target)\n",
    "        \n",
    "        # each state is encoded with its \"best\" target value\n",
    "        # the target value is equal to the discounted next value\n",
    "        # we dont care which action led to this value!\n",
    "        critic.remember((s1.reshape(-1,4), best_val))\n",
    "        \n",
    "        # if terminal, append another replay for final location (otherwise final loc will never be recorded)\n",
    "        if done:\n",
    "            critic.remember((s2.reshape(-1,4), r))\n",
    "        \n",
    "        # move on to next state and continue\n",
    "        s1 = s2\n",
    "        steps += 1\n",
    "    \n",
    "    critic.replay(SAMPLE_BATCH_SIZE)\n",
    "    \n",
    "    if episode % 500 == 0:\n",
    "        print (episode, steps)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## actor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from actor import Actor\n",
    "actor = Actor(OBS_SPACE, ACTION_SPACE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0, 200)\n",
      "(100, 200)\n",
      "(200, 200)\n",
      "(300, 200)\n",
      "(400, 200)\n",
      "(500, 200)\n",
      "(600, 200)\n",
      "(700, 200)\n",
      "(800, 200)\n",
      "(900, 200)\n"
     ]
    }
   ],
   "source": [
    "n_steps = []\n",
    "\n",
    "for episode in range(1000):\n",
    "    \n",
    "    s1 = env.reset()\n",
    "    done = False\n",
    "    steps = 0\n",
    "    \n",
    "    while not done:\n",
    "        # get value for current state from critic\n",
    "        orig_val = critic.predict(s1.reshape(-1,4))\n",
    "        \n",
    "        # propose an action\n",
    "        a = agent.act(s1.reshape(-1,4))\n",
    "        \n",
    "        # what are the consequences of taking that action?\n",
    "        s2, r, done, i = env.step(a)\n",
    "        \n",
    "        # get value for next state from critic\n",
    "        new_val = critic.predict(s2.reshape(-1,4))\n",
    "        \n",
    "        # The actor is updated\n",
    "        # by using the difference of the value the critic\n",
    "        # placed on the old state vs. the value the critic\n",
    "        # places on the new state.. encouraging the actor\n",
    "        # to move into more valuable states.\n",
    "        \n",
    "        # actor learns to predict a set of actor deltas\n",
    "        actor_delta = new_val - orig_val                \n",
    "        actor.remember([s1.reshape(-1,4), a, actor_delta])\n",
    "    \n",
    "        # move on to next state and continue\n",
    "        s1 = s2\n",
    "        steps += 1\n",
    "    \n",
    "        # TODO: inside while loop or not?\n",
    "    actor.replay()\n",
    "    n_steps.append(steps)\n",
    "    \n",
    "    if episode % 100 == 0:\n",
    "        print (episode, steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x1230eb9d0>"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbcAAAEXCAYAAAAuiwoFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xu4XdO9//H3x7Vx6XYpkUQItcMvODhtHaoXRXGKaikn\naWnq0larRY9HhR+qyq9O20Nz2tNSQZ1WKeIS4Sl5ooRWtXHpIYIVgiQicYlNVYn4/v6YY8fKyl5r\nj6xL9s7an9fzrGfNOeYcY333aPmac445hiICMzOzdrJaXwdgZmbWbE5uZmbWdpzczMys7Ti5mZlZ\n23FyMzOztuPkZmZmbcfJzczM2o6Tm9kqRtIvJUX6LJE0V9L/SBpWds6dkib0ZZxmfcnJzWzVdDcw\nBNgC+DywC3Btn0Zk1o84uZmtmt6KiOcjYl5ETAN+Aewu6b19HZhZf+DkZraKkzQU+BywJH3MBrw1\n+joAM6vLnpL+RvEfqINS2X9GxOt9GJNZv+HkZrZqug8YC7wHOBzYBzijTyMy60ec3MxWTW9ExKy0\n/Yik9wM/Ab7chzGZ9Rt+5mbWHs4GjpL0wb4OxKw/8JWbWRuIiJKkm4HzgP1S8UaSdq449dWIeGrl\nRme28jm5mbWPHwJ/kLRn2v9s+pS7Ddh/ZQZl1hfklbjNzKzd+JmbmZm1HSc3MzNrO3UlN0mfkPTx\nZgdjZmbWDFnJTdJdkvZI26cCVwO/kXR6K4MzMzOrR9aAEkkvAZtGxBJJs4BPA68Bf4iILVocY926\nuro8WsbMrM11dHSosiz3VYDVgEizICgiHgWQtGET4zMzM2uK3OR2D/BTivWjbgBIie7FFsVlZmZW\nt9wBJV8CXgH+F/hOKtsOGN+CmPqlUqnU1yH0W+6b2tw/tbl/anP/1Cfryi0iXgJOryi7pSURmZmZ\nNSh3tOS/d89RJ2k3Sc9Kmi1p99aGZ2ZmtuJyb0t+C5idtr8PXACcC/y4FUGZmZk1Ije5dUREl6T1\ngZ2An0TEpcC2OZUlDZf0e0mPSpoh6cRUfljaf6dyqQ5Jp0maJelxSftVaXcjSVMkldK3R2+amVl2\ncpsj6cPAaGBaet/tvcCSzPpvAydHxChgN+B4SaOAR4BDgGnlJ6djo4HtKWYw/5mk1XtodxwwNSI6\ngalp38zMBrjcVwFOAa4D3gIOTWUHAn/OqRwR84H5afs1STOBYRExBUBa7v27g4GrI+JNYHZ6cXxX\n4N4eztszbV8B3Amcmvk3mZlZm8odLXkrMLSi+Nr0WSGSRgC7APfVOG0Y8Key/bmprNLglDgBngcG\nr2g8ZmbWfrIXK5W0HXAYRUL5BvB+YC2Kd99y21gPmAicFBGvrmCsNUVESKo63VYz3hXx+ybVuW9q\nc//U5v6pzf2zvM7OzprHs5KbpMOAn1Ekps8D3wDWA84H9slsY81U/8qIuL6X0+cBw8v2N09llRZI\nGhIR8yUNARZWa7C3juhNqVRquI125b6pzf1Tm/unNvdPfXIHlJwD7BMRx/HuIJK/Uoyc7JWKh2qX\nAjMj4oKMKpOA0ZLWlrQV0EnPz/cmAWPT9ljgppx4zMysveUmt0159/ZjlH3nzrq/B3AksJekh9Ln\nU5I+K2kusDtwi6TbACJiBnAN8CjwO+D4iFgCIGlC2WsD5wOflFSiuII8PzMeMzNrY7nP3O6nSE7/\nU1Y2mvzRkvcAyw2JTG6oUuc84Lweyo8t234J2DsnBjMzGzhyk9sJwO2SjgHWTVdYI4F9WxaZmZlZ\nnXJfBXgsjZY8EJgMzAEmR8TfWhmcmZlZPXJHSw4D/h4R15SVbShpaEQ817LozMzM6pA7oORGiuH4\n5TanyvMyMzOzvpSb3EZGxMPlBWl/u+aHZGZm1pjc5PaCpG3KC9L+S80PyczMrDG5ye0yYKKkAyWN\nknQQxUTKE1oXmpmZWX1yXwU4H1gM/IhiWqxnKWYcyZltxMzMbKXKfRXgHeCH6WNmZtavZd2WlDRO\n0ocqynaV9O3WhGVmZla/3GduJ1LM81juUeCk5oZjZmbWuNzkthbFM7dybwHvaW44ZmZmjctNbvcD\nX68oOw54oLnhmJmZNS53tOS3gCmSjgSepFiFezPgk60KzMzMrF65oyVnSBpJMXHycOB6PHGymZn1\nU7lXbqREdnULYzEzM2uK3FUB7qbKqtsR8bGM+sMpFjodnNr5RUSMl7QR8FtgBPA0cHhELJL0BeCU\nsib+CfjniHioot2zgS8DL6Si0yPi1py/yczM2lfulVvlNFubAccAv86s/zZwckQ8IGl94H5JU4Av\nAVMj4nxJ44BxwKkRcSVwJYCkHYEbKxNbmQsj4keZcZiZ2QCQ+8ztisoySROBy4FzMurPB+an7dck\nzQSGAQcDe6bTrgDuBE6tqD4G3w41M7MVoIge7zb2XlEaBDwfER0rWG8EMA3YAXg2IjZI5QIWde+X\nnf8kcHBEPNJDW2cDRwFdwHSKq8NF3ce7urqW/nGlUmlFwjQzs36ss7Nz6XZHR4cqj2clN0lHVxSt\nAxwCLI6I/XKDkbQecBdwXkRcL+mV8mQmaVFEbFi2/y/AhIjYsUp7g4EXKZ7jfQ8YEhFLYy1Pbo0q\nlUrLdKa9y31Tm/unNvdPbe6f3vWU3HKfuR1Zsf868Efgwtwfl7QmMBG4MiKuT8ULJA2JiPmShgAL\nK6qNBq6q1mZELChr/xJgcm48ZmbWvnKfuX2ikR9JtxwvBWZGRPkyOZOAsRRL6owFbiqrsxpwOPDR\nGu0OSc/zAD4LLHfr0szMBp7cVwFGAS9FxIJ0a/EU4B3ghxHx94wm9qC4+ntYUveox9Mpkto1ko4B\nnqFIZt0+BsyJiKcqYpkAXBQR04EfSNqZ4rbk08BXc/4eMzNrb7m3Ja+iSDwLKBYs3Rb4B3Axy9+y\nXE5E3AMsd0802btKnTuB3XooP7Zsu9ffNjOzgSc3uY2IiMfT7cVDgFHAG8DslkVmZmZWp9zk9o/0\n8vUoiuH7L0paAy95Y2Zm/VBucvsNcAewPvDTVPbP+MrNzMz6odzRkt+StC/Fe22/T8XvUCyFY2Zm\n1q+syKoAt1fsT29+OGZmZo3LXYnbzMxsleHkZmZmbScruaXZQszMzFYJvSYtSasDr0taeyXEY2Zm\n1rBek1tELAGeADZufThmZmaNyx0teSUwWdJ4YC7FXI4ARMQdrQjMzMysXrnJ7Wvp++yK8gC2blo0\nZmZmTZD7EvdWrQ7EzMysWbJf4k5zSX4YGEZxa/LeiHi7VYGZmZnVK3c9t+2Am4FBwBxgOMVkygdF\nxMwWxmdmZrbCct9f+xnwC2B4ROweEZsDF6VyMzOzfiU3ue0MXBARUVb241TeK0nDJf1e0qOSZkg6\nMZVvJGmKpFL63jCVj5D0hqSH0ueiKu32WN/MzAa23OT2HPDxirKPpvIcbwMnR8QoitW1j5c0ChgH\nTI2ITmBq2u/2ZETsnD7HVWm3Vn0zMxugcgeUnA5MkjQZeAbYEjgAOCKnckTMB+an7dckzaQYmHIw\nsGc67QrgTuDUzJhoQn0zM2tDWvZOY40TpZHA4cBQiiu2ayLiiRX+QWkEMA3YgWJV7w1SuYBFEbFB\nOmcGUAK6gDMi4u4e2nqlp/rdx7u6upb+caVSaUVDNTOzfqqzs3PpdkdHhyqPV71yk/RcRAxN25dF\nxNHAuY0EI2k9YCJwUkS8WuSjQkSEpO5kNB/YIiJekvQB4EZJ20fEq9Xarqi/nPKOqEepVGq4jXbl\nvqnN/VOb+6c29099aj1zW1NS93ySn2v0hyStSZHYroyI61PxAklD0vEhwEKAiHgzIl5K2/cDTwIj\ne2i2x/pmZjaw1UpuFwNzJD0LrCPp2Z4+OT+SbhleCsyMiAvKDk0CxqbtscBN6fxN0moESNoa6ASe\n6qHpHuubmdnAVvW2ZEScIeliisEjtwNHNvA7e6T6D0t6KJWdDpwPXCPpGIqBKoenYx8DzpG0GHgH\nOC4iXgaQNAG4KCKm16hvZmYDWM3RkhExh+Lq7aCIuKveH4mIe4DlHvgle/dw/kSKW5g9tXVs2fZL\nPdU3M7OBLes9t4iY2upAzMzMmiX3JW4zM7NVhpObmZm1HSc3MzNrO7Ve4j4np4GIOKt54ZiZmTWu\n1mjJ4WXb7wEOBf5CMeR+C2BXqoxoNDMz60u13nM7qntb0tXAmDREv7vsEOCw1oZnZma24nKfuf0r\ncGNF2STgU80Nx8zMrHG5yW0WcHxF2dco5nw0MzPrV3LXczsWuEHSt4F5FGuxvQ0c0qrAzMzM6pWV\n3CLiQUmdFKtoD6VYkubeiFjcyuDMzMzqUdd7bhExDVhL0rpNjsfMzKxhWclN0o7AE8AlFEvXAHwc\nuKxFcZmZmdUt98rt58BZEbEd0H0r8i7gIy2JyszMrAG5yW174NdpOwAi4nVgUCuCMjMza0Rucnsa\n+EB5gaRdKV4R6JWk4ZJ+L+lRSTMknZjKN5I0RVIpfW+Yyj8p6X5JD6fvvaq0e7akeZIeSh+/d2dm\nZtnJ7UzgFknfpRhIchpwLXBGZv23gZMjYhTFiMvjJY0CxgFTI6ITmJr2AV4EDoqIHYGxwK9qtH1h\nROycPrdmxmNmZm0sd7HSycD+wCYUz9q2BA6JiNsz68+PiAfS9mvATIp35Q4GrkinXQF8Jp3zYEQ8\nl8pnAIMkrZ31F5mZ2YDX63tuklanGBX5lYj4eqM/KGkEsAtwHzA4IuanQ88Dg3uocijwQES8WaXJ\nb0r6IjCd4upwUaMxmpnZqk0R0ftJ0nxgi0Zf2pa0HsWV33kRcb2kVyJig7LjiyJiw7L97SnmsNw3\nIpab6kvSYIpbmAF8DxgSEUd3H+/q6lr6x5VKpUZCNzOzfqSzs3PpdkdHhyqP5ya3bwMbAN+pN8FJ\nWhOYDNwWEReksseBPSNivqQhwJ0RsW06tjlwB3BURPwho/0RwOSI2KG7rDy5NapUKi3TmfYu901t\n7p/a3D+1uX9611Nyyx1Q8k3gFOA1SXMkPdv9yaksSRQvf8/sTmzJJIoBI6Tvm9L5GwC3AONqJbaU\nELt9Fngk8+8xM7M2ljtx8hEN/s4ewJHAw5IeSmWnA+cD10g6hmIR1MPTsW8A2wBnSepe6XvfiFgo\naQJwUURMB34gaWeK25JPA19tME4zM2sDuRMn39XIj0TEPcByl43J3j2cfy5wbpW2ji3bPrKRuMzM\nrD1VTW6S/m9EnJe2z6l2XkScVe2YmZlZX6h15bZ52fbwVgdiZmbWLFWTW0R8rWz7qJUTjpmZWeOy\nnrlJ2rrasYh4qnnhmJmZNS53tOQsihGJ5YNCut8hW72pEZmZmTUod7TkMu/DSdoM+A5wdyuCMjMz\na0TuS9zLiIjngZOA7zc3HDMzs8bVldySbYF1mhWImZlZs+QOKLmbd5+xQZHUtgeqvv9mZmbWV3IH\nlEyo2H8d+GtEeKp9MzPrd3IHlFzR+1lmZmb9Q+5tyazbj56Ky8zM+oPc25KdFCti/4Vi9v4tgF2B\nicA/0jlNWzvNzMysEbnJTcCYiJi4tEA6BDjMU3OZmVl/k/sqwL8CN1aUTQI+1dxwzMzMGpeb3GYB\nx1eUfQ14srnhmJmZNS43uR0L/LukuZLukzQXODmV90rScEm/l/SopBmSTkzlG0maIqmUvjcsq3Oa\npFmSHpe0X5V2q9Y3M7OBKyu5RcSDFINKxgAXAJ8HOiPigczfeRs4OSJGAbsBx0saBYwDpkZEJzA1\n7ZOOjaZ4UXx/4GeSepqgucf6ZmY2sCkib5CjpDUpEtPQiPitpHUBIuL1Ff5R6Sbgp+mzZ0TMlzQE\nuDMitpV0Wmr7++n824CzI+LeinYe76l+9/Gurq6mjeDc4PJ5zWrKzGzAe+WoYU1rq6OjQ5VlWVdu\nknYEngAuAS5NxR8HLlvRICSNAHYB7gMGR8T8dOh5YHDaHgbMKas2N5VVqlbfzMwGsNxXAX4OnBUR\nv5K0KJXdRZHssklaj+LduJMi4lXp3WQbESGp7iut3uqXSo3OFOY5os3MmqXRfyd3dnbWPJ6b3LYH\nfp22A4rbkZIG5QaSbmtOBK6MiOtT8QJJQ8puKy5M5fOA4WXVN09llarVX05vHdGre3xb0sysWRr+\nd3IvcpPb08AHgOndBZJ2pXhFoFcqLtEuBWZGxAVlhyYBY4Hz0/dNZeW/kXQBMJRiMMufe2i6Wv2m\n+8tH/t7y/zFWVaVSyX1Tg/unNvdPbe6f+uS+CnAmcIuk7wJrpQEf1wJnZNbfAzgS2EvSQ+nzKYqk\n9ElJJWCftE9EzACuAR4FfgccHxFLACRNkPTB1G6P9c3MbGDLXRVgsqT9gS9TPGvbEjgkIu7PrH8P\nxRRePdm7Sp3zgPN6KD+2bPulavXNzGzg6jW5pffLLgO+EhFfb31IZmZmjen1tmS6Hbgv8E7rwzEz\nM2tc7jO3C4HvphGPZmZm/VruaMlvAptRzC/5AmVrt0XEFq0IzMzMrF65ye2IlkZhZmbWRLmjJe9q\ndSBmZmbNkvvMzczMbJXh5GZmZm3Hyc3MzNpOXclN0iBJazc7GDMzs2bIXc/tR2miZCQdALwMLJJ0\nUCuDMzMzq0fuldsXgEfS9lkUrwZ8Gvh/rQjKzMysEbnvua0TEX+XtDGwdURMBJC0ZetCMzMzq09u\ncntC0heAbYApAJLeB7zRqsDMzMzqlZvcvg6MBxYDR6ey/YDbWxGUmZlZI3JnKPkL8OGKsiuBK1sR\nlJmZWSOyXwWQtJekSyTdkr6zFwmVdJmkhZIeKSvbSdK9kh6WdLOk96byL5St1v2QpHck7dxDm2dL\nmlexsreZmVn2qwAnA1dTvAJwC/AS8JtUnuOXwP4VZROAcRGxI3ADcAoUV4QRsXNE7AwcCcyOiIeq\ntHth97kRcWtmLGZm1uZyn7n9O7BXRJRfef2KYnDJf/ZWOSKmSRpRUTwSmJa2pwC3AWdWnDOGIqma\nmZlly01uALMq9p+ibF23OswADgZuBA4Dhvdwzr+lc6r5pqQvAtOBkyNiUbUTS6VSA6E2r4125b6p\nzf1Tm/unNvfP8jo7O2seV0Tv+UnSl4E9gbOBuRSJ6EzgLuCy7vMi4p0abYwAJkfEDml/O+C/gI2B\nScAJEbFx2fn/AkxIty17am8w8CJFgv0eMCQiji4/p6urq5Hku4xSqdRrZw5U7pva3D+1uX9qc//0\nrqOjQ5VluVduF6fvMRTJpLuhL6RjSuWr5wYTEY8B+wJIGgkcUHHKaOCqGvUXdG9LugSYnPvbZmbW\n3nKT21bN/mFJm0bEQkmrAWcAF5UdWw04HPhojfpDImJ+2v0s704PZmZmA1zue27PwNKkM7gsqWSR\ndBXFbc33SZoLfAdYT9Lx6ZTrgcvLqnwMmBMRT1W0MwG4KCKmAz9IrwgE8DTw1RWJyczM2ldWcpO0\nAfAz4HMUs5SsK+nTwK4RcUZv9SNiTJVD46ucfyewWw/lx5ZtH9l75GZmNhDlvsR9EdAFbAm8lcru\npRjNaGZm1q/kPnPbGxgaEYslBUBEvCBp09aFZmZmVp/cK7cu4H3lBZK2AFbo2ZuZmdnKkJvcJgAT\nJX0CWE3S7sAVlI1wNDMz6y9yb0v+B8Xabf8NrEnx4vbFVBkQYmZm1pdyk9vgiBhPRTKTtBnwfNOj\nMjMza0DubcknqpQ/2qxAzMzMmiU3uS03b1daf63qXJJmZmZ9peZtSUlzKGYAGSTp2YrDG1Nj7kcz\nM7O+0tsztyMortpupVg4tFsACyLi8VYFZmZmVq+ayS0i7gKQ9L6I+PvKCcnMzKwxuc/cjkuTFCNp\nN0nPSpot6cMtjM3MzKwuucntW8DstP194ALgXODCVgRlZmbWiNz33DoiokvS+sBOwD4RsUTSf7Yw\nNjMzs7rkJrc56Rbk9sC0lNjeCyxpXWhmZmb1yb0teQpwHfB/ge+lsgOBP+dUlnSZpIWSHikr20nS\nvZIelnRzSpZIGiHpDUkPpU+P81dK2kjSFEml9L1h5t9iZmZtLiu5RcStETE0IkZExP2p+Frg05m/\n80tg/4qyCcC4iNgRuIEigXZ7MiJ2Tp/jqrQ5DpgaEZ3A1LRvZmaWfeW2nIhYHBGLM8+dBrxcUTwS\nmJa2pwCHrmAIB1OsTED6/swK1jczszZVd3JrghkUCQrgMGB42bGt0i3JuyR9tEr9wRHRvZ7c88Dg\nFsVpZmarGEXEyvkhaQQwOSJ2SPvbAf9FMY3XJOCEiNhY0trAehHxkqQPADcC20fEqxXtvRIRG5Tt\nL4qIZZ67dXV1Lf3jSqVSa/4wMzNb6To7O5dud3R0LDf/cdXRkpJ+GBGnpO29IuKOZgYWEY8B+6b2\nRwIHpPI3gTfT9v2SnqS4hTm9ookFkoZExHxJQ4CFtX6vvCPqUSqVGm6jXblvanP/1Ob+qc39U59a\ntyW/UrZ9Y7N/WNKm6Xs14AzSqt6SNpG0etreGugEnuqhiUnA2LQ9Frip2TGamdmqqdZ7bn+VdB3F\nmm1rSzqnp5Mi4qzefkTSVcCewPskzQW+A6wn6fh0yvXA5Wn7Y8A5khZTLKlzXES8nNqZAFwUEdOB\n84FrJB0DPAMc3lscZmY2MNRKbp+juHrbkmJlgOE9nJP1wC4ixlQ5NL6yICImAhOrtHNs2fZLwN45\nv29mZgNL1eQWEQsp5o9E0hoRcdRKi8rMzKwBWdNvRcRRaQaQg4BhwDyKkY+V766ZmZn1uaz33CTt\nDjwJHAf8E/BVYFYqNzMz61dyJ07+MfD1iLi6u0DSv1G8p/ahVgRmZmZWr9wZSkYC11SUXQds09xw\nzMzMGpeb3ErA6IqywyhuVZqZmfUrubclTwImSzqB4p2yERQvVx/YorjMzMzqljta8o+S3k8xRdZQ\n4GbgVo+WNDOz/ij3yo2IWAT8uoWxmJmZNUVfLnljZmbWEk5uZmbWdnJf4nYSNDOzVUavSSstP/N6\nWkTUzMys3+s1uUXEEuAJihWzzczM+r3c0ZJXUrznNh6YS9lSN81eodvMzKxRucnta+n77IryALZu\nWjRmZmZNkDVQJCK2qvLJSmySLpO0UNIjZWU7SbpX0sOSbpb03lT+SUn3p/L7Je1Vpc2zJc2T9FD6\nfConFjMza3/ZoyAlrSnpo2k1ACStK2ndzOq/BPavKJsAjIuIHYEbgFNS+YvAQal8LPCrGu1eGBE7\np8+tuX+LmZm1t9xXAXakGFRyCXBpKv44cFlO/YiYBlRO1TUSmJa2pwCHpnMfjIjnUvkMYJBHapqZ\n2YrIvXL7OXBWRGwHLE5ldwEfaeC3ZwAHp+3DgOE9nHMo8EBEvFmljW9K+t9023PDBmIxM7M2oojo\n/SRpEbBRRISklyNio1S+dDujjRHA5IjYIe1vR7HY6cbAJOCEiNi47PztU/m+EbHc0jqSBlPcwgzg\ne8CQiDi6/Jyurq6lf1ypVMoJ08zMVgGdnZ1Ltzs6OlR5PHe05NPAB4Dp3QWSdgVm1RtYRDwG7Jva\nGkmx4kB325tTPIf7Yk+JLdVfUHb+JcDkWr9X3hH1KJVKDbfRrtw3tbl/anP/1Ob+qU/ubckzgVsk\nfRdYS9JpwLXAGfX+sKRN0/dqqZ2L0v4GwC0Ug03+UKP+kLLdzwKPVDvXzMwGltxXASZTjHbchOJZ\n25bAIRFxe059SVcB9wLbSpor6RhgjKQngMeA54DL0+nfALYBziob5t+dCCdI+mA67wfpdYH/BT4B\nfCsnFjMza38rsp7bg8DX6/mRiBhT5dD4Hs49Fzi3SjvHlm0fWU8sZmbW/nJfBVhL0jmSSpJeT9/f\nk/SeVgdoZma2onKv3H4ObAucADxDcVvydGAYcHSNemZmZitdbnL7DPD+iHgl7T8q6T6K0ZJObmZm\n1q/kjpZ8HlinomwQML+54ZiZmTWu6pVbxYTFvwJ+J+knFEveDAeOB/6nteGZmZmtuFq3JS/toez0\niv2vAv/RvHDMzMwaVzW5RcRWKzMQMzOzZsle8sbMzGxVkfue206S7pD0sqS30mexpLdaHaCZmdmK\nyn0V4CpgIsV7bm+0LhwzM7PG5Sa3zSjWc+t9fRwzM7M+lvvM7Qrg860MxMzMrFlyr9zOB+6VdDqw\noPxAROzVcxUzM7O+kZvcrgNmUywg6mduZmbWr+Umt52BjSPCoyPNzKzfy33mdjcwqpWBmJmZNUvu\nldts4HZJN7D8M7ezeqss6TLgQGBhROyQynYCLgLWA54GvhARr6ZjpwHHAEuAEyLith7a3Aj4LTAi\n1T88IhZl/j1mZtbGcq/c1gFuAdaimDS5/JPjl8D+FWUTgHERsSPFs7xTACSNAkYD26c6P5O0eg9t\njgOmRkQnMDXtm5mZ5V25RcRRjfxIREyTNKKieCQwLW1PAW4DzgQOBq6OiDeB2ZJmAbsC91bUPxjY\nM21fAdwJnNpInGZm1h6ykpukrasdi4in6vztGRQJ6kbgMN69ChwG/KnsvLmprNLgiOheT+55YHCt\nHyuVSnWG2dw22pX7pjb3T23un9rcP8vr7OyseTz3mdssIACVlXXPVtLTLcMcRwP/JelMYBJQ90jM\niAhJNWdP6a0jelMqlRpuo125b2pz/9Tm/qnN/VOf3NuSyzybk7QZ8B2KUZR1iYjHgH1TeyOBA9Kh\neSz7LG/zVFZpgaQhETFf0hBgYb2xmJlZe6lryZuIeB44Cfh+vT8sadP0vRpwBsXISSiu4kZLWlvS\nVkAn8OcempgEjE3bY4Gb6o3FzMzaSyPruW1LMYqyV5KuohgQsq2kuZKOAcZIegJ4DHgOuBwgImYA\n1wCPAr8Djo+IJamdCZI+mJo9H/ikpBKwT9o3MzPLHlByN+8+Y4MiqW0PnJNTPyLGVDk0vsr55wHn\n9VB+bNn2S8DeOb9vZmYDS+6AkgkV+68Df40ID+ExM7N+J3dAyRWtDsTMzKxZcm9LrgV8iWIC5fXK\nj0XEF5sflpmZWf2Us7h2GhCyE3Az8PfyYxHx3daE1riuri6vHG5m1uY6OjpUWZb7zG1/YKuIeKW5\nIZmZmTVf7qsAzwJrtzIQMzOzZsm9LXkyxfyP41l+yZs7WhOamZlZfXKT2+wqhyIiqk6qbGZm1hey\nbktGxFYmUK42AAAF/0lEQVRVPm2f2CTtL+lxSbMkDcg14yQNl/R7SY9KmiHpxFS+kaQpkkrpe8Oy\nOqelPntc0n59F/3KIWl1SQ9Kmpz23TdlJG0g6TpJj0maKWl391FB0rfSP1ePSLpK0nvcN41rZPqt\ntpcWSf1v4F+BURRTho3q26j6xNvAyRExCtgNOD71Q48Lxq7AgrPt5ERgZtm++2ZZ44HfRcR2FCOv\nZ+I+QtIw4ATggxGxA8UqK6Nx3zTMya22XYFZEfFURLwFXE2xBt2AEhHzI+KBtP0axb+YhlH0RfcL\n/lcAn0nbSxecjYjZFEsm7bpyo155JG1OsapF+Uw+7ptEUgfwMeBSgIh4K428dh8V1gAGSVqDYmrD\n53DfNMzJrbZhwJyy/WoLpw4YaUX1XYD7qL5g7EDrtx8D3wbeKStz37xrK+AF4PJ063aCpHVxHxER\n84AfUYxInw90RcTtuG8a5uRm2SStB0wEToqIV8uPRTEyacC9NC/pQGBhRNxf7ZyB2jdl1gD+Gfh5\nROxCMTftMs+vB2ofpWdpB1P8B8BQYF1JR5SfM1D7plFObrXlLpza9iStSZHYroyI61PxgrRQLBUL\nxg6kftsD+LSkpyluW+8l6de4b8rNBeZGxH1p/zqKZOc+Kpbrmh0RL0TEYuB64MO4bxrm5FbbX4BO\nSVul+TVHUyySOqBIEsXzkpkRcUHZoWoLxuYuOLvKi4jTImLziBhB8f+POyLiCNw3S6XFjedI2jYV\n7U2xXqP7qLgduZukddI/Z3tTPNN23zQod/qtASki3pb0DeA2ilFMl6XFVAeaPYAjgYclPZTKTqdY\nIPaatPjsM8DhUCw4K6l7wdm3KVtwdgBx3yzrm8CV6T8SnwKOoviP6wHdRxFxn6TrgAco/tYHgV9Q\nTFA/oPumUVkvcZuZma1KfFvSzMzajpObmZm1HSc3MzNrO05uZmbWdpzczMys7Ti5mfVzacb4PZvc\n5i8lndvMNs36E7/nZtbPRcT2fR2D2arGV25mZtZ2nNzMVhJJQyVNlPSCpNmSTkjlZ6eFPH8r6TVJ\nD0jaqaze05L2Sdu7Spou6VVJCyRdUHbep9MtzFck3Snp/5Qd2yW1+5qk3wLvqYjtQEkPpbp/lPRP\nZcdOlTQv1X1c0t4t7CazpnByM1sJJK0G3Az8lWKJkr2Bk8pWUj4YuBbYCPgNcGOarLrSeGB8RLwX\neD9wTWp/JHAVcBKwCXArcLOktdKUVzcCv0rtXwscWhbbLsBlwFeBjYGLgUlp/sJtgW8AH4qI9YH9\ngKeb0SdmreTkZrZyfAjYJCLOSYt1PgVcQjHZMsD9EXFdmhn+Aoorq916aGcxsI2k90XE3yLiT6n8\n34BbImJKauNHwCCKGeZ3A9YEfhwRiyPiOopJwbt9Bbg4Iu6LiCURcQXwZqq3BFgbGCVpzYh4OiKe\nbFqvmLWIk5vZyrElMDTd9ntF0isUk093L0K5dAHKiHiHYpmYoT20cwwwEnhM0l/SenKkc5+paGMO\nxVXiUGBeLDuR7DNl21sCJ1fENhwYGhGzKK4GzwYWSrpaUk9xmfUrTm5mK8ccinW7Nij7rB8Rn0rH\nl67RlW5hbg48V9lIRJQiYgywKfAfwHVpVevnKJJUdxtKbc6jWOF5WCrrtkVFbOdVxLZORFyVfvM3\nEfGR1H6k3zXr15zczFaOPwOvpcEZgyStLmkHSR9Kxz8g6RBJa1BcKb0J/KmyEUlHSNokXZm9korf\noXj2doCkvdOzupNTG38E7qVYHuUESWtKOgTYtazZS4DjJP2LCutKOkDS+pK2lbSXpLWBfwBvpN8z\n69ec3MxWgrTm1oHAzsBs4EVgAtCRTrmJ4rnZIoq18w5Jz84q7Q/MkPQ3isEloyPijYh4HDgC+Elq\n+yDgoPR87y3gEOBLwMvpd7pXUycipgNfBn6afn9WOheK523npzafp7hiPK2x3jBrPa/nZtbHJJ0N\nbJNW8DazJvCVm5mZtR0nNzMzazu+LWlmZm3HV25mZtZ2nNzMzKztOLmZmVnbcXIzM7O24+RmZmZt\nx8nNzMzazv8Hm8hFczTj5igAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x124aff490>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(moving_average(n_steps, 50))\n",
    "plt.ylabel(\"number of steps required for success\")\n",
    "plt.xlabel(\"episodes\")\n",
    "plt.title(\"RL\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}