kuka_rl_2_parallel.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Kuka\n",
    "\n",
    "---\n",
    "\n",
    "You are welcome to use this coding environment to train your agent for the project.  Follow the instructions below to get started!\n",
    "\n",
    "### Start the Environment\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make sure that you're in the right virtual environment and the right python version."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pybullet in /home/monster/anaconda3/envs/python36/lib/python3.6/site-packages (2.8.1)\n",
      "Requirement already satisfied: tensorboardX in /home/monster/anaconda3/envs/python36/lib/python3.6/site-packages (2.0)\n",
      "Requirement already satisfied: six in /home/monster/anaconda3/envs/python36/lib/python3.6/site-packages (from tensorboardX) (1.13.0)\n",
      "Requirement already satisfied: protobuf>=3.8.0 in /home/monster/anaconda3/envs/python36/lib/python3.6/site-packages (from tensorboardX) (3.11.2)\n",
      "Requirement already satisfied: numpy in /home/monster/anaconda3/envs/python36/lib/python3.6/site-packages (from tensorboardX) (1.17.4)\n",
      "Requirement already satisfied: setuptools in /home/monster/anaconda3/envs/python36/lib/python3.6/site-packages (from protobuf>=3.8.0->tensorboardX) (42.0.2.post20191203)\n"
     ]
    }
   ],
   "source": [
    "!python --version\n",
    "!pip install pybullet\n",
    "!pip install tensorboardX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current_dir=/home/monster/anaconda3/envs/python36/lib/python3.6/site-packages/pybullet_envs/bullet\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import sys\n",
    "from collections import deque\n",
    "import timeit\n",
    "from datetime import timedelta\n",
    "from copy import deepcopy\n",
    "import numpy as np\n",
    "import random\n",
    "from PIL import Image\n",
    "from tensorboardX import SummaryWriter\n",
    "\n",
    "import functools\n",
    "import multiprocessing as mp\n",
    "from multiprocessing import Pipe\n",
    "from multiprocessing import Process\n",
    "import signal\n",
    "import warnings\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision.transforms as T\n",
    "import torch.optim as optim\n",
    "\n",
    "from gym import spaces\n",
    "from pybullet_envs.bullet.kuka_diverse_object_gym_env import KukaDiverseObjectEnv  \n",
    "import pybullet as p\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "resize = T.Compose([T.ToPILImage(),\n",
    "                    T.Resize(40, interpolation=Image.CUBIC),\n",
    "                    T.ToTensor()])\n",
    "def worker(remote, env_fn):\n",
    "    # Ignore CTRL+C in the worker process\n",
    "    signal.signal(signal.SIGINT, signal.SIG_IGN)\n",
    "    env = env_fn()\n",
    "    try:\n",
    "        while True:\n",
    "            cmd, data = remote.recv()\n",
    "            if cmd == 'step':\n",
    "                ob, reward, done, info = env.step(data)\n",
    "                remote.send((ob, reward, done, info))\n",
    "            elif cmd == 'get_screen':\n",
    "                screen = env._get_observation()\n",
    "                screen = env._get_observation().transpose((2, 0, 1))\n",
    "                screen = np.ascontiguousarray(screen, dtype=np.float32) / 255\n",
    "                screen = torch.from_numpy(screen)\n",
    "                screen = resize(screen).unsqueeze(0)\n",
    "                remote.send(screen)\n",
    "            elif cmd == 'reset':\n",
    "                ob = env.reset()\n",
    "                remote.send(ob)\n",
    "            elif cmd == 'close':\n",
    "                remote.close()\n",
    "                break\n",
    "            elif cmd == 'get_spaces':\n",
    "                remote.send((env.action_space, env.observation_space))\n",
    "            else:\n",
    "                raise NotImplementedError\n",
    "    finally:\n",
    "        env.close()\n",
    "\n",
    "resize = T.Compose([T.ToPILImage(),\n",
    "                    T.Resize(40, interpolation=Image.CUBIC),\n",
    "                    T.ToTensor()])\n",
    "class MultiprocessVectorEnv:\n",
    "    def __init__(self, env_fns):\n",
    "        nenvs = len(env_fns)\n",
    "        self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)])\n",
    "        self.ps = \\\n",
    "            [Process(target=worker, args=(work_remote, env_fn))\n",
    "             for (work_remote, env_fn) in zip(self.work_remotes, env_fns)]\n",
    "        for p in self.ps:\n",
    "            p.start()\n",
    "        self.last_obs = [None] * self.num_envs\n",
    "        self.remotes[0].send(('get_spaces', None))\n",
    "        self.action_space, self.observation_space = self.remotes[0].recv()\n",
    "        self.closed = False\n",
    "\n",
    "    def __del__(self):\n",
    "        if not self.closed:\n",
    "            self.close()\n",
    "\n",
    "\n",
    "    def step(self, actions):\n",
    "        self._assert_not_closed()\n",
    "        for remote, action in zip(self.remotes, actions):\n",
    "            remote.send(('step', action))\n",
    "        results = [remote.recv() for remote in self.remotes]\n",
    "        self.last_obs, rews, dones, infos = zip(*results)\n",
    "        return self.last_obs, rews, dones, infos\n",
    "    \n",
    "    def get_screen(self):\n",
    "        for remote in self.remotes:\n",
    "            remote.send(('get_screen', None))\n",
    "        results = [remote.recv() for remote in self.remotes]\n",
    "        screens = torch.cat(results,dim=0)\n",
    "        return screens\n",
    "\n",
    "    def reset(self, mask=None):\n",
    "        self._assert_not_closed()\n",
    "        if mask is None:\n",
    "            mask = np.zeros(self.num_envs)\n",
    "        for m, remote in zip(mask, self.remotes):\n",
    "            if not m:\n",
    "                remote.send(('reset', None))\n",
    "\n",
    "        obs = [remote.recv() if not m else o for m, remote,\n",
    "               o in zip(mask, self.remotes, self.last_obs)]\n",
    "        self.last_obs = obs\n",
    "        return obs\n",
    "\n",
    "    def close(self):\n",
    "        self._assert_not_closed()\n",
    "        self.closed = True\n",
    "        for remote in self.remotes:\n",
    "            remote.send(('close', None))\n",
    "        for p in self.ps:\n",
    "            p.join()\n",
    "\n",
    "    @property\n",
    "    def num_envs(self):\n",
    "        return len(self.remotes)\n",
    "\n",
    "    def _assert_not_closed(self):\n",
    "        assert not self.closed, \"This env is already closed\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_env(idx, test):\n",
    "    env = KukaDiverseObjectEnv(renders=False, isDiscrete=False, removeHeightHack=False, maxSteps=20)\n",
    "    env.observation_space = spaces.Box(low=0., high=1., shape=(84, 84, 3), dtype=np.float32)\n",
    "    env.action_space = spaces.Box(low=-1, high=1, shape=(5,1))\n",
    "    return env\n",
    "\n",
    "def make_batch_env(test):\n",
    "    return MultiprocessVectorEnv(\n",
    "        [functools.partial(make_env, idx, test)\n",
    "            for idx in range(mp.cpu_count()*2)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "envs = make_batch_env(test=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if gpu is to be used\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Actor-Critic implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "def build_hidden_layer(input_dim, hidden_layers):\n",
    "    \"\"\"Build hidden layer.\n",
    "    Params\n",
    "    ======\n",
    "        input_dim (int): Dimension of hidden layer input\n",
    "        hidden_layers (list(int)): Dimension of hidden layers\n",
    "    \"\"\"\n",
    "    hidden = nn.ModuleList([nn.Linear(input_dim, hidden_layers[0])])\n",
    "    if len(hidden_layers)>1:\n",
    "        layer_sizes = zip(hidden_layers[:-1], hidden_layers[1:])\n",
    "        hidden.extend([nn.Linear(h1, h2) for h1, h2 in layer_sizes])\n",
    "    return hidden\n",
    "\n",
    "class ActorCritic(nn.Module):\n",
    "    def __init__(self,state_size,action_size,shared_layers,\n",
    "                 critic_hidden_layers=[],actor_hidden_layers=[],\n",
    "                 seed=0, init_type=None):\n",
    "        \"\"\"Initialize parameters and build policy.\n",
    "        Params\n",
    "        ======\n",
    "            state_size (int,int,int): Dimension of each state\n",
    "            action_size (int): Dimension of each action\n",
    "            shared_layers (list(int)): Dimension of the shared hidden layers\n",
    "            critic_hidden_layers (list(int)): Dimension of the critic's hidden layers\n",
    "            actor_hidden_layers (list(int)): Dimension of the actor's hidden layers\n",
    "            seed (int): Random seed\n",
    "            init_type (str): Initialization type\n",
    "        \"\"\"\n",
    "        super(ActorCritic, self).__init__()\n",
    "        self.init_type = init_type\n",
    "        self.seed = torch.manual_seed(seed)\n",
    "        self.sigma = nn.Parameter(torch.zeros(action_size))\n",
    "\n",
    "        # Add shared hidden layer\n",
    "        self.conv1 = nn.Conv2d(3, 16, kernel_size=5, stride=2)\n",
    "        self.bn1 = nn.BatchNorm2d(16)\n",
    "        self.conv2 = nn.Conv2d(16, 32, kernel_size=5, stride=2)\n",
    "        self.bn2 = nn.BatchNorm2d(32)\n",
    "        self.conv3 = nn.Conv2d(32, 32, kernel_size=5, stride=2)\n",
    "        self.bn3 = nn.BatchNorm2d(32)\n",
    "\n",
    "        # Number of Linear input connections depends on output of conv2d layers\n",
    "        # and therefore the input image size, so compute it.\n",
    "        def conv2d_size_out(size, kernel_size = 5, stride = 2):\n",
    "            return (size - (kernel_size - 1) - 1) // stride  + 1\n",
    "        convw = conv2d_size_out(conv2d_size_out(conv2d_size_out(state_size[0])))\n",
    "        convh = conv2d_size_out(conv2d_size_out(conv2d_size_out(state_size[1])))\n",
    "        linear_input_size = convh * convw * 32\n",
    "        self.shared_layers = build_hidden_layer(input_dim=linear_input_size,\n",
    "                                                hidden_layers=shared_layers)\n",
    "\n",
    "        # Add critic layers\n",
    "        if critic_hidden_layers:\n",
    "            # Add hidden layers for critic net if critic_hidden_layers is not empty\n",
    "            self.critic_hidden = build_hidden_layer(input_dim=shared_layers[-1],\n",
    "                                                    hidden_layers=critic_hidden_layers)\n",
    "            self.critic = nn.Linear(critic_hidden_layers[-1], 1)\n",
    "        else:\n",
    "            self.critic_hidden = None\n",
    "            self.critic = nn.Linear(shared_layers[-1], 1)\n",
    "\n",
    "        # Add actor layers\n",
    "        if actor_hidden_layers:\n",
    "            # Add hidden layers for actor net if actor_hidden_layers is not empty\n",
    "            self.actor_hidden = build_hidden_layer(input_dim=shared_layers[-1],\n",
    "                                                   hidden_layers=actor_hidden_layers)\n",
    "            self.actor = nn.Linear(actor_hidden_layers[-1], action_size)\n",
    "        else:\n",
    "            self.actor_hidden = None\n",
    "            self.actor = nn.Linear(shared_layers[-1], action_size)\n",
    "\n",
    "        # Apply Tanh() to bound the actions\n",
    "        self.tanh = nn.Tanh()\n",
    "\n",
    "        # Initialize hidden and actor-critic layers\n",
    "        if self.init_type is not None:\n",
    "            self.shared_layers.apply(self._initialize)\n",
    "            self.critic.apply(self._initialize)\n",
    "            self.actor.apply(self._initialize)\n",
    "            if self.critic_hidden is not None:\n",
    "                self.critic_hidden.apply(self._initialize)\n",
    "            if self.actor_hidden is not None:\n",
    "                self.actor_hidden.apply(self._initialize)\n",
    "\n",
    "    def _initialize(self, n):\n",
    "        \"\"\"Initialize network weights.\n",
    "        \"\"\"\n",
    "        if isinstance(n, nn.Linear):\n",
    "            if self.init_type=='xavier-uniform':\n",
    "                nn.init.xavier_uniform_(n.weight.data)\n",
    "            elif self.init_type=='xavier-normal':\n",
    "                nn.init.xavier_normal_(n.weight.data)\n",
    "            elif self.init_type=='kaiming-uniform':\n",
    "                nn.init.kaiming_uniform_(n.weight.data)\n",
    "            elif self.init_type=='kaiming-normal':\n",
    "                nn.init.kaiming_normal_(n.weight.data)\n",
    "            elif self.init_type=='orthogonal':\n",
    "                nn.init.orthogonal_(n.weight.data)\n",
    "            elif self.init_type=='uniform':\n",
    "                nn.init.uniform_(n.weight.data)\n",
    "            elif self.init_type=='normal':\n",
    "                nn.init.normal_(n.weight.data)\n",
    "            else:\n",
    "                raise KeyError('initialization type is not found in the set of existing types')\n",
    "\n",
    "    def forward(self, state):\n",
    "        \"\"\"Build a network that maps state -> (action, value).\"\"\"\n",
    "        def apply_multi_layer(layers,x,f=F.leaky_relu):\n",
    "            for layer in layers:\n",
    "                x = f(layer(x))\n",
    "            return x\n",
    "\n",
    "        state = F.relu(self.bn1(self.conv1(state)))\n",
    "        state = F.relu(self.bn2(self.conv2(state)))\n",
    "        state = F.relu(self.bn3(self.conv3(state)))\n",
    "        state = apply_multi_layer(self.shared_layers,state.view(state.size(0),-1))\n",
    "\n",
    "        v_hid = state\n",
    "        if self.critic_hidden is not None:\n",
    "            v_hid = apply_multi_layer(self.critic_hidden,v_hid)\n",
    "\n",
    "        a_hid = state\n",
    "        if self.actor_hidden is not None:\n",
    "            a_hid = apply_multi_layer(self.actor_hidden,a_hid)\n",
    "\n",
    "        a = self.tanh(self.actor(a_hid))\n",
    "        value = self.critic(v_hid).squeeze(-1)\n",
    "        return a, value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Examine the state and action spaces."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of agents: 32\n",
      "Size of each action: 5\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAEICAYAAACQ6CLfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXmUXXWV77/7jjUPqcwzhACCQBQQFLURpziwwNZW8bUPXXY7PF2tS9uh8a3l8OzVutpxPX222Cg4DziAsxgQBZF5NBESkkCGqkoqVZWab91hvz/uCd5b+/tL3aQqt1I5+7NWVuru+zvn/M6553eH72//vltUFY7jxI/EXHfAcZy5wQe/48QUH/yOE1N88DtOTPHB7zgxxQe/48QUH/zHESLyJhG5ba77cTwhImtFREUkNdd9OdGIzeAXkZ0iMi4iIxX/vjjX/ZprRORiEdl9DPf/URH51rHav3P0xO3d9FJV/d1cd2K+ISIpVS3MdT+OBSfyuU1HbD75D4eIfFlEflTx+FMisknKdIrIz0Vkv4gMRH+vrGj7exH5hIj8Kfo28TMR6RKRb4vIkIjcLSJrK9qriPyLiGwXkT4R+U8Roa+DiJwuIjeJSL+IPCoirz3MObSLyDUi0i0ie6I+Jac5v2YAvwKwvOLb0PLo0/p6EfmWiAwBeJOIPEtE7hCRwegYXxSRTMU+z6zoa6+IXCUiGwFcBeB10b4frKGvSRH5dHRttgN4xTSv3QejfQxH1+iFFfu5SkQej567V0RWVbwG7xSRrQC2TnetRSQb9enJ6Nz+S0Qao+cuFpHdIvI+EdkXndObD9fn4wZVjcU/ADsBvCjwXBOAxwC8CcDzAPQBWBk91wXg1VGbVgA/BPDTim1/D2AbgHUA2gFsjvb1IpS/WX0DwNcr2iuAWwAsALA6avtP0XNvAnBb9HczgF0A3hzt5xlRv84InMNPAHwl2m4xgLsAvK2G87sYwO4p+/oogDyAy1H+gGgEcC6AC6O+rAWwBcB7ovatALoBvA9AQ/T4gop9fesI+vp2AH8FsCq6RrdE1yxFzvm06Botjx6vBbAu+vv9AB6O2giAcwB0VbwGN0X7b5zuWgP4HIAbo/atAH4G4D8qrl8BwMcBpAG8HMAYgM65vuenHRNz3YG6nWh58I8AGKz4988Vz18AoB/AEwCuOMx+NgAYqHj8ewAfrnj8GQC/qnh8KYAHKh4rgI0Vj/8XgE3R32/C3wb/6wD8ccqxvwLgI6RPSwDkADRWxK4AcMt054fw4P/DNNfzPQB+UnGs+wPtPoqKwT9dXwHcDODtFc+9BOHBfwqAfSi/0aanPPcogMsCfVIAl1Q8Dl5rlN84RhG9qUTPPRvAjorrN17Zv6hPF871PT/dv7j95r9cA7/5VfXO6GvmYgA/OBQXkSaU3/k3AuiMwq0iklTVYvS4t2JX4+Rxy5TD7ar4+wkAy0mX1gC4QEQGK2IpAN8MtE0D6BaRQ7FE5XFC53cYKvsIETkVwGcBnIfyN4kUgHujp1cBeLyGfdbS1+Ww14eiqttE5D0ov8GcKSK/AfBeVd1bQ58qj3G4a70I5fO9t6K/AiBZ0faAVusGY7Cv+XGH/+aPEJF3AsgC2AvgAxVPvQ/lr44XqGobgOcf2mQGh1tV8ffq6JhT2QXgVlXtqPjXoqrvCLTNAVhY0bZNVc881OAw5xda1jk1/mWUv46vj67DVfjbNdgF4OQa9zNdX7thr08QVf2Oqj4X5QGsAD5VcZx1h9t0Sp9C17oP5TfwMyuea1fV435wT4cPfjz1qfYJAP8I4I0APiAiG6KnW1F+8QdFZAHKXwVnyvsjIXEVgHcD+D5p83MAp4rIG0UkHf07X0SeNrWhqnYD+C2Az4hIm4gkRGSdiPxdDefXC6BLRNqn6XMrgCEAIyJyOoDKN6GfA1gmIu+JxLFWEbmgYv9rD4ma0/UV5W8l/yIiK0WkE8CHQh0SkdNE5BIRyQKYQPl1KkVP/zeA/yMi66XM2SLSFdhV8FqragnAVwF8TkQWR8ddISIvneZ6HffEbfD/TKrn+X8i5eSRbwH4lKo+qKpbUf5U+2Z0U30eZVGoD8CfAfx6FvpxA8pfmR8A8AsA10xtoKrDKP/efT3Kn9Y9KH+qZQP7/J8AMigLjgMArkd5QB72/FT1rwC+C2B7pOSznyAA8K8A3gBgGOXB8NQbVtTXF6Osb/SgrKC/IHr6h9H/B0TkvsP1NXruqwB+A+BBAPcB+HGgP4iuxSdRfm16UP5J82/Rc59F+Y3ktyi/aV2D8utoqOFafxBlUffP0ezH71D+NjivkUigcOqEiCjKX523zXVfnHgTt09+x3EifPA7Tkzxr/2OE1P8k99xYsqMknyi3O0voJzw8N+q+snDtW9vbdIlXR217pwE+bcUIW0D6fIBQt9+2H5rO36IYNsj+QI2kwyDw/SBXjPyzXAyn6fbj08WbSzH18wkEvZYR3IdQ5ervSljYg3ZtInli3wP+4dLJqaBz8giGT4LcjwnKauTJpYInG+C3mO0qbkVdo8oDkxoTRfyqAd/tBDjSyhP8ewGcLeI3Kiqm0PbLOnqwP/93/9UFUskkrStkJtDxd5cAJBM2tNIpRt4W/I6lgI/fZKkb0nyyqQy9oYDAIXtbzoVmKlTe9NJYJSzS6YlfoOWYPfb2MD7kE7aQVIo2MG7q6fXxADgkZ2DJvbwjgHatrWlycTYGwIAJJP2hCcL/F7YeP5JJva0tQtNrGeQvyn91605E8uB30sjSbvf1z3KcrCAk0tPmlhjhg+/JhLO8mGCbLL6HnnJDRO8IWEmX/ufBWCbqm5X1UkA3wNw2Qz25zhOHZnJ4F+B6vzo3VGsChF5q4jcIyL3HBwem8HhHMeZTY654KeqV6vqeap6Xnur/arnOM7cMBPBbw+qF2CsjGKHRacoF0J+zwGAkN/WTAcIxUulkDhIjif2dzEAKPkdXizZSyZFvj3Tz4pF/luTiV2JwLVhv82z7IcigCQROR7aan9/AsDBUfvNrH9g1MR+fhuXdRYvXGRira1tgX7ZcysW+e/4J57YaWK5gJAo59t1QClybbta6eb4j39oNrE/Pc5f31/ccoeJNeSHadtkmry+IRGPCX68qWl7BJrpjD757wawXkROkrKjy+tRNjxwHGcecNSf/KpaEJF3obwIIwnga6r6l1nrmeM4x5QZzfOr6i8B/HKW+uI4Th3xDD/HiSk++B0nptTVw09hE9k0oPAqrBqcCBRtEZLNqCWuBhdJxlsmzTP06OQCmUQolfg5kENB0vz9NpOy55bOcLV/eGzcxH5404O0bXOTzU777e0P0ba7e2023qnrTzWxl730ZXT77du3m1goZbenp8fE+vr207Z/+tPtJlYocAW+MWHV9g+/4woTa2/ncv+YTfDD+afyKeqmW62+XZzg56BZm1UZUvvrhX/yO05M8cHvODHFB7/jxBQf/I4TU+oq+AmAxJSc12Jg6THrWDLJ27IU1nyei3BUgJJAKjB7b2TZwQHvACHqYEsLNZDF9b+yqaI79vTTtoWSFbvu32zFNoBfmwYiPgFAR5tNbV22ZLGJMWEPANJpm3a8YMEC2vbBBx8wsR07+H6zDVa0bAxc81/fer+JveN/XGpiHR3cdp+Jt+x6A0CJLc8OiXjkFguZaLF4IFvdtD0SYy7/5HecmOKD33Fiig9+x4kpPvgdJ6b44HecmFJftV8EqSmmhfmAE6wqddrkbUkebsgZlRlGKEmtLe/XxphzbTJgMvLYTlt898c38zTckVGbVzrKck0BKMkbbm3mKajMEGRy0jrJAkCRmJKw65XL8e337bNq/ckn80K5qZS9ZqMj1jgEAJ2NSQTML9mMxTdu+IOJvf8tdgYAADLE6VdDUjuJh9R2Nl8QLI/MZgYCbfk8RG34J7/jxBQf/I4TU3zwO05M8cHvODFlpuW6dgIYBlAEUFDV86bZwFRlSQbW0iuROAoBNaWBaHvZJi6AjYzaiiZ79/OqMk0kDfaaG28zsd79tlINAGqlWghUmmHCXKYh8PKQ6xASmphglw4InMwFOUXa5iZ5VZgdT+wwseXLTSkHAEA2a9Ocs0RsA4Curi4TO+ecM/l+SX9/f/tdJrbzCW40/e3Pv9fEcgHxOEeE04ARNJjBc8D0GSVyuLA4ePSFdmdD7X+BqvbNwn4cx6kj/rXfcWLKTAe/AvitiNwrIm9lDarKdQ0F5nEdx6k7M/3a/1xV3SMiiwHcJCJ/VdWqjApVvRrA1QBw2skrj/4HiuM4s8qMPvlVdU/0/z4AP0G5cq/jOPOAo/7kF5FmAAlVHY7+fgmAjx9uG1VFfkq991SKK7yZlFW/01ne3S3bu02sGFBBtz1pa8v/5k+80BAzD2nMWmOJUPpngTgIM3MNgPuJlAIzAyVSQzCd4rMmzFyiEHBMFrXx/gE7E5IKpDM3N1oFf3iE16676DnPNbFT1q2hbYcGDphYMjRjQZT5FJlJ6R8aodv/ZesuEzvrzPW0bevap5nYyJ4ttG2BmYQEvgcXiMFNKjAzUJjyUhzJV+uZfO1fAuAn0cVOAfiOqv56BvtzHKeOzKRW33YA58xiXxzHqSM+1ec4McUHv+PElLqu508kBM2N1YJZ94GDtO2fH7apoq3N3Pn25rs2m9jIOF9zztb5ZwNlsRgsXTYEE5pC9q4snTmUukndggOuscx4luiFAIDJgr1mQ4O2/NTSJUvp9qetW2tiTS3cJbe51ZbLWpXla//v7ba+CP39VuQFgLb2dhqfyv4+ntL9wU9ea2Ife+8baNtzL7VlwO77i03/BoD8QZtOXKD3hxXxgLA4mJrB5Ll/8jtOTPHB7zgxxQe/48QUH/yOE1N88DtOTKmr2n9gcARf/9ntVbGBoTHaduvufSaWTnB1NJO2p5ENuLsqkb9ZfTYAUJJmmUjbPgTKDVICvhAQuhP+3sxmLPJ5m0oM8Lp8i7vaAp2wx1vQYdsWCtzMI8ecmEf4bE73Xqt+L160kLadmBg3sT177QwAAOzttfdNoWD7lc3ydOgde+z2d9z/GG174blnmdj4BJ9lArnFsoFZlySJpwL3zVRDEK/V5zjOtPjgd5yY4oPfcWKKD37HiSl1FfzGJvJ44LFqoSdUVqulkaybD+SlHoHeRtfCsxRYAEiQPbOUWwm8h/L0XN5btg49JA4u6rIprE3NtkwVwP0DikVeIi1LnJRL9OLwjmXS1pth//4e3rbRpveGznds3Ap+G845m7Z9eLMV58bHh0wsTfoKAC3N9r67/pc8ZXfD6Seb2FlXvp+2ffTqD5tYPmDfm0oQ5+rAxZma9nsk2b7+ye84McUHv+PEFB/8jhNTfPA7TkyZVvATka8BeCWAfar69Ci2AMD3AawFsBPAa1WVL5Cu3hcapmTjFUNq2xEIa0Ui4mnA/JIJYJmAMSgVtmgoJDnaeFMjzyzr6rSZdOlAKbMSMdpMJ/g5MONIDVzH8YmcDZLXob+/n27fTtbS5ye5uMgEzkzGim0AsHK1FdYOEp8BAMiQjEYmgk0tG/dUv4q29UTAG+L9n7zGxL7z+Q/QthN5u99AdTJq1pkMSHnpRPV1nG3B71oAG6fEPgRgk6quB7Apeuw4zjxi2sEfFeGY+lZ/GYDror+vA3D5LPfLcZxjzNHO8y9R1UM+Sj0o23hTojJebwWAbCbgLe84Tt2ZseCn5UyW4E8NVb1aVc9T1fNSgcQKx3Hqz9EO/l4RWQYA0f92HaTjOMc1R/u1/0YAVwL4ZPT/DbVuaCtb8S8NNI2WudaCv4MVAgub8wUyM8AWWwNIk5JQDeSnS5r4CQBAF0nDTYTOgZQnI1meAPhMxkSer7EvkhTSsTHuocBShJlanwi4zrLXZ9Xq1bQtT6/lJ9zZucDEBvq52j8xYa9DmpSECzkjI2FnIZKBmYECubZ3P7KVtj399HNNbOyvd9G2rBxaOvAxPXUSYVbX84vIdwHcAeA0EdktIm9BedC/WES2AnhR9NhxnHnEtJ/8qmrNycu8cJb74jhOHfEMP8eJKT74HSem1HU9P8DSOkOpsbW/L7EU4ZAo1dZmRa1MII02S8w6m5psCqoEBCEmFI2NcrEtSUTDRlLvHgAKpGTYvh4+4ZJptOmuHe0dtC0Ti1pbbdqxElEMABqIB8M4WYsPAEPDu2s6FgBMTtr02t79fbTtwEG7dj9DBD9IKK2cBfn5pokK9+mrf0LbvuPvn2diK8duJy2BbNq+ZqlAlbipL4Wv53ccZ1p88DtOTPHB7zgxxQe/48QUH/yOE1PqrPYrilPNJUL5iERgbWngqnxDk1WJmWkHAAgx/sgESjcdHLKlphJJZtDRRLefJOYY3d28zNTSpUtNbP+QVa4BPrvQ3Mz7wNyK+we470pHZ6eJMTOQVKBs2vDQsIkJuV4AN0DpP8AV/C2bHzGxIXIsgCv7bDKmVOL94sYfvC11Nrb56wCAn/7hQRP77Gv+gbbd+7sfmViyhS+KE6k+XuDwFP/kd5yY4oPfcWKKD37HiSk++B0npsxBem+1IrGQuNYCQEMDSaMN7DNJ1sIPDfO68NmsTZnNTxLXWgCd7VYAy+dt2xJJtwWAkZERGwwInAMBR1xGivgMpAOi5cSYXd/eRlx2AeBAnxXcmP/AosWL6fZjROA8sP8AbVsgngS793STlkAL8RlY2GXX+APA0MguExPY+yO4nJ8KxbxxE0m/Hge/l/qG7evwlZs207YbyY0+MbUuV0RySrr8rK7ndxznxMQHv+PEFB/8jhNTfPA7TkypxcPvayKyT0QeqYh9VET2iMgD0b+XH9tuOo4z29Si9l8L4IsAvjEl/jlV/fSRHCyTTmPVcq4U18K+fdywgqXXdnRwRXtk2Crwvb29tO3iJbYWyTBJuc1N9tDtG5usKUMyYDLC1O8jccltbLSKOAAMDpL+9nEFnpEmsyMPPcxV6gwxJEkHCrWwtk9/2im0LXNc3h84hwyZCWGpvBIw8yjkC6Qtn2cayVtjFjYTAwBFYkjyu628BmD7elub8OUN3K04N1ltljKrZh6Bcl2O48xzZvKb/10i8lD0s8BOiDuOc1xztIP/ywDWAdgAoBvAZ0INReStInKPiNyTy/GvOY7j1J+jGvyq2quqRVUtAfgqgGcdpu1TtfqygSw0x3Hqz1Gl94rIsooqva8CYBdcE1TVCCohh9pc3n5LyOdt6SgA6B1mQiCXPvpICmtHB//V0tFhXW5HiftuRwcX27LEf2A0sA59knwrCok3OZJGOz7ORUeWFD02zlNQmSvwgpQ9h85OLqYuWthlYosXLaJthw/a9OuhYX5txkkJrmXLuHA8PmHdgp940noo0LX44Om9Ab0PBXI7loj/QXknJFbi9/P4mmebWC7zGN/vjgeqHuoRLOifdvBH5bouBrBQRHYD+AiAi0VkA8r3504Ab6v5iI7jHBccbbmua45BXxzHqSOe4ec4McUHv+PEFB/8jhNT6mrmUVb7qxXOfCAdsrfXKvgtTVxVLxWtclskMQBoJ3Xq0mnujDpBVOa2dms+MpHj9egGiEtuNmtNSgBAyUxGIaBIT+asWp8jaakAMEniSwNKeTNJk16xwqY4h67t6Oio7ReZmSj3y56vcXaO6Oy0xh1P7t5D2xYK9nyZ+a4GXIUTYlOqQ+fLjE6CWjt5ojHD74Vf33yHie1q5vfYFSuqU8gLATMRhn/yO05M8cHvODHFB7/jxBQf/I4TU+rr3iuATHHa7d3H1ym3NrfYWGsrbdtC4ilStgnga/eTAdGRue+WiD1qYyMXbopkHXpvLz/fbNau/c9NchEvQwTKdetOom0Vtg+dbTw9N0PW3k+QdNnJI+hXKSCBtRHhdfvOnbTtvv12RXljoETaokW27Flrqz3WlkcfpdsreX1DKbPs2qqGhEQSF77fLCmH9vCYHQ8A8JJctchaDByf9qnmlo7jnFD44HecmOKD33Fiig9+x4kpPvgdJ6bUVe0XESST1YpwZ8Bllxk4HCQGEAAwyWrthRRaorAWiyH12irwQ0O2D/v7bBovAKSSxEk2UEytrcMaYSxcwOsYJkm+ahOpZwcASWJOMXRwkLY90GdnIjo6rdHJwCD3cx0ZITMDgdTYLJlZaG3mszlNDfbcGpu42j8yYlOMmRtHWxu/toMD5NqQNF4ASCftOeRL/F5iacMh4w/mFtxAZhYA4Lv7q81S+gu83iHtU80tHcc5ofDB7zgxxQe/48QUH/yOE1NqMfBchXKpriUor0q+WlW/ICILAHwfwFqUTTxfq6pc+frbvpCekt6rRJACeMptX6BE08SEddRdunRZoK0VB7u7ebmuPFkL38DchgMZlU3Ntu0ZZ/KSVEr0nESCX5scEUN3PP44bcvEwVAZMLb2v/+gLfc1McmFqlTSpveytfgA0Npi01VDqbEH+u1tNR6oAZEha+SbiDi4cvlKun2JXNuhgNuxKrkOIaE5wVKBaVPqFsxKtAHAwGR1vEBKk4Wo5ZO/AOB9qnoGgAsBvFNEzgDwIQCbVHU9gE3RY8dx5gm11OrrVtX7or+HAWwBsALAZQCui5pdB+DyY9VJx3FmnyP6zS8iawE8A8CdAJZUFO7oQflnAdvGy3U5znFIzYNfRFoA/AjAe1S16oegltdB0l8wXq7LcY5PasrwE5E0ygP/26r64yjce6hsl4gsA8BqZlVRLBQxOEVASgZqIQ0P20yt0FLldNqKPFse5eWNmLDGBCEA6Oi068CXLV9h27Xy9fysvNhAP8+uayECWMj8cn+fvdTMTBIAcpP2hEM15HN525ZlJK5Yaq8BAKTSVkg80M9F2oMDVkjMBHwRWDZfUwMv88Ze30zanu/K5Wvo9q2t9li33Hob71eDzQANibQMCXgd8FJiAZ+AxFTRsfZyXdP2VMq5htcA2KKqn6146kYAV0Z/XwnghpqP6jjOnFPLJ/9FAN4I4GEROVQV8CoAnwTwAxF5C4AnALz22HTRcZxjQS21+m5DcCYbL5zd7jiOUy88w89xYooPfseJKXVdz5+bzGPnE3urYhJwMGVpkjm2bh98/fOKlTx9M52ybVtbuKdAZ5dd8737ySdNLJu0a/EBoFC06Z8T4zYVGeAzDiFX4YlxO4sQmgnJkJkQYioc7MOyxQtNLJ3hzsgdbXY9/vIldnsA2NtjZywmCvxeSBJfhKCqzdJoxaYtr1zKS5Z1ddpzeOThR2jbwaFhe3jivAsAKZJSHfI6SJAxocGZgdrTec1xjnpLx3HmNT74HSem+OB3nJjig99xYkp9y3VBoVNMC0uhFAKSVrp8xSradEG7NXjsJMaTAJAQksJK0ymBfT09JsZEsb6+Prq9kLX0AY0Hu3bbYyUD6+6ZABZcN5Ek5cUyPJ05SVJTly5eZGLtAfPLdNYKgczEFAASKXushHAhcQ8p6dZatKnXAHD2ogtMrGd4r4mNF7jwetrpp5nYZZdupG1v/MVvTGxg0KYtA0CelQFjucgAlJh9klsJgC3zRsuCBfBPfseJKT74HSem+OB3nJjig99xYooPfseJKXVW+8WYHRQD8jdLZ+wgqj4ALOi06blj4yO0LXOCZaYdAJDL2XTi8TGrEre28vTgnU/uMrHGBm5YwUwgCsQMBADSGds2VXg6bZvb9goTazhjE227bI01GmlosCYj2Swvq5Vi/Urzz5c1q1ebWGOWG3SsWWNTtTN9PKX65MQ5Jra6ZZ2J7Ru0rw0A7Nix1cSWB5x+21rsrMfBIVIuDNypN5QKzCgGXIHHxqvdhkPl4Bj+ye84McUHv+PEFB/8jhNTfPA7TkwRnUYgOEy5ro8C+GcAh3Ivr1LVXx5uXw0NWV29str5VUPpvazMVCB18awzTzWxXM6WXQKAbdu2m9iqlTxteGLS1hkYGrLpm8EUVtLd0Bp9VhqMOckCQBrLTezg5rfQthlSQz6dDr3mVnxNiO3v+DivQX/OxXtM7GkXc1PnwUErpq5fcjJtm9huxcFswK24KWlFQ1bCKwkutpVgPRiSAV38nonfmtiXf/wl2nZ0ZNzEQo7L1KsgZNgwZe1/T08PcrnJmnJ8a1H7D5Xruk9EWgHcKyI3Rc99TlU/XcuBHMc5vqjFwLMbQHf097CIHCrX5TjOPGYm5boA4F0i8pCIfE1E6DK6ynJdRWJr5TjO3DCTcl1fBrAOwAaUvxl8hm1XWa4rtETVcZz6U9PgZ+W6VLVXVYtaXpT8VQDPOnbddBxntpn2N3+oXNehOn3Rw1cB4Ban1ftCKlN9yNBPAeavEUpd7O21Zg/9JI0XADJZq/z27OOKdEPW1mJLJu37ZUOjbQcA60453cSGD/JafcPDNt7eztNouxptamt2tJe23Tlo+9Z7kKdJM1NeLVrhONXC6+91reo2sW/edDdtu2VXv4m97u8O0rbXbvqCib34pIto23995ptNLE9cnx8aeIhuzww2mrJ81qU/bw1YVi1fSttu3faEiSUCDh3JBBuWvG3RDJTazTxmUq7rChHZgPK8xE4Ab6v5qI7jzDkzKdd12Dl9x3GObzzDz3Fiig9+x4kpdV3PXyopcrnqNerMMbaMFfeKBS4O7ttvxaOmJi7CNTVZwS+Xs2m8AJAnXgMXXfR8E1uydAnd/nnPv8TE7rrjj7TtPX++zcQaG/n69kXNVpn78KXLaNu7d1tfg4//ejdtOzBhBcZi0QqnC5u5B8NEnxU4z+jiadbdB+80se/9/h7adkHLAhO7ec99tK0Q59uzO2wKyq7CZrp9W7v1Zhg+wB15O1us18FZTz+btt21y4qDpUAJLpb2m0yHUsirzzdxBNPp/snvODHFB7/jxBQf/I4TU3zwO05M8cHvODGl7rX6SqVqI4hSiacjTlUxASBUhkzIE6H6eyOjTH3mOxaiumbTtl/PuuA5dPuJCWvg0NZhlWsAOO3MDSY2WeTuvYlBu9+BMX6+5yyxqbyXbuQOsz/dZF1qDw4Nm9jeJ/n2+x+zx7ri3WfRtvftu9/ExiYCswgFax5y8WnWkRcAnux/zMQmSTby0mbu2FwqWAW+r8fOJgHA5r5HSbR2c5pMitcmTBPzkde8zNYgBIDTT642ovnAx7mZCO1SzS0dxzmh8MHXr3djAAAMiklEQVTvODHFB7/jxBQf/I4TU+oq+CUSCTQ2VK+NZiWxACCZsu9Lk4HyVSDxfJE7zDY32bXZqYCj7oIFVhTa8TgRlIjLLwAMDtr16X/dzG0PtGCvQz7gddCQsmmliYDQNJm38Y4l/DqOi+1bCymR9opX81JozY322qxew4W1/MNWOM0F0rfPWrHQxF7//GfSttuf3Gtim7dYx+bxSZ52fN8frRBZCLwORSJEhtyqFi2yHgxnrueu0W981cUm1hHwdshOMWdubOQiIsM/+R0npvjgd5yY4oPfcWKKD37HiSm1GHg2APgDgGzU/npV/YiInATgewC6ANwL4I2qypWvQwdLJrGws1qs2t3NhZcCyW4LvVOxbL5EoHUmY9f5ZzL8MkySdf5tSxbZYwU8CfJ5u/3osM2YAwDNW8FPAydcbLDr/EslLkoJuQ5Dg2O07XkvtWvOm7NW8Gts4eLTokX2dSgl+Ovb0WpFy4VtfL9vuOgZJrZnjzULBYCHHrSi5ZbHtppYqEpdgYiOy5ZasQ4AVi6zPg4SWKP/vrdeZmKZwBr9VIpkrAYE7L391W1J1bcgtXzy5wBcoqrnoOzRv1FELgTwKZTLdZ0CYAAALxbnOM5xybSDX8scmttJR/8UwCUAro/i1wG4/Jj00HGcY0KtRTuSkW33PgA3AXgcwKCqHvqSsRuB+n2V5bpYJVrHceaGmgZ/VJlnA4CVKFfmsWZt4W2fKteVDvzGcRyn/hyR2q+qgwBuAfBsAB0iTxVvXwnAFmd3HOe4pRa1fxGAvKoOikgjgBejLPbdAuA1KCv+VwK4Ybp9dXW04E2XP68q9rH/92PeMSJ1J0ipLICX/AqlZA4NWSfWri6u5rK03UyDVb8zpKwXAAhR4MdHeLmuJEkxDgj4OFCwacO/nbiXts0mbbpnr/D16YkGUporba95S3Mb3T5NZiHYjAkAvPOVLzCxx7Zuo21v+v2tJnbf/Q/TtiAzL+wbZ1srL1n2zLNPMbGLzn0abfvsZ5xmYqEUdCXp1xqYcijkyexVgnsdXHt7dX7vAZ55Tanle/gyANdJ2RM5AeAHqvpzEdkM4Hsi8gkA96Ncz89xnHlCLeW6HgJgJlpVdTu8Mq/jzFs8w89xYooPfseJKfWde1OgWKitnnierJXOpPlaZWrWGRBTJoh/wBAxqQSApkYr5I2M23TV++78E91+eIjUmw+4kE7k7H5JqXgAQCFlRbRN4PXmM0l7Do0pLlBmSduGBmsmmQ1M2T76qBXs9nTblGEAKBBh7I67AqJlJmNimaztFwA89zwrzq1ba0uZtQcEv0suPMPEiKcnAGCMiZkB41h2mzOT2nJb2zid5PdNU3O1P0Uo1Zzhn/yOE1N88DtOTPHB7zgxxQe/48QUH/yOE1PqqvanUgksmmLmsfG5Z9O2v/njgyamAaMEFk8m+amxWYQ8MdIAgGLWzi489KB1d+3v2UW3b2m2ijIrLQYAQtTcRIKfAzMqySasIg4AQsTnVIqrzEwp3rXbmmbcevtddPsDAzZ1ub9/gLZNkvJVza3czOPkldZA5dJLzqVtzzztJBNrb7MzAwUz61RmlCj4iUTt1ysEm5EKTCIgm7XXZtNfeR8GJ6rv0aIGyoUR/JPfcWKKD37HiSk++B0npvjgd5yYUlfBT0SQzVQLFKeusamXAHDLXVtMLD8ZKO3FSiQF0nuZ2DU5yddgj45al9s02X5wmLvhjk5Y8UgD6Z/Lly42sYA2iCJxch3PjdO2zCdgfJIbBWwh6bkjo6MmNkZSnAFgUVeniZ180mraFrDX4V1vfAlt2dpkfQI62mwM4D4OoyO2vyL8/qCCbEBEKxXtOYjwz1MhoqEE7tEEyeveNchTsie1ejyVXPBzHGc6fPA7Tkzxwe84McUHv+PElGkHv4g0iMhdIvKgiPxFRD4Wxa8VkR0i8kD0b8Ox767jOLNFLWr/oXJdIyKSBnCbiPwqeu79qnr9YbatQlUxkatW1s9cR2t94IXEVOGnv7uHts0Qjw86AwCgRNJ7Q2mWRVIvMEkchEcD6ndn1qarLuxcQNuyVNFSifcsN2nPYX9PH227e89eE2P1CgFAyWdBY5Nte9opa+j2zz/PlnN4zrnW4RYAcmTmJhNIyU6Q1zKUnsvE+hypmZjJ8nTo5BGk7BbIrEsqdN+RmYHQsUZyzPmDv2bpVPXNH0ofZ9Ri4KkAWLkux3HmMUdVrktV74ye+ncReUhEPici9K2pslzX0AifD3ccp/4cVbkuEXk6gH9DuWzX+QAWAPhgYNunynW1tTSxJo7jzAFHW65ro6p2RxV8cwC+Dvfwd5x5xVGX6xKRZaraLWWF4XIAjxxNB4qBmlSZtP0VkUlx916WqlksBGpdEUEk9A6oJFWySISbQJYmxkh6b6HAS2Uxl9yQsPL4DusfkA44Gzc01e4pcD4pVbVskU3ZfdWLz6PbF8g1Z0IXAKRJGTF2bQFQUwKWtlzeBxHhUkTcC4ipStKOEUjZPRKRNpUi5efAq1bfvNXeC4/u5wJlR3v1dUzMpuCHcLmum6M3BgHwAIC313xUx3HmnJmU67rkmPTIcZy64Bl+jhNTfPA7Tkzxwe84MaWuZh6qatRYBU+HPHW1dWxd0sXdXffut66xxBw2ipP3u4BCWiTGEIwMcfkFgIMHh0yMzSAAPAU1FXCNzTZaI4tTVnNTlOdf8HQTGxvjxh8vuMDWuWtosOc2zmrUAWDl5JIhpZzES4GPIlbTjtZnBDfNYK9vKVAIUdhLHholTNgPTP0U8nbHLY38hJvINU+ludpvZhyOQO33T37HiSk++B0npvjgd5yY4oPfcWJKXQU/QCBTBIpSiQsUZ6xfaWKrV1iHWwDYtc+mzCYDpa6UCjK1r1DO54nTb2BzlrocSsNd0GnTaJsabZonAFzxyufY7dttGi8ArFuz1MQCxrXIkzXyk3kbY+XCAKBIRDQJCnOsPBkXONkFDp0De31L5HVIBBRhJTtOBlLQE8R/oBTK9VabyrttH+/D/busuNfATCsASOA+rwX/5HecmOKD33Fiig9+x4kpPvgdJ6b44HecmFLnWn1AcopCytV3YHTcppD+/UvOp21391q1f0/Pfto2QdRRDai5zPSCGTiEjCUWddkU5ULAsOLVG60R0ulrl9O2na02vTdkIjE2RlJxg6nPNqbkGmhgeoOl8rLaedGO7fEDqc+TRCkPmVYw91z2+miJ12dkKd2lwMwAe9U1kM7cQPpwcILfN72jVtlf0BlyNq6OH4l7r3/yO05M8cHvODHFB7/jxBQf/I4TUyQkuB2Tg4nsB/BE9HAhAF5jan7j5zX/OJHObY2qWqWZUNfBX3VgkXtUlXtAz2P8vOYfJ/K5HQ7/2u84McUHv+PElLkc/FfP4bGPJX5e848T+dyCzNlvfsdx5hb/2u84McUHv+PElLoPfhHZKCKPisg2EflQvY8/m4jI10Rkn4g8UhFbICI3icjW6H/rz3WcIyKrROQWEdksIn8RkXdH8Xl9biLSICJ3iciD0Xl9LIqfJCJ3Rvfk90WEm+SfYNR18EeVfr8E4GUAzgBwhYicUc8+zDLXAtg4JfYhAJtUdT2ATdHj+UYBwPtU9QwAFwJ4Z/Q6zfdzywG4RFXPAbABwEYRuRDApwB8TlVPATAA4C1z2Me6Ue9P/mcB2Kaq21V1EsD3AFxW5z7MGqr6BwBT1xNfBuC66O/rAFxe107NAqrarar3RX8PA9gCYAXm+blpmZHoYTr6pwAuAXB9FJ9353W01HvwrwCwq+Lx7ih2IrFEVbujv3sALJnLzswUEVmLcon2O3ECnJuIJEXkAQD7ANwE4HEAg6pPmQaciPckxQW/Y4iW51Hn7VyqiLQA+BGA96hqVeHB+XpuqlpU1Q0AVqL8TfT0Oe7SnFHvwb8HwKqKxyuj2IlEr4gsA4Do/31z3J+jQkTSKA/8b6vqj6PwCXFuAKCqgwBuAfBsAB0icsgS50S8Jyn1Hvx3A1gfqasZAK8HcGOd+3CsuRHAldHfVwK4YQ77clRI2QvqGgBbVPWzFU/N63MTkUUi0hH93QjgxSjrGbcAeE3UbN6d19FS9ww/EXk5gM8DSAL4mqr+e107MIuIyHcBXIzyktBeAB8B8FMAPwCwGuXly69VVWsyeBwjIs8F8EcADwM4ZDp4Fcq/++ftuYnI2SgLekmUP/h+oKofF5GTURafFwC4H8A/qmpu7npaHzy913Fiigt+jhNTfPA7Tkzxwe84McUHv+PEFB/8jhNTfPA7Tkzxwe84MeX/A8A4j3IRIq4mAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "envs.reset()\n",
    "\n",
    "# number of agents\n",
    "num_agents = envs.num_envs\n",
    "print('Number of agents:', num_agents)\n",
    "\n",
    "init_screen = envs.get_screen().to(device)\n",
    "_, _, screen_height, screen_width = init_screen.shape\n",
    "\n",
    "# size of each action\n",
    "action_size = envs.action_space.shape[0]\n",
    "print('Size of each action:', action_size)\n",
    "\n",
    "plt.figure()\n",
    "plt.imshow(init_screen[0].cpu().squeeze(0).permute(1, 2, 0).numpy(),\n",
    "           interpolation='none')\n",
    "plt.title('Example extracted screen')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "writer = SummaryWriter()\n",
    "i_episode = 0\n",
    "def collect_trajectories(envs, policy, tmax=200, nrand=5):\n",
    "    \n",
    "    global i_episode \n",
    "    global writer\n",
    "    \n",
    "    episode_rewards = 0\n",
    "    \n",
    "    \n",
    "    def to_tensor(x, dtype=np.float32):\n",
    "        return torch.from_numpy(np.array(x).astype(dtype)).to(device)\n",
    "    \n",
    "    #initialize returning lists and start the game!\n",
    "    state_list=[]\n",
    "    reward_list=[]\n",
    "    prob_list=[]\n",
    "    action_list=[]\n",
    "    value_list=[]\n",
    "    done_list=[]\n",
    "\n",
    "    state = envs.reset()\n",
    "\n",
    "    # perform nrand random steps\n",
    "    for _ in range(nrand):\n",
    "        action = np.random.randn(num_agents, action_size)\n",
    "        action = np.clip(action, -1.0, 1.0)\n",
    "        _, reward, done, _   = envs.step(action)\n",
    "        reward = torch.tensor(reward, device=device)\n",
    "        \n",
    "\n",
    "    for t in range(tmax):\n",
    "        states = envs.get_screen().to(device)\n",
    "        action_est, values = policy(states)\n",
    "        sigma = nn.Parameter(torch.zeros(action_size))\n",
    "        dist = torch.distributions.Normal(action_est, F.softplus(sigma).to(device))\n",
    "        actions = dist.sample()\n",
    "        log_probs = dist.log_prob(actions)\n",
    "        log_probs = torch.sum(log_probs, dim=-1).detach()\n",
    "        values = values.detach()\n",
    "        actions = actions.detach()\n",
    "        \n",
    "        env_actions = actions.cpu().numpy()\n",
    "        _, reward, done, _  = envs.step(env_actions)\n",
    "        rewards = to_tensor(reward)\n",
    "        dones = to_tensor(done)\n",
    "\n",
    "        state_list.append(states.unsqueeze(0))\n",
    "        prob_list.append(log_probs.unsqueeze(0))\n",
    "        action_list.append(actions.unsqueeze(0))\n",
    "        reward_list.append(rewards.unsqueeze(0))\n",
    "        value_list.append(values.unsqueeze(0))\n",
    "        done_list.append(dones)\n",
    "\n",
    "        if np.any(dones.cpu().numpy()):\n",
    "            episode_rewards += rewards.sum(dim=0)\n",
    "            i_episode += dones.sum(dim=0)\n",
    "            writer.add_scalar('Episodes average rewards', episode_rewards.item()/dones.sum(dim=0).item(), i_episode.item())\n",
    "            state = envs.reset()\n",
    "            episode_rewards = 0\n",
    "                \n",
    "    state_list = torch.cat(state_list, dim=0)\n",
    "    prob_list = torch.cat(prob_list, dim=0)\n",
    "    action_list = torch.cat(action_list, dim=0)\n",
    "    reward_list = torch.cat(reward_list, dim=0)\n",
    "    value_list = torch.cat(value_list, dim=0)\n",
    "    done_list = torch.cat(done_list, dim=0)\n",
    "    return prob_list, state_list, action_list, reward_list, value_list, done_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_returns(rewards, values, dones):\n",
    "    n_step = len(rewards)\n",
    "    n_agent = len(rewards[0])\n",
    "\n",
    "    # Create empty buffer\n",
    "    GAE = torch.zeros(n_step,n_agent).float().to(device)\n",
    "    returns = torch.zeros(n_step,n_agent).float().to(device)\n",
    "\n",
    "    # Set start values\n",
    "    GAE_current = torch.zeros(n_agent).float().to(device)\n",
    "\n",
    "    TAU = 0.95\n",
    "    discount = 0.99\n",
    "    values_next = values[-1].detach()\n",
    "    returns_current = values[-1].detach()\n",
    "    for irow in reversed(range(n_step)):\n",
    "        values_current = values[irow]\n",
    "        rewards_current = rewards[irow]\n",
    "        gamma = discount * (1. - dones[irow].float())\n",
    "\n",
    "        # Calculate TD Error\n",
    "        td_error = rewards_current + gamma * values_next - values_current\n",
    "        # Update GAE, returns\n",
    "        GAE_current = td_error + gamma * TAU * GAE_current\n",
    "        returns_current = rewards_current + gamma * returns_current\n",
    "        # Set GAE, returns to buffer\n",
    "        GAE[irow] = GAE_current\n",
    "        returns[irow] = returns_current\n",
    "\n",
    "        values_next = values_current\n",
    "\n",
    "    return GAE, returns\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_screen():\n",
    "    # Returned screen requested by gym is 400x600x3, but is sometimes larger\n",
    "    # such as 800x1200x3. Transpose it into torch order (CHW).\n",
    "    #env.render(mode='human')\n",
    "    screen = env._get_observation().transpose((2, 0, 1))\n",
    "    # Convert to float, rescale, convert to torch tensor\n",
    "    # (this doesn't require a copy)\n",
    "    screen = np.ascontiguousarray(screen, dtype=np.float32) / 255\n",
    "    screen = torch.from_numpy(screen)\n",
    "    # Resize, and add a batch dimension (BCHW)\n",
    "    return resize(screen).unsqueeze(0).to(device)\n",
    "def eval_policy(envs, policy, tmax=1000):\n",
    "    reward_list=[]\n",
    "    state = envs.reset()\n",
    "    for t in range(tmax):\n",
    "        states = get_screen()\n",
    "        action_est, values = policy(states)\n",
    "        sigma = nn.Parameter(torch.zeros(action_size))\n",
    "        dist = torch.distributions.Normal(action_est, F.softplus(sigma).to(device))\n",
    "        actions = dist.sample()\n",
    "        _, reward, done, _  = envs.step(actions[0])\n",
    "        dones = done\n",
    "        reward_list.append(np.mean(reward))\n",
    "\n",
    "        # stop if any of the trajectories is done to have retangular lists\n",
    "        if np.any(dones):\n",
    "            break\n",
    "    return reward_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Network Architecture\n",
    "An actor-critic structure with continuous action space is used for this project. The policy consists of 3 parts, a shared hidden layers, actor, and critic.\n",
    "The actor layer outputs the mean value of a normal distribution, from which the agent's action is sampled. The critic layer yields the value function.\n",
    "\n",
    "- Shared layer:\n",
    "```\n",
    "Input State(48,48,3) -> Conv2d(3, 16, 5, 2) -> BatchNorm2d(16) -> Conv2d(16, 32, 5, 2)-> BatchNorm2d(32)\n",
    "-> Conv2d(32, 32, 5, 2) -> BatchNorm2d(32) -> Dense(128) -> LeakyReLU -> Dense(128) -> LeakyReLU -> Dense(64) -> LeakyReLU\n",
    "```\n",
    "- Actor and Critic layers:\n",
    "```\n",
    "LeakyRelu -> Dense(64) -> LeakyRelu -> Dense(4)-> tanh -> Actor's output\n",
    "LeakyReLU -> Dense(64) -> LeakyRelu -> Dense(1) -> Critic's output\n",
    "```\n",
    "\n",
    "### Model update using PPO/GAE\n",
    "The hyperparameters used during training are:\n",
    "\n",
    "Parameter | Value | Description\n",
    "------------ | ------------- | -------------\n",
    "Number of Agents | 1 | Number of agents trained simultaneously\n",
    "tmax | 20 | Maximum number of steps per episode\n",
    "Epochs | 10 | Number of training epoch per batch sampling\n",
    "Batch size | 128 | Size of batch taken from the accumulated  trajectories\n",
    "Discount (gamma) | 0.993 | Discount rate \n",
    "Epsilon | 0.07 | Ratio used to clip r = new_probs/old_probs during training\n",
    "Gradient clip | 10.0 | Maximum gradient norm \n",
    "Beta | 0.01 | Entropy coefficient \n",
    "Tau | 0.95 | tau coefficient in GAE\n",
    "Learning rate | 2e-4 | Learning rate \n",
    "Optimizer | Adam | Optimization method\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run your own policy!\n",
    "policy=ActorCritic(state_size=(screen_height, screen_width),\n",
    "              action_size=action_size,\n",
    "              shared_layers=[128, 64],\n",
    "              critic_hidden_layers=[64],\n",
    "              actor_hidden_layers=[64],\n",
    "              init_type='xavier-uniform',\n",
    "              seed=0).to(device)\n",
    "\n",
    "# we use the adam optimizer with learning rate 2e-4\n",
    "# optim.SGD is also possible\n",
    "optimizer = optim.Adam(policy.parameters(), lr=2e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = 'policy_ppo.pt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best mean reward updated 30.000 -> 32.500, model saved\n",
      "Best mean reward updated 32.500 -> 38.000, model saved\n",
      "Best mean reward updated 38.000 -> 39.000, model saved\n",
      "Best mean reward updated 39.000 -> 39.200, model saved\n",
      "Best mean reward updated 39.200 -> 39.600, model saved\n",
      "Best mean reward updated 39.600 -> 40.182, model saved\n",
      "Best mean reward updated 40.182 -> 40.250, model saved\n",
      "Best mean reward updated 40.250 -> 40.286, model saved\n",
      "Best mean reward updated 40.286 -> 40.733, model saved\n",
      "Best mean reward updated 40.733 -> 41.438, model saved\n",
      "Best mean reward updated 41.438 -> 41.824, model saved\n",
      "Best mean reward updated 41.824 -> 42.389, model saved\n",
      "Best mean reward updated 42.389 -> 43.000, model saved\n",
      "Best mean reward updated 43.000 -> 43.300, model saved\n",
      "Best mean reward updated 43.300 -> 44.143, model saved\n",
      "Best mean reward updated 44.143 -> 44.304, model saved\n",
      "Best mean reward updated 44.304 -> 44.667, model saved\n",
      "Best mean reward updated 44.667 -> 44.760, model saved\n",
      "Best mean reward updated 44.760 -> 44.808, model saved\n",
      "Best mean reward updated 44.808 -> 45.333, model saved\n",
      "Best mean reward updated 45.333 -> 46.207, model saved\n",
      "Best mean reward updated 46.207 -> 46.613, model saved\n",
      "Best mean reward updated 46.613 -> 46.875, model saved\n",
      "Best mean reward updated 46.875 -> 47.182, model saved\n",
      "Best mean reward updated 47.182 -> 47.353, model saved\n",
      "Best mean reward updated 47.353 -> 47.571, model saved\n",
      "Best mean reward updated 47.571 -> 48.000, model saved\n",
      "Best mean reward updated 48.000 -> 48.342, model saved\n",
      "Best mean reward updated 48.342 -> 48.667, model saved\n",
      "Best mean reward updated 48.667 -> 48.750, model saved\n",
      "Best mean reward updated 48.750 -> 48.881, model saved\n",
      "Best mean reward updated 48.881 -> 49.023, model saved\n",
      "Best mean reward updated 49.023 -> 49.295, model saved\n",
      "Best mean reward updated 49.295 -> 49.578, model saved\n",
      "Best mean reward updated 49.578 -> 49.935, model saved\n",
      "Best mean reward updated 49.935 -> 50.128, model saved\n",
      "Environment solved in 47 seasons!\tAverage Score: 50.13\n",
      "Average Score: 50.13\n",
      "Elapsed time: 0:26:12.673436\n"
     ]
    }
   ],
   "source": [
    "best_mean_reward = None\n",
    "\n",
    "scores_window = deque(maxlen=100)  # last 100 scores\n",
    "\n",
    "discount = 0.993\n",
    "epsilon = 0.07\n",
    "beta = .01\n",
    "opt_epoch = 10\n",
    "season = 1000000\n",
    "batch_size = 128\n",
    "tmax = 1000//num_agents#env episode steps\n",
    "save_scores = []\n",
    "start_time = timeit.default_timer()\n",
    "\n",
    "for s in range(season):\n",
    "    policy.eval()\n",
    "    old_probs_lst, states_lst, actions_lst, rewards_lst, values_lst, dones_list = collect_trajectories(envs=envs,\n",
    "                                                                                                       policy=policy,\n",
    "                                                                                                       tmax=tmax,\n",
    "                                                                                                       nrand = 5)\n",
    "\n",
    "    season_score = rewards_lst.sum(dim=0).sum().item()\n",
    "    scores_window.append(season_score)\n",
    "    save_scores.append(season_score)\n",
    "    \n",
    "    gea, target_value = calc_returns(rewards = rewards_lst,\n",
    "                                     values = values_lst,\n",
    "                                     dones=dones_list)\n",
    "    gea = (gea - gea.mean()) / (gea.std() + 1e-8)\n",
    "\n",
    "    policy.train()\n",
    "\n",
    "    # cat all agents\n",
    "    def concat_all(v):\n",
    "        #print(v.shape)\n",
    "        if len(v.shape) == 3:#actions\n",
    "            return v.reshape([-1, v.shape[-1]])\n",
    "        if len(v.shape) == 5:#states\n",
    "            v = v.reshape([-1, v.shape[-3], v.shape[-2],v.shape[-1]])\n",
    "            #print(v.shape)\n",
    "            return v\n",
    "        return v.reshape([-1])\n",
    "\n",
    "    old_probs_lst = concat_all(old_probs_lst)\n",
    "    states_lst = concat_all(states_lst)\n",
    "    actions_lst = concat_all(actions_lst)\n",
    "    rewards_lst = concat_all(rewards_lst)\n",
    "    values_lst = concat_all(values_lst)\n",
    "    gea = concat_all(gea)\n",
    "    target_value = concat_all(target_value)\n",
    "    \n",
    "    # gradient ascent step\n",
    "    n_sample = len(old_probs_lst)//batch_size\n",
    "    idx = np.arange(len(old_probs_lst))\n",
    "    np.random.shuffle(idx)\n",
    "    for epoch in range(opt_epoch):\n",
    "        for b in range(n_sample):\n",
    "            ind = idx[b*batch_size:(b+1)*batch_size]\n",
    "            g = gea[ind]\n",
    "            tv = target_value[ind]\n",
    "            actions = actions_lst[ind]\n",
    "            old_probs = old_probs_lst[ind]\n",
    "\n",
    "            action_est, values = policy(states_lst[ind])\n",
    "            sigma = nn.Parameter(torch.zeros(action_size))\n",
    "            dist = torch.distributions.Normal(action_est, F.softplus(sigma).to(device))\n",
    "            log_probs = dist.log_prob(actions)\n",
    "            log_probs = torch.sum(log_probs, dim=-1)\n",
    "            entropy = torch.sum(dist.entropy(), dim=-1)\n",
    "\n",
    "            ratio = torch.exp(log_probs - old_probs)\n",
    "            ratio_clipped = torch.clamp(ratio, 1 - epsilon, 1 + epsilon)\n",
    "            L_CLIP = torch.mean(torch.min(ratio*g, ratio_clipped*g))\n",
    "            # entropy bonus\n",
    "            S = entropy.mean()\n",
    "            # squared-error value function loss\n",
    "            L_VF = 0.5 * (tv - values).pow(2).mean()\n",
    "            # clipped surrogate\n",
    "            L = -(L_CLIP - L_VF + beta*S)\n",
    "            optimizer.zero_grad()\n",
    "            # This may need retain_graph=True on the backward pass\n",
    "            # as pytorch automatically frees the computational graph after\n",
    "            # the backward pass to save memory\n",
    "            # Without this, the chain of derivative may get lost\n",
    "            L.backward(retain_graph=True)\n",
    "            torch.nn.utils.clip_grad_norm_(policy.parameters(), 10.0)\n",
    "            optimizer.step()\n",
    "            del(L)\n",
    "\n",
    "    # the clipping parameter reduces as time goes on\n",
    "    epsilon*=.999\n",
    "    \n",
    "    # the regulation term also reduces\n",
    "    # this reduces exploration in later runs\n",
    "    beta*=.998\n",
    "\n",
    "    mean_reward = np.mean(scores_window)\n",
    "    writer.add_scalar(\"epsilon\", epsilon, s)\n",
    "    writer.add_scalar(\"beta\", beta, s)\n",
    "    writer.add_scalar(\"Score\", mean_reward, s)\n",
    "    # display some progress every n iterations\n",
    "    if best_mean_reward is None or best_mean_reward < mean_reward:\n",
    "                # For saving the model and possibly resuming training\n",
    "                torch.save({\n",
    "                        'policy_state_dict': policy.state_dict(),\n",
    "                        'optimizer_state_dict': optimizer.state_dict(),\n",
    "                        'epsilon': epsilon,\n",
    "                        'beta': beta\n",
    "                        }, PATH)\n",
    "                if best_mean_reward is not None:\n",
    "                    print(\"Best mean reward updated %.3f -> %.3f, model saved\" % (best_mean_reward, mean_reward))\n",
    "                best_mean_reward = mean_reward\n",
    "    if s>=25 and mean_reward>50:\n",
    "        print('Environment solved in {:d} seasons!\\tAverage Score: {:.2f}'.format(s+1, mean_reward))\n",
    "        break\n",
    "\n",
    "\n",
    "print('Average Score: {:.2f}'.format(mean_reward))\n",
    "elapsed = timeit.default_timer() - start_time\n",
    "print(\"Elapsed time: {}\".format(timedelta(seconds=elapsed)))\n",
    "writer.close()\n",
    "envs.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztvXl8XGd97/9+ZkYzkkb7akveF9lx7NhJnJDNkQIJaxrncinL7RIoP9Lyu6VQll5aLrT9ld5Ce8tSLi0EaMltC4QtOBAgCUlkJ4EkthPHS+JN3rWOds1ImtHMPL8/zpzRSJpV0uzf9+ullzVH58w8eqyZz/nuSmuNIAiCULxYsr0AQRAEIbuIEAiCIBQ5IgSCIAhFjgiBIAhCkSNCIAiCUOSIEAiCIBQ5IgSCIAhFjgiBIAhCkSNCIAiCUOTYsr2AZGhoaNDr1q1b1LUejwen07m8C8pDZB8MZB9mkb0wKOR9OHz48KDWujHReXkhBOvWrePQoUOLurazs5OOjo7lXVAeIvtgIPswi+yFQSHvg1LqYjLniWtIEAShyBEhEARBKHJECARBEIocEQJBEIQiR4RAEAShyBEhEARBKHJECARBEIocEQJByGOGPT4ePdqb7WUIeY4IgSDkMT9+6Qr//TsvMTY5k+2lCHmMCIEg5DFjU4YAjE+LEAiLR4RAEPIYjzdg/OvzZ3klQj4jQiAIeYzHawiAe1qEQFg8IgSCkMe4Q5aA2ytCICweEQJByGNMi8B0EQnCYkibECiltiiljkR8jSulPqKUqlNKPaGUOhP6tzZdaxCEQifsGvJKsFhYPGkTAq31Ka31Lq31LuB6YBJ4GPgk8KTWejPwZOixIAiLwB2yBNxiEQhLIFOuoTcAXVrri8Be4MHQ8QeBezO0BkEoOGZdQxIjEBZPpoTg3cB3Q983a63NUsg+oDlDaxCEgmNSgsXCMqC01ul9AaXsQA9wtda6Xyk1qrWuifj5iNZ6QZxAKXU/cD9Ac3Pz9d/73vcW9fput5uKiorFLb6AkH0wKLR9uP9xD74gdKy28d6rHSldW2h7sVgKeR/uuOOOw1rr3YnOy8TM4rcAL2mt+0OP+5VSK7XWvUqplcBAtIu01g8ADwDs3r1bL3amaCHPI00F2QeDQtoHfyCI75e/AKC6vomOjmtTur6Q9mIpyD5kxjX0HmbdQgCPAPeFvr8P2JeBNQhCweHxzQaIpaBMWAppFQKllBO4C/hxxOHPAXcppc4Ad4YeC4KQIpEBYokRCEshra4hrbUHqJ93bAgji0gQhCUQKQTSa0hYClJZLAh5iukaqnDYxDUkLAkRAkHIU0yLoLnKIQVlwpIQIRCEPMUdFoJSKSgTloQIgSDkKZ4IIZiaCeAPBLO8IiFfESEQhDwlUghgbjqpIKSCCIEg5ClmXKCp0qgoFveQsFhECAQhT/F4/VgUNISEQGoJhMUiQiAIeYrH58dpt1HpMMqBRAiExSJCIAh5isfrx+mw4QwJgbiGhMUiQiAIeYrHG8DpsFJhWgRSVCYsEhECQchT3F4/FQ7brBCIRSAsEhECQchTZl1D1vBjQVgMIgSCkKe4Q0JQURqKEUgdQV5yZWSSqSz/34kQCEKeYmQNWXHYrJRYFRMSI8g73F4/b/7SM3z5yTNZXYcIgSDkKZPeQDhjyOmwiWsoD3n8RB9ur5/DF4ezug4RAkHIU8xgMRitqEUI8o99R3oAONEzTiCY3vnx8RAhEIQ8xB8I4vUHwxZBhcPGhAhBXjHo9vLs2UFW1ZYx6QtwftCdtbWIEAhCHuIJ9RkqRNfQxSEPPzh0OdvLSDs/P9ZLIKj5szdvBeDolbGsrUWEQBDyEHdoNGVFKHW0kFxD33nxEp/44VHOubJ3h5wJ9h3pYeuKSt66fQVlJVaOdYsQCIKQAuaHfiG6hkY8PgAeeaUnyytJH5eHJzl8cYR7drVgs1rY1lLFMbEIBEFIBbOK2Gk3XUPWgrEIRiZnAHjkSA9aZy+Amk5Mkfuta1oA2NFandWAsQiBIOQhk/NiBBWOknDcIN8ZnTQsgnODHo53j2d5NenhkSM97F5by+q6csAQgqmZQNbcYSIEgpCHhC2CcIzAitvrJ5jFFMTlYmRyhls21lNiVew70p3t5Sw7J/vGOdU/wd5dLeFj16yqBrIXMBYhEIQ8xHQDVURkDQFMzuS/VTA66WNtvZP2tiZ+erQnq/n16WDfkR6sFsVbd6wMH9vQWEG5PXsBYxECQchDPL55weLSwphJoLVmdHKG2vIS9u5qoX/cywvnh7K9rGUjGNQ8cqSH2zY1UF/hCB+3WhTbVlaJEAiCkDzueRaB+W++9xtye/34g5racjt3XtVMud3KI0cKJ3vopUsjdI9OzXELmexYVc2rWQoYixAIQh5izit22Iy3sJk9lO8WwWgoY6imvIQyu5U3Xb2Cnx/rxevPf5cXGG4hh83CG69eseBnZsC4KwsBYxECQchDPKGGc0opoHBcQyOhjKHacjsA9+xqYXzaz/5Trmwua1mYCQR59Fgvd25rDltwkWQzYCxCIAh5iCei4RxEuIbyXghmLQKA2zY1UOe0s68AisuePTvIsMfH3p0L3UIA6xuMgPHxLMQJRAgEIQ/x+PzhQDFQMAPszRqCmpBFUGK18LYdK/nVq/15P4rzp0d6qCq10b6lMerPrRbF1S1VHL0ymuGViRAIQl7ijphFALMWQb4LgdleojZkEQDs3dWC1x/k8RN92VrWkpnyBXjsRB9v3bESh80a87wdrTW82juOPxDM4OpECAQhLzFcQ7MfKIXmGqoumxWC69bU0lpTFu7dn488/mofHl+Ae6JkC0WyY1UV0zNBulyeDK3MQIRAEPIQj9cfzhQCKC2xYFH5bxGMTvqoKrVhs85+NFksint2tfDs2UEG3d4sri51tNZ8+7nzfOKHR1lXX87r1tfHPX9Haw1Axt1DIgSCkIeYg+tNlFKhVtT5nWY5MjlDrdO+4PjeXS0EgpqfH+vNwqoWh2vCy/u+fZC/+umr3LapgR9+8BasFhX3mg0NTpxZCBgvzGESBCHn8Xj94T5DJhUOW94XlI1M+sKB4ki2rqhiS3Ml+4708Ps3r8v8wlLkqZP9fOIHR3F7/fzN3qv53ZvWhlN942GxKK5uqeZohoVALAJByEM8vrnBYiiMKWVme4lo3LOrhcMXR7g8PJnhVSXPlC/Ap39ynD/49iGaqkr52Ydu4/duXpeUCJjsWFXNaxkOGItFIBQNl4cn+fYJL7fuCVJizd97oJlAEJ8/SIV97tu3otQW7kGUr4xM+tjUVBH1Z/fsbOEfHjvF+x88SGOlY8HPnXYb//COnVTHEJJ0MzE9w3/9l19zut/NB/as5+Nv2hI3QygWO1qrmZ4JctblZuuKqjSsdCH5+24QhBR57EQfnZf9XBjMbEbGcjN/OplJIbiGRidnwsVk81ldV84H9qynqrQE70xwztfo5AyPv9qf1QZ1z5wZ5HS/m6+851o+9bZtixIBMCwCyGyFcVotAqVUDfBNYDuggT8ATgEPAeuAC8A7tdYj6VyHIAD0jE4DsymK+cr8hnMmTruNvrHpbCxpWfD5g7i9/nB7iWh86m3boh4fn57hmr96PONpl5EcvDBMaYmFN29f2EcoFdbXO6lw2DjePcY7d69eptXFJ90WwZeBX2qttwI7gdeATwJPaq03A0+GHgtC2ukZnQJm+9nkK55508lMKkrzO0YwOrWwmCxZqkpLaKp0ZKVhm8mhCyNcu7p2yW5HS7jCOHMWQdqEQClVDdwOfAtAa+3TWo8Ce4EHQ6c9CNybrjUIQiQ9Y4YQjOa5EJgWQXm0rKE8FoKxcJ+h2BZBPDY2VmRNCNxePyd6xrhhXe2yPN+O1swGjNNpEawHXMC/KaVeVkp9UynlBJq11mYycB/QnMY1CEKYWYsgv11Dk74YrqHQAPtEA997Rqe4/m+e4MJYbtUcmP8v8VxD8djY5OScy5OVgfdHLo0S1LB7Xd2yPN+OVdV4/UHODGRG2NIZI7AB1wEf0lq/oJT6MvPcQFprrZSK+r+mlLofuB+gubmZzs7ORS3C7XYv+tpCotj3wRfQDLoNS+DoyS469eUsr2jxHOozhOC1oy/juTBrFQx0+whqePypThzW2OmKr7j8DHl8PHtJsy6H/iYO9xu/V9drr+DvTj3QqkdnGJua4aePd1LlSD5dczneGw+f8aEAz6XjdPYk/9qxmPIYlsAPnnyB21elPwsqnUJwBbiitX4h9PiHGELQr5RaqbXuVUqtBAaiXay1fgB4AGD37t26o6NjUYvo7OxksdcWEsW+D+cHPfBEJwCV9Svo6LgmuwtaAoOHr8CRV+i49WbW1JeHj192XOAHp09w3Y23RE2vNBk4eBkOH+XUuDWn/ib6D16Cl49x5+230FpTlvL1ltMu/vPkizRvvobXbYjfyiGS5XhvfOPs82xrmeEtd+5Z0vOYBIOaz774ODMVK+no2L4szxmPtLmGtNZ9wGWl1JbQoTcArwKPAPeFjt0H7EvXGgTBxHQLQSEEi8300bl3zWbwOFG7ZleoX8/liSD947mTZTTrGlrcHfDGUP1BpjOHZgJBXr40yg3L5BYCI2C8vbUqYxXG6c4a+hDwn0qpo8Au4H8BnwPuUkqdAe4MPRaEtNIdEoK6UhUeh5hrDLq9XBlJXDXrjlNHAIkbz7kmZhu37T+dO5O/RiZ92K0WykoWl3+/sqqUshJrxgPGr/WOM+kLsHuZAsUmZsB4JgMB47QKgdb6iNZ6t9b6Gq31vVrrEa31kNb6DVrrzVrrO7XWw+lcgyCAYREoBasrLTlrEfzVIyf44H+8lPA8j9eP1aLC84pNKpK1CCa8bGhwUuNQOSUEox6jmCyVdgyRWCyKDY3OjAvBwQtGGdTutctnEQBsb63G5w9ypj/9v49UFgtFQc/oFI0VDqodKmezhi4MebichEVgtKC2LvjADLuGElQXuya8NFY62NFg5dkzgxkfghKLkUnfojOGTLKRQnrowjCr68pYUV26rM/b0dbEYx+5nbbm6C03lhMRAqEo6BmdpqWmjIoSxeikLysphonoG/MyOjmD1x8/rdPjC0Qdfh4eYJ+g35DLPSsEY1MzvJKFYenRiNdeIlk2NlZwZWSK6ZnMpMZqrTl4YWTZrQGA6vIStqyonDObIV2IEAhFQc/oFC01pVTYwR/UeHy5lUPv8wfDQ1eG3PFdV555swhMUnENNVY62FZvxaJyJ06wLBZBkxOtQ1liGeDi0CSDbu+yxwcyjQiBUPBorekenaKl2rAIYHY2bq4Qmb0TGcyNxvyhNCbJuIYmfX7cXj+NlQ4q7Ipdq2tySAhmqHUu3SIAMuYeOnjBCHEuZ8ZQNhAhEAqeYY8Prz8Ydg0BOZc5lIoQGPOKFwpBeYkVlWBc5eCEIYCNFUadQXtbE0evjDKcZWHUWjMaYyhNKqxvcKIUdA1kxiI4dGGE6rISNjWm34+fTkQIhILH7DraUlNGhT1kEeRY5lBvRNdQV4K5vB5vYEENARhZM067DXeccZUut/E6ZsFZ+5ZGtIZnzmTXKnB7/fiDetE1BCalJVZW1ZZlziK4OMzutbVYEoygzHVECISCx6whaI2wCHJNCCLbRyflGrJHbwrgdFhxe2NbO+Zzm0Kwo7Wa2vKSrLuHRpfYcC6STGUODbm9nHN5lq2/UDYRIRAKHrOquKWmNGddQ33j05SVWKkpL0nsGvJFjxEACQfYzxcCq0WxZ3MjB04PEgxmL5PKFOalBovBEIJzLk/af59DF436geXqOJpNRAiEgqd3bAqHzUKd044Zi8xFi2BldSlNlY6EQjDpXTiv2KTCYYubNeSa8GJRUO+c7UXU3tbIoNvLq73jMa9ze/286+u/4Scvdyf4TRbHUttLRLKh0cnUTIDeNLfPOHRhGLvNEp4ols+IEAgFT8/oNK01ZSilsFoUlaW2nLMIesemaK4qpbHSETdG4PMH8QWCVESJEYCRORRXCNxe6pwOrBE+7T1tDUD8NNK/+emrvHB+mG89ez7Rr7IozBkRy+UaAuhKcwvngxdG2LmqetEjKXMJEQKh4OkenaIloptlbbk95yyC/nEvK6tLaayIbxHEmldsYriG4lsE8zuTNlWWcnVLVUwheOxEHw8dusz6BifHusfS4n8fXUaLIBMppFO+AMe7xwoiPgAiBEIRYBaTmdSWl+RUm4lAUNM/Ps2K6pBFMOGNWfkcq+GcSTKuoWgtqtvbGnnp4gjj03P3ZWB8mk/+6CjbW6v49/ffiFLwyJGeZH+1pDGFubps6ULQUGGnqtSWViE4cnkUf1AXRHwARAiEAsfrDzAw4Z1jEdSU23NqXOWQ24s/qA2LoNLB1EwgZuWz2T4idtZQEkJQEV0I/EHNr88OhY9prfnED48yNRPgS++6llW15dyysZ5HXulZ9hYdo5MzVJXalqWdglKKjU0VS6olCAY1Q3FcdIdChWTXrxGLQBBynv4x4808VwhKcso11BcKapoxAoidQhprFoFJvAH2Wutwn6H5XLe2lkqHbY576N+fv8j+0y4+9dar2BTq9b93ZyvnBz0cW+Y++SOTPmqdS48PmCw1hfTHL3ez+29/xd/94jV8/oVN+Q5eHGFLcyXVy+DKygVECISCJrKGwKS23M6oJ3dcQ2Yx2crqMhorDBdWLCEwi8WiVRabx2cCOmrjurGpGWYCOqoQlFgt3LqpgQOnXWitOTswwd8++hodWxr53ZvWhs970/YV2K0W9i2ze2hkcmZZAsUmGxsrGJjwLnB1JcuRyyMo4Ov7z/H2f3mOsxGB50BQ89LFkbzvLxSJCIFQ0MzWEMy1CCa8/owM/EgGs5jMjBFAbCGYTBAjcNoNSyFav6H5NQTzad/SSPfoFK/1TvDh7x3B6bDx9++4Zk676+qyEu7Y2shPX+khsIx5+qOTPmqWIT5gsrHRCcC5RU4r6xrwsHN1DQ/83vV0j0xx91ee4T9fuIjWmpN947i9/rzvLxSJCIFQ0JhCsLI6Mlhs3HnmSgpp3/g0JVZFvdMeIQTRc+BN/39Mi6DU+DCNVlQWFoIoMQKA29saAfij/zjMiZ5xPvf2HTRVLuyxv3dXKwMTXl44N7TgZ4vF6Dy6jELQtLQU0i6Xm42NFbzx6hU89pHbuWFdHZ96+Dgf+L+HefxEP4BYBIKQL/SMTdFQYac0Yvyh2fM+VwLGfWPTNFWWYrEoaspKsFlUzFqCxOmjIYsgSpzAfM5YFkFrTRmbmyq4NDzJe25czRuvXhH1vNdvbaLCYUvKPXTowjCHQxW48TCmky2fa2hNXTk2i1pUnGB8eoaBCW84DbWpqpQH33cjn757GwdOu/jyk2dYWV06x92Y74gQCAVNd2ggTSSmRZArKaS9Y1Nhi8ViUTTEqSUws4liBYvjDbBP5BoCePt1q7i6pYr/+bZtMc8pLbHyxqub+fnx3rhDdC4OebjvX1/kM/uOxzwHjOHvE17/srSXMCmxWlhbX74oITDdSaZ7CYz/l/fftp59f3wrO1dVc8/OlkWP1MxFRAiEgqYnNIcgklnXUG5YBP3jXpojXFeNcdpMuL1+bBaFPUaaZbwB9q4JL3abharS6NYEwAc7NvLon+yJaXGY7N3VysS0n85T0YvQ/IEgf/rQETy+QMK+P+FisiXOIpiPkTmUeozAdCeZ7qVIrlpZxb4/vo0/f+tVS15fLiFCIBQsWutQMdlcIZh1DWXfItBaGxZB1TwhiOMacjpsMe9G400pM2sIluNO9taN9dQ77TGLy/65s4uXLo1y26aGhH1/lrO9RCQbmyq4OORJOSmgy+XGZlGsqStf1vXkMiIEQsEyNjXDpC8wp6oYCOer50ItwdjUDNMzwTmDz+O1mXDHGEpjEtc1FKOGYDHYrBbuvmYlv3qtn4l5KZpHLo/y5SfPcO+uFv749ZuA+EHb5Ww4F8nGxgpmAprLw5MpXdflcrO2vpySDMwKzhWK5zcVCorvH7rMZ3/2atxzotUQgJFiWWJVOREjMIvJVsxzDQ26fVHdKZMxhtKYhAfYx7IIlkkIAO7Z1YrXH+SxUBaN+bof+d7LrKgq5a/3bk+q789ytqCOxPTxp+oe6nJ5wusuFkQIhLzjRM8Yn3r4GN967nzcNgCRk8kiUUrlTJuJ2WKyuUIQCOqoFku8WQQw23oipmtoGYXgujU1rKotY9+R2dbUn330NS4OT/KP79xJdVlJUn1/Zl1Dy2sRbFhE87mZQJCLQ56o8YFCRoRAyCumZwL86UNHKLFa0BqePTsY89xoxWQmtTnSZmK2mGx2jeFagigil8g1ZLUoykqsCwrKZgJBhid9MWsIFoNSir27Wnju7CCuCS9PvNrPd1+8xB/evpGbNtSHz0nU92e28+jyWgTVZSU0VjpSqiW4PDzJTECLRRALpdRtSqn3hb5vVEqtT9+yBCE6n//lSU73u/nq71xHndPO/hhZK2AIgd1moT5KD5uacntuuIbGplEKmiLu1ONVF3vijKk0qSi1hZvTmQx7fGgdP3V0Mezd1UpQw7d/fZ7/8aOjbFtZxUfvaptzTqK+PyOTM9itFsrty9/Xf2OjMyWLoCtK6mgxkJQQKKX+EvgfwJ+HDpUA/5GuRQlCNA6cdvFvz13gvbes444tTezZ3MCBM66YqYndo1O0VJdGHSxeW16yZNeQ1nrJbRb6xqZpqHDMCUyad+3RhSBAeZwYAZitqOfm9ydTQ7AY2por2bqikq8+3YXH6+fL796F3Tb3YyVR35/RSR815SVpycs3U0iT7ZZqisYGsQii8l+AewAPgNa6B6hM16IEYT4jHh8f/8ErbGqq4JNv2QqYIxZ9MUcsRksdNaldBovgK0+dZc/nn8K/hJ5FvePTc+IDEN8iSOQaAqPYbH6wOF1CAIZVAPAXb72Kzc0LPxYS9f0x2kssr1to9rUrGJuaYciTnOh3DbhprHQsy1yEfCJZIfBpQ1I1gFKquOwmIatorfnzHx9jZNLHl961K9wuYs9mozdOrMlaPVGqik2qQxbBYvvqj0/P8I0D5+gZm+Zk38SingOgf2yaFVVzhcDpsFFuty4QAq11uI4gHhUO24IYQaI+Q0vhfbeu4+u/dz2/f/PaqD9P1PfH6Dyang/eVHsOGT2Giu/jLVkh+L5S6utAjVLqA8CvgG+kb1mCMMsPD1/hlyf6+Ngbt7C9dXZQeGOlg+2tVVHjBDOBIP0TsYWgttzOTEDHHACTiP98/hITobtuc0jJYugdm5qTOmoSrajMFwjiD+qEFkG0KWWJ+gwthdISK2+6ekVM106ivj+jabUIkk8h1VoXZeooJCkEWuv/DfwQ+BGwBfiM1vor6VyYIABcGprkrx45wevW1/GBPRsW/Ly9rZHDlxaOWOwbm0ZraK1Z+CELs8VLI0m6DCKZngnwrWfPs2dzAy3VpRxMoqlaNCZ9fsan/dGFIEpRmdlR1JkgqOp0LAwWuya8VJba5jTfyxSJ+v6MTM4se3sJk5bqMkpLLEkFjIc8PsamZkQIoqGUsiqlntZaP6G1/oTW+uNa6ycysTihcHB7/SnfOfsDQf70+0ewWBRfeNcurFGCvu1tTQSCml/PSyONlzoKs+0MFtNm4kcvXWHQ7eWDHRvZva6OQxeGF+Vi6otSQ2ASrd9Qos6jJrFcQ+mwBpIlVt8frXUoWJwei8BiUWxoSG5aWbweQ4VOQiHQWgeAoFKqOtG5ghCLr3V28Y6v/YaLQ8lXef7qtQEOXxzhr37r6pgtf69dU7NgxCIY7achthDMdiBNzSLwB4J8ff85dq6u4eYN9dywrpb+cS9XRqZSeh6YFYLmquRcQ4kG15tEdQ3FmFWcKWL1/fH4AswE9LIOpZlPW3MFx7vHE4p1saaOQvIxAjdwTCn1LaXUP5lf6VyYUFg8fWoAgJ++kvyIw0de6aahws7eXS0xzzFHLO4/5ZrzRg9XFVfHEoKQayhFIfjF8T4uDU/ywfaNKKXYHZpSdXARcYLIEZXzaaxwMDo5M6fNc7IWgdNhw+sPzvnQXc4+Q4shVt8f0zWXrhgBwK2bGhh0e3mtN35Qv8vlprTEEvNvppBJVgh+DHwaOAAcjvgShIQMTExzosdI8fzJkZ6k3CgT0zP86rUB7r6mBVuC5l/tWxrpGZueM1e2e3SKOqedshj+9MW4hrTW/EtnFxsanbxxWzNg5NFXlto4eCH1OEG4z1AMiwBgyD0rVLPTyRLXEcDcfkPZdw1FD9qa+5+urCEw4kgQO7vMpMvlZkNDRdS6k0In2WDxg8B3mRWA74SOCUJCnjlt+O/fc+Mazg64E96ZATx2oh+fP8g9cawBk9ujvNGNGoLogWKY/eBJxSJ45swgr/aO80ftG8MfFlaL4vq1tRy+mLpF0Dc2TU15SVSxilZLMBkeSpPYNQSzwjHp8+P2+rMqBLH6/oQbzkWp/l4umqpKuWplFftPD8Q9r8vlLsr4ACRfWdwBnAG+CvwzcFopdXsa1yUUEPtPu2iocPDxN7Zhsyj2vdKd8Jp9R7pZXVfGtatrEp5rjlhcIARxTPwSq4VKhy0li+BfOrtYUVXKvaECKpMb1tVxut+dcqVyb5QaApNoQhCOESRoMeEMWwSGcAxOGOvKZowgVt+f2c6j6S3gam9r5NCFkajN+MDIBLsyMlWU8QFI3jX0j8AbtdbtWuvbgTcBX0zfsoRCIRDUPHPGxe1tDdRXOLi9rZGfHumJO7HKNeHlubOD7N3ZmnTbgfa2Rl44N8ykz4/Wmu6R2FXFJjXO5NtMvHxphN+cG+L/2bN+QQuF3WuNIebJzOaNpG88eg0BRG885/HGH1xvYraidntnQs8xPec5s0W0vj+zrqH0WQRg/H34o2SXmZwf9KA1RZk6CskLQYnW+pT5QGt9GqPfUFyUUheUUseUUkeUUodCx+qUUk8opc6E/q1d3NKFfOBY9xgjkzNhP+3eXS30jE1zKM6H5qNHewhq4gaJ59O+pRFfIMgL54YZn/bj8QUSDhdPpc3E1/Z3UV1WwntuXLPgZztX11BiVSnHCfrGvFFTRwHqnQstguTTR80B9oE5z5F9IVjY98e0CNKZNQRw/dqyE9jdAAAgAElEQVRanHZrzDiBKVAiBPE5pJT6plKqI/T1DeBQktfeobXepbXeHXr8SeBJrfVm4MnQY6FA2X/KhVKz7SDuvKqZshLrnB7289n3Sg9XrayK2rcmFjesq6O0xML+066ENQQmyc4kODswwWMn+rnvlnVRP4RLS6zsaK1OqU7C5w8y6PZGTR0FsNss1JaXzHMNBSixqgUWyXyc84LF6WwvkQrR+v6MTs5QWWpLmBCwVOw2C7dsamD/aVfUZIWuAQ9KwfoGcQ3F44PAq8CfhL5eDR1bDHsBM9D8IHDvIp9HyAP2nx7gmlU11IWCgU6Hjbu2NfPosV58/oXN2i4NTfLypVHu2Zm8NQDGh/HNG+o5MEcIYgeLwZxJkNgi+Pr+c5SWWHjvLetinnPDujqOXhljeia5lhX947GLyUzmF5Ul02cIIoLF07NCoBTh/4NsEa3vTzrbS8ynva2RKyNTnBtcWMvS5XLTWlMWM8us0ElWCGzAl7XWb9davx34JyCZHdPA40qpw0qp+0PHmrXWvaHv+4DmlFYs5A2jkz6OXB4Nu4VM9u5qYXRyhmfPLjTTHwkFkn9r58qUX6+9rZFzgx6ePzcELBxROR/DNRTfIhhye/nJkW7efcOauB+ku9fV4QsEOdY9ltRa+8cXDqSZz/yismRmEcDCrCGX20u90572u+5EREshHZmcSXug2CScRhqlN5XRbK443UJgfMAnw5PAnRiFZQBlwOPALQmuu01r3a2UagKeUEqdjPyh1lorpaJGDUPCcT9Ac3MznZ2dSS51Lm63e9HXFhLZ2IcXe/0ENVS5L9PZOVtIpoMaZwk88NjLWPpm74i11nznuSnaai2cfeVFzqb4eqUew8L43vPnsSo4fvg3WOYFmyP3YXTAx8S0nyefejpq+wqAoy4/MwHNSn8fnZ2x89Cnfcaf8UNPHcKzIfEd7gu9xof05VPH6OyJ/gEd9ExzaTQYXu/FnmmUP5jw/9EfCsQfP3WGTv9FXjs/TZnSC67L9N9EUGvsFtj/0mu0TJ0D4FL/FBUlKmPrWOFUPPz8KTb4L4aPjU+4OdOvaC2xFe1nRbJCUKq1DttzWmu3Uqo80UVa6+7QvwNKqYeBG4F+pdRKrXWvUmolEDW5V2v9APAAwO7du3VHR0eSS51LZ2cni722kMjGPjz6g1eoLuvnvffcseBudO/YMR5+qZsbb7mN8tBd7qs94/Q89gx/c9fVdNwUvaVxPLTW/POJTi4NT7KmrpzX33HHgnMi9+Gi/QI/OXuCnTfeQkMM//mp/V3ASd795tupTnDn+uXj+xm2lNPRcUPCtZ45cA5eeY3funNPzN73z3le5cjzl2hvb0cpxTfPvkCTw09Hx60Jn9/x1C9oXLmajo6r+OLxZ1lXU0JHx+vmnJONv4lNR5/BW+qgo+NGAD7z4tNsXFVDR8e1GXn9t0yc4DsvXOKmW/eEG/D96BdP4QtM0X7dVjpel/rfXSGQrK3oUUpdZz5QSu0G4jZXUUo5lVKV5vfAG4HjwCPAfaHT7gP2pbpoIffRWrP/tIvbNjdEdUns3dnC1EyAJ17tDx975JUebBbF23ak7hYCYz6uaf4nig/AbFFZvIDxqb4JVlSVJhQBgBvW1XLownDc1FiTvvFpyu1Wqkpj34s1VjqYmgmEW2W7k4wRwNx+Q9muKo5kY9PcBnAjaWw4F432tka8/iAvnJ8N7PeGLMlidg0lKwQfAX6glHpGKfUM8D3gjxNc0ww8q5R6BXgReFRr/Uvgc8BdSqkzGO6mzy1u6enB6w9wZWQy8YlCXE72TTAw4V0QHzC5YV0dK6tLeeSI4TIKBjU/faWHPZsblhTUvD0sBIn7xdSEG8/FDhif7Jtgy4rkspd2r61jfNrPmSSGoPSNTbOiujRuncT8orJkYwQQakXtNWoqst1nKJKNjU6ujEwxPRPAHwgyMe3PWLAY4KYN9Thsljlxgl63Dq1NhCAqSqkblFIrtNYHga3AQ8AM8EvgfLxrtdbntNY7Q19Xa63/NnR8SGv9Bq31Zq31nVrrxU/1SAPfeeESd33hQMwKRCE5zHztWEJgsSju2dnC/tMuRjw+Dl8aoXt0Kjz2cLHcsrEep93KpiRaBSSaSeAPBDnrcrM1SSG4IYUGdL1jUzGrik0aK4yfzxGCFC2CsakZZgI666mjJhsbK9DaKOAanTIEOF2zCKJRWmLldRvq57Sb6PUEqSq10VCR3ayqbJLIIvg6YL5Lbgb+AqPNxAgh/32hcX7Qw9RMgBNJZn8I0dl/ysXWFZUx8+QB7tnVgj+o+fnxXvYd6aa0xMJd25aWROZ02HjyYx28/7b1Cc+tTdB47sKQB58/mLRFsLqujKZKR1L1BP3j3phVxSYNlcb6TCEw5hUnl95oCkGuFJOZbIzoOWS65DLpGgLj5qTL5Ql3Qu31BNnYVJF0FXshkkgIrBF37O8CHtBa/0hr/WlgU3qXlh3MtL5k0wCFhbi9fg5dHKZ9S3RrwGTbyio2NVXwo8NXePRoL3dtW5H0HW88VlSX4rAl/sBM1HjOnEWcrBAopbhhXV3CCuNAUNMfZWj9fMy7eNfEtDGv2BdIen+MAfaBnBOC9Q1OlDIKuEyXXKbSR01MK/XAGcNq7fXoonYLQRJCoJQy//LeADwV8bOlv2NzkP5x440jQrB4ftM1xExAx3QLmSil2LuzhZcujTIyOcPeFIvIlkqFw4bNomLGCE71TWC1qKTcTCa719XSPToVLmqLxpDbiz+oE7qGasvtWC0Kl9uL1x8kENTJu4ZKSwyLIFSH0JQjQlBmt9JaU0aXyx12ydWUZdYi2NjopLWmjAOnXYxPzzDqFSFIJATfBfYrpfZhZAk9A6CU2gQU5CflgFgES2b/6QHK7VZ2r61LeK7ZZrq6rCQc6M0USqm4bSZO9k2wvsGZlHVhYsYJ4vVS6kuimAyMOEpDhR3XhDfphnMmFQ7rXNdQReIsqkxh9BxyZ2QWQTSUUrRvaeS5s0OcDll9xdp11CSuEIQCvB8Dvo1RHGbmxVmAD6V3aZknGDQyLOw2C+dcHiamU59nW+xorek85eKWjQ0Je+IArK13cvc1K3nfreuSOn+5MdpMRBeCUylkDJlsXVGJ026NGyfojTOreD5mm4nw4PpkXUN2I2vINeHFbrVQVZY7BvzGxgrOuTwMZ2AWQSza2xpxe/384NAVY01FOofAJJmZxc9rrR/WWnsijp3WWr+U3qVlnpFJHzMBzS0b6wHCU7WE5Dk/6OHKyFTC+EAk/+e/XcdH7mxL46piE6sDqcfr59LwJFtTaHwHYLNauG5tbdw4QbxZxfNprDDaTMzOIkgyWFxqY9IXoG98msZKR04FQjc2OZmaCfBqzzglVpX077Sc3LKxHptF8fCRbqwK1tQlrI8taLLbfCTHMOMDb7jKyFw5dkXcQ6kSThvdnFk3z2KpKS9hLIoQnO5PLVAcye61dZzsG2c8hkXZNz5NiVVRn8SdcNgi8CXXgtrEdCFdGJqkIUfiAyamP/7wxRFqyu1ZEanK0hKuX1uLzx+kqVxRkuU+TNmmuH/7eQxMGHdq21ZW0VJdKnGCRbD/tIsNDU7W1OfHHVasxnOnQr7jrSuqUn7OG9bVojW8FCNO0Dc2TXNVaVKzcRsrHQy6fWE3ZfJZQyEhGPTkTA2BiSkE3aNTGc8YisS0Wlc65WNQdiCCgfHZDIvtrdUiBCkyOunj+XNDGQ/6LgVjStnMgh71J/smKLdbWVWbuEJ5PrvW1GC1KH5w6Ape/8K21L1jU0nFB8BwDQWCmisjRhZS8sFi47yxqZmcSR01aaiwh1trZLqGIBIzq02EQIRgDmYNQVOVgx2t1Zwf9MQ074W5aK351MPH8Qc079y9OtvLSZracju+QDA8GN7kVN8Ebc2VSd21z6fcbuOP2jfw6LFe7v3qrzkTcjOZ9I/HHkgzn8ZK47wLg0bxkzOFgrLZ58gtIVBKhYOz2bQItq2s4qN3tXFba+4E0rOFCEEE/RPT1JaX4LBZ2bGqGoAT3RIwToaHX+7m0WO9fPSNbWxrSd2dki1qoxSVaa051T/BlhQDxZF84k1b+dZ9uxkYn+burzzLv//mAlprtNapWQShD/ELQ0auRrIWgTOHhQBm3UOZ7DM0H6UUf/KGzayskI9B2YEIBsa9NIXuwHa0GkJwrHs0m0vKCy4PT/KZfSe4cV0df3j7xmwvJyVqorSZcLm9DHt8iwoUR/KGq5r5xUf2cNOGej697wTvf/AQXS4P0zPBhDUEJmEhCE3VKk+y6dwciyDHYgQAG0J5+9l0DQmziE0UQf+El6Yq401TX+GgtaaMY3lkEQy6ZwuP5jM5k7g18mIIBDUf/f4RFPCP79wZc8BLrmIOTY+0CGYDxUsTAoCmylK+/b4b+PavL/B3vzjJPf/nWYCEVcUmphBcHpnEbrUkXWuRy64hiLQIsucaEmYRIYhgYHyazU0N4cfbW6s4diU/LILLw5N0/O9OAjF64VfZFXe9PrjsaXJf29/FwQsjfPFdO1mdh7nYZjFTZC3BqRR7DCVCKcX7bl3PzRvr+fB3j3Cqf4K1SWZVOe1WykqsTM0EqC1P/u0aGUvIlfYSkZhut2RjJUJ6ESEIEQxqXBPeOW+aHa3VPHain/HpGapKc/vO5dddgwSCmk/fvW3BXdapvgm+fuAcRy6PhlsgLAfHrozxxSdOc/c1K7l3ie2js0W04TQn+yZoqHBQv8wula0rqtj3x7dyvHuM7SHXYyKUUjRWOrg0PJlSQ76KiIE3saavZZN1DU5+9MGb2dFak+2lCIgQhBme9OEP6jl3KDtWGX+kx7vHuGVjQ6xLc4KDF0aoc9r5g1vXLSjQGZua4RvPnGP/KdeyCcGUL8CHH3qZxkoHf3vvjpyqXE0Fs+HZiGeuRbAcbqFolJZY2Z3i/4EpBMkGigEcNislVkWpzUpZFip3k+H6JHpRCZlBgsUhzNTR5qq5FgHkR4XxoQvD7F5bG/UDubqshE01lnDV73Lwv37+GudcHv7xt3cmNcYxV7HbLFQ4bOEYQSCoOd2feo+hdGIGe1Nt0e102HIyPiDkHiIEIQZCXRqbIiyCOqc9FDDObSEYmJjmwtBk3Lv97Q1WjnWPMRhqS7wUnj45wL8/f5EP7FnPLZty21JKhprykrBr6OKQB28Kw2gygflhnqoQVDhsOddeQshNRAhCmO2n5wfWduRBhfHhUIOz3etqY55zTYPhHnjmzNKtgn997jxr6sr5+Ju2LPm5coHIxnPLmTG0XISFIEUXz9YVVVyTZCxCKG5ECEKYDefmm9I7VlVzcWgyamOyXOHghRFKSyxc3RL7Tb+mykK90z5naPdiOTvgZvfa2pT69OcykRbByb4JlILNTTkoBClaBN+8bzf/8+5t6ViSUGCIEIToH5+mzmlf8OFmxgmO9+SuVXDo4jC7VtfEzTG3KMXtbY0cODNIMEaKaTK4vX56x6YLqn/7fItgXb0zpwKsZowglWCxIKSCCEGIgXmpoyazFca5KQQer58TPeNJZQO1tzUy7PEtSdTOu4wK10Ka6BQ5nGaprSXSwaxFkDviJBQWIgQhBsan5wSKTWqddlbV5m7A+MjlUQJBnVRK4p7NDSjFktxDXS43QEHNeK0ptzMx7WdieoYLQ56cChTD4l1DgpAsIgQh+se9NMfIsNjRWp2zKaQHLwyjFFy7JnFhTn2F0VV1KWmkXS43VovKm3kDyWAW4B2+OILWuRUoBqMdxXtuXENHW1O2lyIUKCIEzM4qjlXuvmNVNZeGczNgfOjCCFtXVCVd+dze1shLl0YW/bt0udysqSsvmEAxzLaZeOG8MWc41ywCi0Xxd2/fkVddXYX8QoQAGPL4CAR1uOHcfHI1TuAPBHnp0gg3xEkbnU97WyNBDc91DS7qNbsGPAUVH4DZDpjPnxuitMTC2vrC+v0EIREiBEQMpKmMbhFsb8lNIXitd4JJXyCllgW7VtdQWWpbVJwgENScH/QUVHwAZl1DR6+MsbmpMu86qArCUhEhAFyhquLmGBZBrdPO6royjueYEBy8YLgyUrEIbFYLezY3sP+0a8F4xkRcGZnEFwgWoBAYFkEgqHPOLSQImUCEgMgRlbFb4u5oreZojg2pOXRxmNaaMlYmOeTEpL2tkb7xaU73u1O6Lpwx1FRYrpPIXkm5FigWhEwgQkBEVXGcdr07Wmu4PDw1p11xNtFac/BCavEBE3O4/P7TAyld1zVg1BBsaCgsi6DSYcMWcgeJRSAUIyIEGLOK6532uJW5uRYwvjQ8iWvCm3JLY4CV1WVsaa5MOY20y+Wm3mkPZ9kUCkqp8FwCEQKhGBEhwJhVnKhd7/ZWI3UvV4TgYKjR3GLnC7RvaeTg+ZGYoy2j0eVyF1x8wKSm3E6d056T830FId2IEGC0cU40Mq+m3M7GRic/ebmb6ZlAhlYWm0MXhqkqtbF5kT1/2tsa8QWCPH9uKOlrulyegosPmKyrd3J9jHkOglDoiBBgBItjZQxF8um7t3G6380/PHYqA6uKz8ELw+xeV4dlkamOu9fVUlZiTdo9NOzxMezxFaxF8E/v2cWX370r28sQhKxQ9EIQCGoG3b6khmh3bGnivpvX8q1nz/PsmcUVZC0HQ24vXS5P3PkDiXDYrNyysT5pIThXgD2GIim32yi3Sy8foTgpeiEY8niNquIkJzl98i1Xsampgo/94EjWMogOX1xafMCkfUsjF4cmuTDoSXhuITabEwTBoOhvgQbGF46ojEeZ3cqX3rWL//LPz/EXDx/jq//tukX7lb+2vyvmh3BNuZ0/vH1D1AydQxdHsFst4UymxdIeSiPtPDXAexvWxz23y+XBbrPQWptazYIgCLlP0QvB7ND65IQAYHtrNR+9awuf/+VJfvxSN//1+lUpv+6Ix8fnfnGSylIb5VGGoAy5fTz88hW+8M5d3DpvLvDBC8Ncs6qa0pKlNX5bW+9kXX05B84M8t5bEwjBgJsNDU5pvyAIBUjahUApZQUOAd1a67uVUuuB7wH1wGHg97TWWavSCg+tT3HI9/23b+DpUwP85SMnuHF9HavrUmvLbLpa/uk913LHloXthY93j/Hh773M737rBe7fs4GPvrENh83KlC/A8e4x3n/bhpReLxbtbY18/9AVpmcCcYWly+WOOwpTEIT8JRMxgg8Dr0U8/jzwRa31JmAEeH8G1hAT0yJIVEcwH6tF8YV37kQBf/rQEQIpjn80hWBTDJ/79tZqfvahPfzO69bw9QPnePs//5qzAxO8cmWUmYBeVEVxNNq3NDI1E+BQqC4hGl5/gEvDkwXXdVQQBIO0CoFSahXwNuCboccKeD3ww9ApDwL3pnMNiegf99JQYafEmvpWrKot52/u3c6hiyN8bX9XStd2uTw4bBZaamL73MvsVj577w6++fu76R2b5u6vPMsXnjgNwPVrl0cIbtpQj91qidtu4uLQJEFNQc0pFgRhlnRbBF8C/gwIhh7XA6Naa7Oc9QrQmuY1xMU1MR2z/XQy7N3Vwt3XrOSLT5zmfBLZNyZdA27WJ+lzv3NbM7/8yB5uXF/Pi+eHaWuuCPfQXyrldhs3rq+Lm0baNSAZQ4JQyKQtRqCUuhsY0FofVkp1LOL6+4H7AZqbm+ns7FzUOtxud9xrz3ZPUWVXi35+gNuqg/wsqPm/v/w17auSmxR2/NIka6ssKb3ue9dr2hx2ah3+lNcbbx9arTM82+/jR794ivqyhfcGT3QZIZwrr73E4Jn8DhYn+nsoJmQvDGQf0hssvhW4Ryn1VqAUqAK+DNQopWwhq2AV0B3tYq31A8ADALt379YdHR2LWkRnZyfxrp187le8bl0THR3XLOr5wRh1+dfPPwbVLXR0XJ3wfK8/gOuxX/KumzfQ0dGW0mu9fpFrjLcPLVdN8NCpA8zUb6LjxjULfr6v/wgt1UO8+c47FvnquUOiv4diQvbCQPYhja4hrfWfa61Xaa3XAe8GntJa/w7wNPCO0Gn3AfvStYZEGFXF3qTaS8TDYlG0NVdwqm8iqfPDPvccCb5ubqpgZXVpTPdQl8st8QFBKGCyUVn8P4CPKqXOYsQMvpWFNQBGq4aghsYUaghisWVFZdJCkGs+d6UU7W2NPHtmkJlAcM7PtNaccxXeeEpBEGbJiBBorTu11neHvj+ntb5Ra71Ja/3bWmtvJtYQDXMgTXOKqaPR2LKiiiGPLzz2Mh5m6uiGHLEIwKgnmPD6OXJ57hS2gQkvbq8/Z6wXQRCWn6LuNbSYquJYmCMOk7EKulweWqpLc6rJ2S2bGrBa1IKh9rlmvQiCsPwUtRCEq4qXGCOA2clWJ/vGE56biz736rISrltTsyBOMDunOLfWKwjC8lHUQtA/Po1S0LAMU6kaKhw0VNgTWgRaa7oGcnPSV3tbI8e6xxh0z7q3ulweKhy2lFtwCIKQPxS1EAxMTFPvdCyqqjgaW1ZUcqo/vhD0j3vx+AI56XNvbzN6Hj1zZtYqMMZTOmVylyAUMMUtBONLTx2NZEtzFaf7J+L2Hcrlvv5Xt1RR77TPiRPkqvUiCMLyUdRC0D8xvawuj60rKpmeCXJpeDLmObnsc7dYFLe3NXLgzCDBoMbj9dMzNp2TaxUEYfkobiEY9y5LxpDJlnDmUOyAcdeAO6d97u1tjQx7fBzvGQv3TspFN5YgCMtH0QqBPxBk0O1NejJZMrQ1V6IUnIwTMO5yeXLa575ncwNKwf5Trpx2YwmCsHwUrRAMeXxonfpAmniU2a2srSuPmzlkBF9z94O1vsLB9pZq9p920TXgxmpRrKlPbeiOIAj5RdEKwXIWk0USr9WE2+unNw987u1tjbx0aYSXLo2ypq4ch21pIzEFQchtilgIQu0lljFrCIxWExeGPEzPBBb87LwrP3zu7VsaCWp49uxgzq9VEISlU7RCMDCRHotg64pKghrOhlozRJIvPvdrV9dQWWq0v8j1tQqCsHSKVgj6x70oBfXO5Zn0ZTLbamKhe6jLlR8+d5vVwm2bGgARAkEoBopWCAbGp2mocGBbpqpik3X1Thw2S9QU0i6XO2987h1bGgHY1CxCIAiFTu60v8wwAxPLW1VsYrUoNjdXRLcIBjx543N/+3WrqCm3c+3qmmwvRRCENFO0FkH/+NKG1sejrXlh5lAgqDk/mD8DXkqsFt509YqcrXcQBGH5KGIhSI9FAEbAeGDCy4jHFz52ZWQSXyCYN0IgCELxUJRC4A8EGfJ402YRbFlRBcwNGM/2GMoP15AgCMVDUQrBhSEPWkNrTVlann9rlJ5DXQNGDcGGBrEIBEHILYpSCA6cHgTg5o31aXn+pkoHNeUlc2YTdLnc1Dvt1C5zuqogCMJSKUoh2H/axYZGJ6vr0pPPr5RiS3PlAteQxAcEQchFik4IpmcCPH9uiPa2xrS+ztYVlZzumyAYGlLT5fJIfEAQhJyk6OoIXjg/jNcfTLsQbFlRhccXoHt0CqfDxrDHJxaBIAg5SdEJwf5TLhw2CzdtSE98wCSy1URteQkg7RoEQchNik8ITg/wug31lJakt81D5LSyxtDMAxECQRBykaKKEVwenqTL5Um7WwigwmFjVW0ZJ/sm6HJ5sNsstNamJ11VEARhKRSVRXDgjAsgI0IARsD4VN8EU74AGxqcWC3SrkEQhNyjqCyC/adctNaUZazx25YVlZwb9HCyb0LcQoIg5CxFIwQ+f5Bfdw3RvqUxY43UtqyoIhDUdI9O5U3XUUEQio+iEYKXLo3g9voz5haC2VYTQM7PKRYEoXgpGiHYf9qFzaK4JU1tJaKxvsFJidWwPsQ1JAhCrlI8QnDKxfVra6ksLcnYa5ZYLWEBWN8griFBEHKTohCCgfFpXu0dp31L5txCJteuqWFDoxOno6gStARByCOK4tPpwBmj22gm4wMmn3rbNiZ9/oy/riAIQrIUhRDsP+2isdLBtpVVGX/tCoeNCrEGBEHIYQreNRTUmmfOuLh9c+bSRgVBEPKJgheC82NBRidnshIfEARByAcKXgiODQZQCvZsasj2UgRBEHKStAmBUqpUKfWiUuoVpdQJpdRfh46vV0q9oJQ6q5R6SCmV1tmNx1wBdq6qkRGRgiAIMUinReAFXq+13gnsAt6slLoJ+DzwRa31JmAEeH+6FjDi8XFuLP1DaARBEPKZtAmBNnCHHpaEvjTweuCHoeMPAvemaw3Pnh1Eg8QHBEEQ4pDWGIFSyqqUOgIMAE8AXcCo1tpMrL8CtKbr9fefduEsgZ2ratL1EoIgCHlPWhPctdYBYJdSqgZ4GNia7LVKqfuB+wGam5vp7OxMfQETPm5p1jxzYH/q1xYYbrd7cXtYYMg+zCJ7YSD7kKGCMq31qFLqaeBmoEYpZQtZBauA7hjXPAA8ALB7927d0dGR8ut2dEBnZyeLubbQkH0wkH2YRfbCQPYhvVlDjSFLAKVUGXAX8BrwNPCO0Gn3AfvStQZBEAQhMem0CFYCDyqlrBiC832t9c+UUq8C31NKfRZ4GfhWGtcgCIIgJCBtQqC1PgpcG+X4OeDGdL2uIAiCkBoFX1ksCIIgxEeEQBAEocgRIRAEQShyRAgEQRCKHBECQRCEIkdprbO9hoQopVzAxUVe3gAMLuNy8hXZBwPZh1lkLwwKeR/Waq0TNlvLCyFYCkqpQ1rr3dleR7aRfTCQfZhF9sJA9kFcQ4IgCEWPCIEgCEKRUwxC8EC2F5AjyD4YyD7MInthUPT7UPAxAkEQBCE+xWARCIIgCHEoaCFQSr1ZKXVKKXVWKfXJbK8nUyil/lUpNaCUOh5xrE4p9YRS6kzo39psrjETKKVWK6WeVkq9qpQ6oZT6cOh4Ue2FUqpUKfWiUuqV0D78dej4eqXUC6H3x0NKKXu215oJQpMTX1ZK/Sz0uCj3IZKCFYJQ++uvAm8BtgHvUUpty+6qMsa3gWG7lo8AAAUbSURBVDfPO/ZJ4Emt9WbgydDjQscPfExrvQ24Cfjvob+BYtsLL/B6rfVOYBfwZqXUTcDngS9qrTcBI8D7s7jGTPJhjNkoJsW6D2EKVggwWl2f1Vqf01r7gO8Be7O8poygtT4ADM87vBd4MPT9g8C9GV1UFtBa92qtXwp9P4Hx5m+lyPZCG7hDD0tCXxp4PfDD0PGC3wcApdQq4G3AN0OPFUW4D/MpZCFoBS5HPL4SOlasNGute0Pf9wHN2VxMplFKrcOYj/ECRbgXIXfIEWAAeALoAkZDI2OheN4fXwL+DAiGHtdTnPswh0IWAiEG2kgVK5p0MaVUBfAj4CNa6/HInxXLXmitA1rrXRhzwm8EtmZ5SRlHKXU3MKC1PpztteQaGRlenyW6gdURj1eFjhUr/UqplVrrXqXUSow7w4JHKVWCIQL/qbX+cehwUe4FgNZ6VCn1NHAzUKOUsoXuhovh/XErcI9S6q1AKVAFfJni24cFFLJFcBDYHMoIsAPvBh7J8pqyySPAfaHv7wP2ZXEtGSHk//0W8JrW+gsRPyqqvVBKNSqlakLflwF3YcRLngbeETqt4PdBa/3nWutVWut1GJ8HT2mtf4ci24doFHRBWUj5vwRYgX/VWv9tlpeUEZRS3wU6MLoq9gN/CfwE+D6wBqOT6zu11vMDygWFUuo24BngGLM+4b/AiBMUzV4opa7BCIJaMW7+vq+1/v+UUhswkijqgJeB39Vae7O30syhlOoAPq61vruY98GkoIVAEARBSEwhu4YEQRCEJBAhEARBKHJECARBEIocEQJBEIQiR4RAEAShyBEhEAoepdSnQl03jyqljiilXpftNc1HKfUmpdRfhzqj/iLb6xGKi0KuLBYElFI3A3cD12mtvUqpBiAX2wzvwShs2gM8m+W1CEWGWARCobMSGDQLhLTWg1rrHgCl1PVKqf1KqcNKqcdC7SZQSn1AKXUw1L//R0qp8tDx31ZKHQ8dPxA6VqqU+jel1LFQj/s7Qsffq5T6sVLql6G5B38fbXFKqXeFmsH9CUbx4zeA9ymlirkKXsgwUlAmFDShhnPPAuXAr4CHtNb7Qz2I9gN7tdYupdS7gDdprf9AKVWvtR4KXf9ZoF9r/RWl1DHgzVrrbqVUTahvz8eAq0PXbQUeB9owWhh8BqPjqRc4Bdymtb7MPEKtMJ7TWt+ilHoSuDfUNlsQMoK4hoSCRmvtVkpdj+FyuQN4SBnT6g4B24EnjM9hrIDZmnp7SABqgArgsdDx54BvK6W+D5gN7G4DvhJ6rZNKqYsYQgDG8JsxAKXUq8Ba5rZGN2kDzoW+d4oICJlGhEAoeLTWAaAT6Azd1d8HHAZOaK1vjnLJtzHuyl9RSr0Xo28TWus/CgWa3wYcDglMPCL71QSI8n5TSh3C6AllC4nFypCr6ENa62eS/iUFYQlIjEAoaJRSW5RSmyMO7cJoNHcKaAwFk1FKlSilrg6dUwn0htxHvxPxXBu11i9orT8DuDDanD9jnqOUasNoZHcq2fVprXcDj2JMTft74FNa610iAkImEYtAKHQqgK+E2jD7gbPA/Vprn1LqHcA/KaWqMd4LXwJOAJ/G6FDqCv1bGXqufwiJisKYdfwKcBL4l5Cl4QfeG8pOSmWN12EEi/9f4AsJzhWEZUeCxYIgCEWOuIYEQRCKHBECQRCEIkeEQBAEocgRIRAEQShyRAgEQRCKHBECQRCEIkeEQBAEocgRIRAEQShy/n9qcmOcv5z8RwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "plt.plot(np.arange(len(save_scores)), save_scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Season #')\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 1, reward: 1.0\n",
      "Episode: 2, reward: 1.0\n",
      "Episode: 3, reward: 0.0\n",
      "Episode: 4, reward: 0.0\n",
      "Episode: 5, reward: 1.0\n",
      "Episode: 6, reward: 0.0\n",
      "Episode: 7, reward: 1.0\n",
      "Episode: 8, reward: 1.0\n",
      "Episode: 9, reward: 0.0\n",
      "Episode: 10, reward: 0.0\n"
     ]
    }
   ],
   "source": [
    "episode = 10\n",
    "scores_window = deque(maxlen=100)  # last 100 scores\n",
    "env = KukaDiverseObjectEnv(renders=False, isDiscrete=False, removeHeightHack=False, maxSteps=20, isTest=True)\n",
    "env.cid = p.connect(p.DIRECT)\n",
    "# load the model\n",
    "checkpoint = torch.load(PATH)\n",
    "policy.load_state_dict(checkpoint['policy_state_dict'])\n",
    "\n",
    "# evaluate the model\n",
    "for e in range(episode):\n",
    "    rewards = eval_policy(envs=env, policy=policy)\n",
    "    reward = np.sum(rewards,0)\n",
    "    print(\"Episode: {0:d}, reward: {1}\".format(e+1, reward), end=\"\\n\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}