{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Double DQN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adapted from fantastic https://github.com/higgsfield/RL-Adventure repository for learning and compare different DRL implementations in Pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math, random\n",
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.autograd as autograd \n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Use Cuda</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "USE_CUDA = torch.cuda.is_available()\n",
    "Variable = lambda *args, **kwargs: autograd.Variable(*args, **kwargs).cuda() if USE_CUDA else autograd.Variable(*args, **kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Replay Buffer</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import deque\n",
    "\n",
    "class ReplayBuffer(object):\n",
    "    def __init__(self, capacity):\n",
    "        self.buffer = deque(maxlen=capacity)\n",
    "    \n",
    "    def push(self, state, action, reward, next_state, done):\n",
    "        state      = np.expand_dims(state, 0)\n",
    "        next_state = np.expand_dims(next_state, 0)\n",
    "            \n",
    "        self.buffer.append((state, action, reward, next_state, done))\n",
    "    \n",
    "    def sample(self, batch_size):\n",
    "        state, action, reward, next_state, done = zip(*random.sample(self.buffer, batch_size))\n",
    "        return np.concatenate(state), action, reward, np.concatenate(next_state), done\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.buffer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Cart Pole Environment</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2020-05-12 11:36:24,758] Making new env: CartPole-v0\n",
      "C:\\Anaconda\\envs\\py35\\lib\\site-packages\\gym\\envs\\registration.py:17: PkgResourcesDeprecationWarning: Parameters to load are deprecated.  Call .resolve and .require separately.\n",
      "  result = entry_point.load(False)\n"
     ]
    }
   ],
   "source": [
    "env_id = \"CartPole-v0\"\n",
    "env = gym.make(env_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Epsilon greedy exploration</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsilon_start = 1.0\n",
    "epsilon_final = 0.01\n",
    "epsilon_decay = 500\n",
    "\n",
    "epsilon_by_frame = lambda frame_idx: epsilon_final + (epsilon_start - epsilon_final) * math.exp(-1. * frame_idx / epsilon_decay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x251ecf21a58>]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGlNJREFUeJzt3X10XPV95/H3d0YayXqyHi3bso1sYzsxIWCigIGchCSEAG1xuyfb2G02hCSl25Zm2XR3D5z0QMv+0yTttpuNm4RNQ9qE4BCaJl5q4nYTSDcUuxY4gB9BNtiWH5D8/CDLsqTv/jFXZiyPpLE88tW99/M6R0dzf/PT6Ht15Y9/+s3v3mvujoiIxEsq7AJERKT4FO4iIjGkcBcRiSGFu4hIDCncRURiSOEuIhJDCncRkRhSuIuIxJDCXUQkhkrC+saNjY3e2toa1rcXEYmkF1988aC7N43VL7Rwb21tpb29PaxvLyISSWa2q5B+mpYREYkhhbuISAwp3EVEYkjhLiISQwp3EZEYGjPczexbZtZlZptGeN7M7Ctm1mFmr5jZdcUvU0RELkYhI/dvA7eP8vwdwILg417ga5deloiIXIoxw93d/wU4PEqXZcDfedY6oNbMZhSrwOE2vHmYL/5kG7o9oIjIyIox594C7MnZ7gzaLmBm95pZu5m1d3d3j+ubvbznKF97bgfHT/eP6+tFRJKgGOFuedryDqvd/VF3b3P3tqamMc+ezauhKgPAoVNnxvX1IiJJUIxw7wRm52zPAvYV4XXzqq8sA+Dwqb6J+hYiIpFXjHBfDXwyWDWzFDjm7vuL8Lp5NVQOjdwV7iIiIxnzwmFm9gRwC9BoZp3Aw0ApgLt/HVgD3Al0AD3APRNVLEB9EO4auYuIjGzMcHf3FWM878AfFK2iMSjcRUTGFrkzVMtL01Rm0hw6qXAXERlJ5MIdoK4yw2GtlhERGVEkw72hMqM3VEVERhHJcK+vzGjOXURkFBEN9zKFu4jIKCIZ7g1V2ZG7ri8jIpJfJMO9vjLDmf5BevoGwi5FRGRSimy4g9a6i4iMJJLhrksQiIiMLpLh/vbIXWvdRUTyiWS4NwRXhtRZqiIi+UUy3OurNOcuIjKaSIZ7ZSZNpiSlcBcRGUEkw93MdAkCEZFRRDLcQZcgEBEZTaTDXSN3EZH8IhvuDbrsr4jIiKIb7lVlWgopIjKCyIZ7Y1UZPX0D9PT1h12KiMikE9lwb6rOnsh08IRG7yIiw0U+3LtP9oZciYjI5BPZcG8MzlLtPqE3VUVEhotsuJ8buSvcRUQuENlwb6gsI2XQrRUzIiIXiGy4p1NGfWWZRu4iInlENtwhO++ucBcRuVCkw72puozukwp3EZHhIh/uBzVyFxG5QLTDvSo7cnf3sEsREZlUoh3u1WX09Q9yvFeXIBARyRX5cAc4qHl3EZHzFBTuZna7mW03sw4zeyDP83PM7Fkz22hmr5jZncUv9UJNVTqRSUQknzHD3czSwErgDmAxsMLMFg/r9sfAk+6+BFgO/HWxC82nUWepiojkVcjI/Xqgw913unsfsApYNqyPAzXB46nAvuKVODKN3EVE8isk3FuAPTnbnUFbrj8BPmFmncAa4A/zvZCZ3Wtm7WbW3t3dPY5yzzd1SimladOcu4jIMIWEu+VpG772cAXwbXefBdwJfMfMLnhtd3/U3dvcva2pqeniqx0mlTIadAkCEZELFBLuncDsnO1ZXDjt8hngSQB3fwEoBxqLUeBYdJaqiMiFCgn3DcACM5trZhmyb5iuHtZnN/BhADN7J9lwv/R5lwI0VZfRdVzhLiKSa8xwd/d+4D5gLbCV7KqYzWb2iJndFXT7I+B3zOxl4AngU36ZThttrimjS9MyIiLnKSmkk7uvIftGaW7bQzmPtwA3F7e0wjTXlHPo1BnODgxSmo70OVkiIkUT+TRsrinHHY3eRURyRD7cp9eUA3DgmG6ULSIyJPLh3hyE+1vHFe4iIkMiH+7Tp2rkLiIyXOTDva6ilExJSiN3EZEckQ93M6O5powDCncRkXMiH+6QfVNVI3cRkbfFItyba8p5S2epioicE4twn15TzoFjvbqXqohIIBbh3lxTzumzA7qXqohIIB7hPlVr3UVEcsUi3HWWqojI+eIV7hq5i4gAMQn3aTXZe6l2KdxFRICYhHt5aZq6ilKN3EVEArEId8iumNGcu4hIVmzCvaV2CvuOKtxFRCBG4T6zdgp7j54OuwwRkUkhVuF+7PRZTp7RiUwiIrEJ95a6KQDs1+hdRCRG4V6bXeveqXAXEYlPuM+szY7c9yncRUTiE+7TqsspSRl7jyjcRURiE+7plDF9arlG7iIixCjcQWvdRUSGxC7ctdZdRCRm4T6zdgoHjvfSPzAYdikiIqGKVbi31E1hYNB564TupyoiyRarcNdySBGRrFiF+9CJTFoOKSJJV1C4m9ntZrbdzDrM7IER+vymmW0xs81m9r3illmYoZG73lQVkaQrGauDmaWBlcBHgE5gg5mtdvctOX0WAA8CN7v7ETObNlEFj6YiU0JdRSmdGrmLSMIVMnK/Huhw953u3gesApYN6/M7wEp3PwLg7l3FLbNwc+or6DzSE9a3FxGZFAoJ9xZgT852Z9CWayGw0MyeN7N1ZnZ7sQq8WLPrK9h9WOEuIslWSLhbnjYftl0CLABuAVYA3zSz2gteyOxeM2s3s/bu7u6LrbUgVzRUsPfIaa11F5FEKyTcO4HZOduzgH15+vzY3c+6+xvAdrJhfx53f9Td29y9rampabw1j2pOfQX9g85+3U9VRBKskHDfACwws7lmlgGWA6uH9fkR8EEAM2skO02zs5iFFmpOfSUAuw5pakZEkmvMcHf3fuA+YC2wFXjS3Teb2SNmdlfQbS1wyMy2AM8C/9XdD01U0aOZ01ABoHl3EUm0MZdCArj7GmDNsLaHch478PngI1TTa8rJpFPsOnwq7FJEREITqzNUIXtd91l1U9ijkbuIJFjswh2yUzOalhGRJItnuNdXsOtQD9nZIhGR5IltuJ/o7efY6bNhlyIiEorYhjtoOaSIJFc8w13LIUUk4eIZ7udG7loOKSLJFMtwr8iUML2mnJ0HFe4ikkyxDHeAeU2V7OxWuItIMsU23Oc2VrKz+6SWQ4pIIsU23Oc1VXG8t59Dp/rCLkVE5LKLcbhnrw6pqRkRSaLYhvv8xioA3jh4MuRKREQuv9iGe0vdFDIlKY3cRSSRYhvu6ZTR2lDBDoW7iCRQbMMdYF5jFTs1LSMiCRTvcG+qZPehHs7qZtkikjAxD/cq+gddN+4QkcSJebhrOaSIJFOsw31+U3Y55OtdmncXkWSJdbhPnVLKjKnlvPbWibBLERG5rGId7gALm6vZfkDhLiLJEvtwXzS9mo7uk/RrxYyIJEj8w725mr7+QXZpxYyIJEj8w316NYCmZkQkUWIf7ldOq8JM4S4iyRL7cC8vTdPaUKkVMyKSKLEPd4CFzVVsV7iLSIIkItwXNVfz5sFT9J4dCLsUEZHLIhHhvnB6NYMOHTpTVUQSIhHh/s4ZNQBs3X885EpERC6PgsLdzG43s+1m1mFmD4zS72Nm5mbWVrwSL93chkoqM2k271O4i0gyjBnuZpYGVgJ3AIuBFWa2OE+/auBzwPpiF3mpUilj8cwaNu09FnYpIiKXRSEj9+uBDnff6e59wCpgWZ5+/x34EtBbxPqK5qqZU9my/zgDgx52KSIiE66QcG8B9uRsdwZt55jZEmC2uz9dxNqK6l0tU+npG+CNg7q2u4jEXyHhbnnazg1/zSwF/CXwR2O+kNm9ZtZuZu3d3d2FV1kE72rJvqm6eZ+mZkQk/goJ905gds72LGBfznY18C7gOTN7E1gKrM73pqq7P+rube7e1tTUNP6qx2F+UxWZkpTm3UUkEQoJ9w3AAjOba2YZYDmweuhJdz/m7o3u3ururcA64C53b5+QisepNJ3indOr2bRXK2ZEJP7GDHd37wfuA9YCW4En3X2zmT1iZndNdIHFdFXLVDbtO4a73lQVkXgrKaSTu68B1gxre2iEvrdcelkT410zp/K99bvZfbiHKxoqwy5HRGTCJOIM1SHXzq4F4Jd7joZciYjIxEpUuC9srqIik+alXUfCLkVEZEIlKtxL0imumVXLS7s1cheReEtUuAMsmVPL1v3HOd2ny/+KSHwlLtyvm1NH/6Dzqta7i0iMJS7cr52TfVN1427Nu4tIfCUu3BuryriioYKXFO4iEmOJC3eAJbOzb6rqZCYRiatEhvt7rqij+8QZ9hw+HXYpIiITIpHhvnReAwDrdh4KuRIRkYmRyHC/cloVjVUZXlC4i0hMJTLczYwb5jWwbuchzbuLSCwlMtwhOzWz/1gvuw/3hF2KiEjRJTbcb5xXD2jeXUTiKbHhPr8pO+++bufhsEsRESm6xIb70Lz7Czs07y4i8ZPYcAd435WNHDjey+tdJ8MuRUSkqBId7h9YmL1J98+3d4dciYhIcSU63GfWTmFhcxXPvdYVdikiIkWV6HAHuGXRNDa8cYRTZ/rDLkVEpGgSH+4fWNhE38AgL+zQkkgRiY/Eh3tbax0VmbSmZkQkVhIf7mUlaW6a38Cz27q1JFJEYiPx4Q5w2+Lp7D16ms37joddiohIUSjcgVsXN5My+MmmA2GXIiJSFAp3oL4yww1zG3hm0/6wSxERKQqFe+COq6ezo/sUHV0nwi5FROSSKdwDty2eDsAzr2pqRkSiT+EemD61nCVzavnHVzU1IyLRp3DP8evXtrDtwAm27teqGRGJNoV7jl+7ZiYlKeMfNu4NuxQRkUtSULib2e1mtt3MOszsgTzPf97MtpjZK2b2UzO7ovilTrz6ygy3LJrGjzbuZWBQJzSJSHSNGe5mlgZWAncAi4EVZrZ4WLeNQJu7vxt4CvhSsQu9XP7ddS10nTjD8x0Hwy5FRGTcChm5Xw90uPtOd+8DVgHLcju4+7PuPnSn6XXArOKWefl86B3TqCkv4akXO8MuRURk3AoJ9xZgT852Z9A2ks8Az1xKUWEqL03zG0ta+MmmAxw6eSbsckRExqWQcLc8bXknpM3sE0Ab8OURnr/XzNrNrL27e/Le/egTS6+gb2CQJ9s1eheRaCok3DuB2Tnbs4B9wzuZ2a3AF4C73D3vkNfdH3X3Nndva2pqGk+9l8WC5mqWzqvn8fW79MaqiERSIeG+AVhgZnPNLAMsB1bndjCzJcA3yAZ7LC6M/h+WttJ55DQ/13XeRSSCxgx3d+8H7gPWAluBJ919s5k9YmZ3Bd2+DFQBPzCzX5rZ6hFeLjJuu6qZadVlPPb8m2GXIiJy0UoK6eTua4A1w9oeynl8a5HrCl1pOsWn3zeXP3tmG692HuPqWVPDLklEpGA6Q3UUv33DHKrLS/jr5zrCLkVE5KIo3EdRXV7KJ2+8gp9sPkBH18mwyxERKZjCfQz33DyXTDql0buIRIrCfQyNVWXcfVMr/7BxL9sP6EYeIhINCvcC/P4t86kqK+HLa7eFXYqISEEU7gWorcjwHz8wn/+7tYsNbx4OuxwRkTEp3Av06Zvn0lxTxp/+n806a1VEJj2Fe4GmZNL88a8sZtPe43x33a6wyxERGZXC/SL86rtn8L4rG/nztdvpOt4bdjkiIiNSuF8EM+ORZVdxpn+Qh1dvxl3TMyIyOSncL9K8piru/8gCntl0gB++pHutisjkpHAfh999/3yub63n4dWb2XO4Z+wvEBG5zBTu45BOGX/xm9cA8LlVGznTPxByRSIi51O4j9Ps+gq+/LF3s3H3UR7+sebfRWRyUbhfgjuunsEffHA+qzbs4bvrd4ddjojIOQVdz11G9vmPLGLr/hM8/ONNTKsu46NXTQ+7JBERjdwvVTplfPW3lnDN7Fr+8ImN/OuOg2GXJCKicC+GikwJj33qvbQ2VPDZv23n+Q4FvIiES+FeJLUVGb772RuYXVfBPY9t4J82Hwi7JBFJMIV7EU2rLuf7v7uUxTNr+L3HX+Jbv3hDq2hEJBQK9yKrrcjw+Gdv4MPvmMYjT2/hv/zgFXrPah28iFxeCvcJUFlWwtc/8R7uv3UBf/9SJ8u++jyb9x0LuywRSRCF+wRJpYz7b13IY/e8l8M9ffz6yuf5Xz99XWezishloXCfYB9cNI1/uv/93HbVdP7in1/jo3/5L/xs21thlyUiMadwvwzqKjOs/K3r+PY97yWVMj797XZWPLqOF3YcCrs0EYkpC2s1R1tbm7e3t4fyvcPU1z/Id9ft4ms/30H3iTNc31rPPTe3cuviZkrT+r9WREZnZi+6e9uY/RTu4eg9O8Cqf9vN//5/b7D36GmmVZfx8ffOZtm1M7lyWnXY5YnIJKVwj4iBQefZbV08vn4Xz73WjTssaq7mzqtn8KF3TOOqmTWkUhZ2mSIySSjcI+it47088+p+/vHV/bTvOoI71FWUctP8Rm6c38C1s2tZNL1a0zciCaZwj7iuE738a8chftFxkF+8fpADwQ25MyUprppZw9UtU7lyWhXzm7IfzTVlmGmELxJ3CvcYcXf2HD7Ny51HeaXzKC/vOcaW/cc5eab/XJ+qshJm1U1hZu0UZtaWM2Nq9nNzdTl1lRnqKzPUVpRSVpIOcU9E5FIVGu4FXc/dzG4H/ieQBr7p7n827Pky4O+A9wCHgI+7+5sXW7TkZ2bMaahgTkMFv3bNTCAb+F0nzrCj6yQd3SfZ0XWSziOn2Xesl5d2H+Foz9m8r1VVVkJdZSl1FRkqMyVUlpVQWZbOfs4MfS6hoixNWUma0rRRVpIiU5Iik85uZ4LtspIUpensRzplpMxIp4y0GakU59qG2lOG/roQuUzGDHczSwMrgY8AncAGM1vt7ltyun0GOOLuV5rZcuCLwMcnomDJMjOaa8pprinnpisbL3i+p6+f/cd6eet4L0d7znL4VB9HTvVxpOcsR3r6OHyqj1Nn+tl79DQ9ff2cOtPPqTMDnJ7g6+CkjAv+I7Ag9Idy34L9G/pvINtu5x7ntlvedsv5utH7Tbr/aiZZQZOsnEk3OBhvNZ/78IJzA7WJUsjI/Xqgw913ApjZKmAZkBvuy4A/CR4/BXzVzMx1ScTQVGRKzs3HX4yBQaenr5+evgH6+gc50z/I2YFB+voH6Rv6PKz97MAgA+4MDjoDg86Ak33s2W13Z2CQt/uc19dx59zVMx2y2wTbDkO/RNkuOe3BE47nPD7/6znv6/2815psv5yT7Z/L5KqGSVeQX0JBU6eUFrGS/AoJ9xZgT852J3DDSH3cvd/MjgENgO5aETHplFFdXkp1+cT/8onIxClkTV2+vzyG/5dVSB/M7F4zazez9u7u7kLqExGRcSgk3DuB2Tnbs4B9I/UxsxJgKnB4+Au5+6Pu3ububU1NTeOrWERExlRIuG8AFpjZXDPLAMuB1cP6rAbuDh5/DPiZ5ttFRMIz5px7MId+H7CW7FLIb7n7ZjN7BGh399XA3wDfMbMOsiP25RNZtIiIjK6gde7uvgZYM6ztoZzHvcC/L25pIiIyXrpIiYhIDCncRURiSOEuIhJDoV04zMy6gV3j/PJGkneClPY5GbTPyXAp+3yFu4+5ljy0cL8UZtZeyFXR4kT7nAza52S4HPusaRkRkRhSuIuIxFBUw/3RsAsIgfY5GbTPyTDh+xzJOXcRERldVEfuIiIyisiFu5ndbmbbzazDzB4Iu57xMrPZZvasmW01s81m9p+C9noz+2czez34XBe0m5l9JdjvV8zsupzXujvo/7qZ3T3S95wszCxtZhvN7Olge66ZrQ/q/35wgTrMrCzY7gieb815jQeD9u1m9tFw9qQwZlZrZk+Z2bbgeN8Y9+NsZv85+L3eZGZPmFl53I6zmX3LzLrMbFNOW9GOq5m9x8xeDb7mK2YXeRsqd4/MB9kLl+0A5gEZ4GVgcdh1jXNfZgDXBY+rgdeAxcCXgAeC9geALwaP7wSeIXvt/KXA+qC9HtgZfK4LHteFvX9j7Pvnge8BTwfbTwLLg8dfB34vePz7wNeDx8uB7wePFwfHvgyYG/xOpMPer1H292+BzwaPM0BtnI8z2Zv3vAFMyTm+n4rbcQbeD1wHbMppK9pxBf4NuDH4mmeAOy6qvrB/QBf5w7wRWJuz/SDwYNh1FWnffkz2PrXbgRlB2wxge/D4G8CKnP7bg+dXAN/IaT+v32T7IHs/gJ8CHwKeDn5xDwIlw48x2SuR3hg8Lgn62fDjnttvsn0ANUHQ2bD22B5n3r4zW31w3J4GPhrH4wy0Dgv3ohzX4LltOe3n9SvkI2rTMvlu+dcSUi1FE/wZugRYDzS7+36A4PO0oNtI+x61n8lfAf8NGAy2G4Cj7t4fbOfWf97tG4Gh2zdGaZ/nAd3AY8FU1DfNrJIYH2d33wv8ObAb2E/2uL1IvI/zkGId15bg8fD2gkUt3Au6nV+UmFkV8PfA/e5+fLSuedp8lPZJx8x+Fehy9xdzm/N09TGei8w+kx2JXgd8zd2XAKfI/rk+ksjvczDPvIzsVMpMoBK4I0/XOB3nsVzsPl7yvkct3Au55V9kmFkp2WB/3N1/GDS/ZWYzgudnAF1B+0j7HqWfyc3AXWb2JrCK7NTMXwG1lr09I5xf/0i3b4zSPncCne6+Pth+imzYx/k43wq84e7d7n4W+CFwE/E+zkOKdVw7g8fD2wsWtXAv5JZ/kRC88/03wFZ3/x85T+XesvBusnPxQ+2fDN51XwocC/7sWwvcZmZ1wYjptqBt0nH3B919lru3kj12P3P33waeJXt7Rrhwn/PdvnE1sDxYZTEXWED2zadJx90PAHvMbFHQ9GFgCzE+zmSnY5aaWUXwez60z7E9zjmKclyD506Y2dLgZ/jJnNcqTNhvSIzjDYw7ya4s2QF8Iex6LmE/3kf2z6xXgF8GH3eSnWv8KfB68Lk+6G/AymC/XwXacl7r00BH8HFP2PtW4P7fwturZeaR/UfbAfwAKAvay4PtjuD5eTlf/4XgZ7Gdi1xFEMK+Xgu0B8f6R2RXRcT6OAN/CmwDNgHfIbviJVbHGXiC7HsKZ8mOtD9TzOMKtAU/vx3AVxn2pvxYHzpDVUQkhqI2LSMiIgVQuIuIxJDCXUQkhhTuIiIxpHAXEYkhhbuISAwp3EVEYkjhLiISQ/8fSrxxVFxHn1MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([epsilon_by_frame(i) for i in range(10000)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Double Deep Q Network</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQN(nn.Module):\n",
    "    def __init__(self, num_inputs, num_actions):\n",
    "        super(DQN, self).__init__()\n",
    "        \n",
    "        self.layers = nn.Sequential(\n",
    "            nn.Linear(env.observation_space.shape[0], 128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, 128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, env.action_space.n)\n",
    "        )\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return self.layers(x)\n",
    "    \n",
    "    def act(self, state, epsilon):\n",
    "        if random.random() > epsilon:\n",
    "            state   = Variable(torch.FloatTensor(state).unsqueeze(0), requires_grad=False)\n",
    "            q_value = self.forward(state)\n",
    "            action  = q_value.max(1)[1].item()\n",
    "        else:\n",
    "            action = random.randrange(env.action_space.n)\n",
    "        return action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "current_model = DQN(env.observation_space.shape[0], env.action_space.n)\n",
    "target_model  = DQN(env.observation_space.shape[0], env.action_space.n)\n",
    "\n",
    "if USE_CUDA:\n",
    "    current_model = current_model.cuda()\n",
    "    target_model  = target_model.cuda()\n",
    "    \n",
    "optimizer = optim.Adam(current_model.parameters())\n",
    "\n",
    "replay_buffer = ReplayBuffer(1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Synchronize current policy net and target net</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_target(current_model, target_model):\n",
    "    target_model.load_state_dict(current_model.state_dict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "update_target(current_model, target_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Computing Temporal Difference Loss</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_td_loss(batch_size):\n",
    "    state, action, reward, next_state, done = replay_buffer.sample(batch_size)\n",
    "\n",
    "    state      = Variable(torch.FloatTensor(np.float32(state)))\n",
    "    next_state = Variable(torch.FloatTensor(np.float32(next_state)))\n",
    "    action     = Variable(torch.LongTensor(action))\n",
    "    reward     = Variable(torch.FloatTensor(reward))\n",
    "    done       = Variable(torch.FloatTensor(done))\n",
    "\n",
    "    q_values      = current_model(state)\n",
    "    next_q_values = current_model(next_state)\n",
    "    next_q_state_values = target_model(next_state) \n",
    "\n",
    "    q_value       = q_values.gather(1, action.unsqueeze(1)).squeeze(1) \n",
    "    next_q_value = next_q_state_values.gather(1, torch.max(next_q_values, 1)[1].unsqueeze(1)).squeeze(1)\n",
    "    expected_q_value = reward + gamma * next_q_value * (1 - done)\n",
    "    \n",
    "    loss = (q_value - Variable(expected_q_value.data)).pow(2).mean()\n",
    "        \n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    \n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot(frame_idx, rewards, losses):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s. reward: %s' % (frame_idx, np.mean(rewards[-10:])))\n",
    "    plt.plot(rewards)\n",
    "    plt.subplot(132)\n",
    "    plt.title('loss')\n",
    "    plt.plot(losses)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Training</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAv4AAAE/CAYAAAA+Occ1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3XmYHGd5Lvz7qV5mlzSSRsaSLQsbQ8AGHBBLQgCzhTUBshDIwnpiOJCT5eQ7ByeQwEdCPkJCAmSBzwRik8QGDvsWlhhsbLANMrblFSxLo93S7FtvtTznj6q3u7qnu6eXqu6e7vt3Xbo0090zXTMaTT391P0+r6gqiIiIiIiov1ndPgAiIiIiIoofC38iIiIiogHAwp+IiIiIaACw8CciIiIiGgAs/ImIiIiIBgALfyIiIiKiAcDCv0+IyGNE5A4RWRGR3+/28VC8RGRaRJ7f7eMgIuon/N1K/Y6Ff//43wBuUNUJVf1wtw+mkohcJSI/ERFPRF5f5f4/EpGHRWRJRD4hIkOh+/aJyHdFJCMiD1T+Um7nYweBiDwn+B4sich0lfu/KyIzIrIsIneJyMtD94mIvENEjgX3f0pEttR5rstE5KbguU6IyJ/H9GURERFRk1j4948LANxb604RSXTwWKq5C8BbAfy48g4ReSGAKwE8D8A+ABcC+H9DD7kOwB0AdgB4B4DPishUux/bDBFJNvsxUYjoedcAfALA/6px/x8AOFdVtwC4AsC/i8i5wX2vBfA7AJ4BYDeAEQD/UOe5rgXwPQDbATwbwH8XkV9u+ysgIiKitrHw7wMi8h0AzwHwjyKyKiKPFpGrReQjIvJ1EVkD8BwReWkQB1oWkeMi8u7Q59gnIioibwjuWxCRt4jIU0TkoIgsisg/VjzvG0Xk/uCx3xSRC2odo6r+k6peDyBX5e7XAfi4qt6rqgsA/gLA64PneDSAJwF4l6pmVfVzAO4G8KsRfOxG39dpEXm7iBwEsCYiSRHZLSKfCzrkR0ysSkSGRSQrIjuD998pIo7pjovIX4rIB4O3G/l3eJOIHAPwneD23xGRoyIyJyLvaOT4DVX9oar+G4DDNe4/qKqOeRdACsD5wfu/BP/7e1xVVwH8NYDfEJHRGk+3D8B/qKqrqg8BuBnAJc0cLxFRt4nIkIh8UEROBX8+aK4mi8hOEflqcF6cD65yWsF9bxeRk+LHbn8iIs/r7ldCVI6Ffx9Q1ecCuAnA76nquKr+NLjrNwG8F8AE/AJsDX4HdxuAl8Lvxr6i4tM9DcDFAH4DwAfhd8mfD794e5WIPBsAgo/7UwC/AmAqeP7rWvwSLoF/RcC4C8A5IrIjuO+wqq5U3H9JBB/biNfA/15tA+AB+ErwOfbAv8rwhyLyQlXNAfgR/C43ADwLwFH4nXLz/o3B2438OzwbwGMBvFBEHgfgI/A777vhX704zzxQRH5BRBab+JrWCU5iOQC3AbgBwAFzV/AHofeH4P+MVPNBAK8VkZSIPAbAzwH4r3aOjYioC94B4OkALgPwRABPBfDO4L4/BnAC/rnvHPjnQg1+5/0egKeo6gSAFwKY7uxhE9XHwr+/fUlVv6+qnqrmVPUGVb07eP8g/EL92RUf8xfBY78Fv0C9TlXPqupJ+MX9zwaPezOA/09V7w+6xX8F4LJ6Xf86xgEshd43b09Uuc/cPxHBxzbiw0G3OwvgKQCmVPU9qlpQ1cMAPgbg1cFjbwTw7CCe8wQAHw7eHw4+9iYAaPDf4d2quhY8768B+Kqqfk9V8wD+DP6LEASf72ZV3dbE17SOqr4M/vflJQC+qarm8/8ngP8WXInYCuDtwe21Ov5fDY43C+AB+FcLftTOsRERdcFvAXhPcP6bgR8h/Z3gPhvAuQAuUFVbVW9SVQXgwm+MPE5EUqo6HVz5JOoZLPz72/HwOyLyNCkt5FwC8BYAOys+5kzo7WyV98eDty8A8KHgUucigHn43eA9LRznKoDwglHz9kqV+8z9povfzsc2Ivw9vADAbvM1B1/3n8Lv+AB+4X85/HjR3QC+Db+gfzqAQ6o6CzT87xB+3t3h91V1DcBcE19DQ4IT2H/Cv8pgcvmfgP/C5Ab4a0i+G9x+ovLjRWQ7gG8AeA+AYfhxoReKyFujPlYiopjthn/V1jga3AYAfwPgEIBvichhEbkSAFT1EIA/BPBuAGfFH4awG0Q9hIV/f9OK968F8GUA56vqVgAfRXmMoxnHAbxZVbeF/oyo6g9a+Fz3wr+UajwRwBlVnQvuu1BEJiruvzeCj21E+Ht4HMCRiq95QlVfEtz/AwCPAfBKADeq6n0A9sKP89wY+jyN/DuEn/c0Spl7BPn6HU18Dc1KArgIAIKrEu9S1X2qeh78793J4E+lCwG4qvpJVXVU9QSAT8G/ikBEtJmcgt/sMfYGt0FVV1T1j1X1QvjroP6nyfKr6rWq+gvBxyr8dVFEPYOF/2CZADCvqjkReSr8NQCt+iiAPxGRSwBARLaKyK/XerCIpIPIiwBIBYthzc/fJwG8SUQeJyKT8HOUVwNAsF7hTgDvCj7mlfBjNJ+L4GOb9UMAy8HirRERSYjIpSLylOD5MgBuB/A2lAr9H8CPRYUL/2b/HT4L4GVBlj8Nv6Pe8P9dEbGC733Kf1eGg88DEfkZEXlx8PWkROS3EVqPICLbReQi8T0OwN/Bv/ztVXmqnwaf/zeD53wE/LUid1V5LBFRL7sOwDtFZEr8oQ1/DuDfAUBEXiYijxIRAbAMP+Ljir+fznODRcA5+FfJ3S4dP1FVLPwHy1sBvEdEVuD/EvtMq59IVb8Av5PxKRFZBnAPgBfX+ZBvwf8l+PMArgreflbwub4B4P3wYyRHgz/vCn3sqwHsB7AA4H0Afi3IXLb1sSLyWyLScPdfVV343Z3LABwBMAvgXwBsDT3sRvgF9g9D70/AH3FpNPXvoKr3wn8xcS387v8CQlEbEXmmiKzW+RTPgv/9/jr8rlUW/r8H4L8QezeAswBm4I/2/A1VNWNXdwYftwY/7/8JVb0q9NwfFZGPBse5DH+x9x8Fx3gn/J+L99b7+oiIetBfwh9ycBB+dPPHwW2AP9zgv+DHSW8B8M+qegP8fP/74J8bHgawC34clKhniL8ehYiIiIiI+hk7/kREREREA4CFPxERERHRAGDhT0REREQ0AFj4ExERERENABb+REREREQDINntAwCAnTt36r59+7p9GEREPen222+fVdWpbh9HN/E8QURUXTPniJ4o/Pft24cDBw50+zCIiHqSiBzt9jF0G88TRETVNXOOYNSHiIiIiGgAsPAnIiIiIhoALPyJiIiIiAYAC38iIiIiogHAwp+IiIiIaACw8CciIiIiGgAs/ImIiIiIBsCGhb+InC8i3xWR+0XkXhH5g+D27SLybRF5MPh7MrhdROTDInJIRA6KyJPi/iKIiIiIiKi+Rjr+DoA/VtXHAng6gLeJyOMAXAngelW9GMD1wfsA8GIAFwd/rgDwkciPmoiIiIiImrLhzr2qehrA6eDtFRG5H8AeAC8HcHnwsGsA3ADg7cHtn1RVBXCriGwTkXODz0MD7Ph8Bjcfmo39ec6fHMUvXLxz3e0Fx8PX7z6NrO1W/ThLgKc9cgf27Rwruz1nu7jl8Bwu2b0FuyaG133csbkMHM/DhVPj0XwBVZxdyeE795+FxvYMGxtOWXjp43cjnVzfL7j5wVkcX8g09HlGUgm87AnnIpmo33dYyzu4//Qy9u/b3tLxRuE7D5zBmeV8zfuf/egp7N42UvdzeJ7i6h9M47k/s2vdzxYREbXnR9PzuGT3FoymNyxpCQ0U/mEisg/AzwK4DcA5pphX1dMisit42B4Ax0MfdiK4razwF5Er4F8RwN69e1s4dNps/u7bP8UX7jgZ+/MkLMF973khhpKJsttvPjSDP/z0nRt+/JP2bsMrn3QeLto5hq8cPIWvHTyN5ZyD1/7cBXjPyy9d9/h3fuke5GwXn3nzz0X2NVT6p+8cwjW3NLwjd2y2DKfwvMeeU3Zbznbxun/9IVyv8ZclW0dSeM7P7Kr7mM//+ATe/ZX7cPBdv4ixoc7/Ql9YK+CNVx+o+5hXXLYbH3z1z9Z9zEMzq3jPV+/DxHCShT8RUYQeXsrh1z96C176+HPxT7/FZHkjGj6bisg4gM8B+ENVXRaRmg+tctu6ikBVrwJwFQDs37+/m41M6pCVnI2Ld43j3970tNie4/N3nMD7v/ETLGVs7NpSXvjPrhQAAF9468/j3K3ru7SZgoNv3XcGX/jxSfzZF+8BAIymE3jRJY/ADx6aw8NLuarP+fBSFlbt/w+RODy7hkt2b8HHX/eUWJ+nltnVPF72DzfjdJXvwcxKHq6neOdLH4uXPWF33c+znLPxi3//PRyeXcNzNnjOtYIL11PkbLcrhf9awQEAvOMlj8UvPXH913Xl5w/i4MmlDT/PgaMLANDVKxdERP3I/J6+//Ryl49k82jobCoiKfhF/3+o6ueDm8+YCI+InAvgbHD7CQDnhz78PACnojpg2rzyjofx4SQesXV9XCYqe7ePAgAWMjZ2bSl/noWMX/hffM4ExmsUkm959jje/KwLcd/pZRyfz+JZj96J0XQSr7nqVsyvFap+zPxaIfbC9OhcBpedvy3W7109O8fTEAHOrqyPvZjbLto1vuHxnbNlCBPDSRybW9vwOc0VBKeJKwlRyjseAGBqYqjq1/WkvZO48aczWM07NX+eAODA9AJ2jKWxb8dobMdKRETNe9eX7sGJhSw+/vruNNW6oZGpPgLg4wDuV9W/C931ZQCvC95+HYAvhW5/bTDd5+kAlpjvJwDI2x6GquTDozQ5mgYALGbWF+mLWRuphGAsnVh3X5iI4JLdW/GiSx9RzAxuH09jrkrh73mK+bUCMoXq6waiYLseTi5mcUEXC8dkwsKOsSHMVCn8zW27JoY2/Dwiggt2jGJ6buP1AI7rF/y26zV5tNEoBIV/tTUNAHDpni1QBe47Vb/TdPvReTz5gknUuUpKRERdcM0tR3H9A2c3fmAfaaQKewaA3wHwXBG5M/jzEgDvA/ACEXkQwAuC9wHg6wAOAzgE4GMA3hr9YdNmlHNcDKfqF93t2jqSAuB3/CstZgrYOpJuqQDbOZbG3Or6oncxa8NTIBdj4X9yIQvX0+LVjG6ZmhjCzEq1qE+ueH8jLtg+hmPzDRT+nl942253Ov6m8K/1YvXSPVsBAHfXifvMrOQxPZfB/n2T0R8gERFRkxqZ6nMzquf2AeB5VR6vAN7W5nFRH+pIx3+sdsd/Yc3G5Giqpc+7fWwIyzkHBccr6wDPr/kvBjK2C1WNpas7HcRiur0w1C/8q3f8LQF2jDVY+O8YxTfvfRiO69Wd7GMiPk63Ov5u/Y7/rolh7JoYwr11Cv/bg3z/ky/o78JfRM4H8EkAjwDgAbhKVT8kItsBfBrAPgDTAF6lqgvBleQPAXgJgAyA16vqj7tx7EREg4Q791LHdKLjbwr7xWyVjn+2UIwCNWvHuP9xCxUvKGZX/fddT2PrTJvu+AVd7vjvmhiqmvGfWc1jx/gQElZjL3ou2DEKx9OqC4XDTMa/2x3/dJ0XJ4/fs7Vux//2o/NIJ63i1YE+xv1eiIg2ARb+1DGd6PiPpBJIJ6x1BToALGZsbG2x478juJIwt1r+ecMLfmvtD9Cuo3MZjKQSDUdp4jI1MYTZ1Ty8isW2Z5fzmBpv/Ngu2OFfuTi6Qc7fZPxN5Ccs77j45xsOFYvzOOQd/9+zVscf8OM+D82sIhNMlqh04OgCnrBn67rRsv1GVU+bjr2qrgAI7/dyTfCwawC8Ini7uN+Lqt4KYFswJIKIiGLEwp86phMdfxHBttEUFtfWd/wXMoWWoz47gsJ2bq284x3O/WdjyvkfnVvDBTtGu744dNfEEGxXsVRxNWVmNY9dW5op/P0rF9MbTPZxixn/9cX97UcX8P5v/AQHjs43/LzNKmX8a//MXrpnKzytPkouZ7u45+QSnjxg+f56+70A2Gi/FyIiihELf+qYTnT8AX+yz2K2esd/W4tRn+1Bx79ypOdchzr+3V7YC5QW71bGfZrt+J8zMYx00tpwga9TJ+pjivJ8rB3/+hl/wI/6AMDdJ9bHfQ6eWILtKvZfMDjz+yv3e6n30Cq3rfuHFpErROSAiByYmZmJ6jCJiAYWC3/qCFVF3nE7EnnYOppaN9UnW3CRdzxsa7HjvzPI+M9WRH3C0Z9acY92eJ7i6Hymq6M8DVPchxf4ep5idjXfVAzJsgQXbB/F9OxGHX+zuHd94W9eDMQZ9dloqg/g70uwczyNe6qM9DRXI/p9Ya9Rb7+X4P6m93tR1atUdb+q7p+amorv4ImIBgQLf+oI21V4CgynOtHxT2GpovA3VwBaXdy7ZTiFhCXFKT5G+ApALoaO/5mVHAqOV8zFd5PZEO1saKTnYtaG42lDM/zDLtgx2njHv0rG30z6iTfjv3HHX0Rw6Z6tuKfKAt/bpxdw4dRY8WpRP+N+L0REmwMLf+oIs1CyEx3/ydH0usW9C0Hmf9tIax1/yxJsH0uvW9w7u5pHMphmky1EX4SaBbA90fGfWN/xP1uc4d/cjsJ7t4/h6FwG/vTf6opTfaoU94UOFP6NTPUBgEt3b8WDZ1fLXvh5nuL2YwvYPyDdfnC/FyKiTWHDOf5EUcjZfhHViY7/1tEUFjN22Vx9M9e/1Yw/4E/2qdy9d36tgHO3DeP4fDaWqM9RM8O/Bzr+40NJjKYTZRn/4q69TSzuBYB9O0eRtV3MrOSLVxIqFef4e+tfHJj4TyHGGf/mcw9t8DN76Z6tcD3F/aeX8bN7/UL/8OwqFjP2wOT7ud8LEdHmwI4/dUSnO/4F1ytbbGvm+k+OtdbxB/wFvpWLe+fXCtizbQRAPIt7j85lkLQE525trqMel8pNvM4u+283s7gXQHGx8tE6cZ96U33sXur479kCAGVxnwPTwcZdAzbRh4ioG7qz28vmxMKfOsLkpTfqnkbBxHnCC3xN9GfbSBsd//GhsvGdrqeYzxRw3qRfxMYxzvPoXAbnTY7U3eG2k6bGywv/meD70eweA+YKRr0FvmYBb9XFvV5nFvdagg2/93u2jWByNIV7TvoLfHO2i6/dfRrbx9K4sMu7LRMR9bPuDrnenHqjmqC+Z/LPnej4mzjPQqg7vxi8CGh1qg+wPuqzmClAFThvMsaO//xaTyzsNXZtGSpb3DuzksdYOoGxoeZSg3smR5CwpO4C39LOvVU6/kHBH2fUJ++4dRf2GmaB790nl/DgmRW84p++j5senMXvPvPCru+9QEREFMbCnzqikx1/s0lXeKOpxUwBwymrrQ3EdoylsZJzirEl8yKg2PGPuPBXVRyd641RnkZlx//sSnOjPI1UwsLubcOYrrN7b2mqT5WMvxf/HP+C420Y8zEu3bMVPzmzgpf9w82YXc3j6jc8Bf/98otiOzYiIqJWcHEvdYTp+A93suMfmuyzkLFbHuVpbB83VxJsPGJrojjhZ/fWYYhEH/VZyNhYyTk9sXmXMTUxhOWcg5zt78I8s5JrqfAH/LjPsTq795qMv1M149+BqI/rYajBF4pP3jsJ11P8/EU78IFXPRG7mpxyRERE1Ans+FNHdKPjv5gJd/xb37XX2DHmF7izQa59Lpjpv2N8CKOpROSFfy9N9DFMQWu6/mdX8i0XuXu3j9Zd3OvUy/h3aI5/ox3/5z12F776P34B17zhqSz6iYioZ7Hwp47Id7Djv7VY+Jfn8Vud4W/sCDr+ZrKP+Xv7WBoj6QQyEUd9emmGv2G6+2ak50yLUR/Af0GzmLHXbbZmmIx/tRx/sfB3o19XYeQdr+6uvWEm529ZzPQTEVHvYuFPHdHJjv9QMoHRdGLdVJ92RnkCfsYfKHX6Z1fNbsApjKQTyEXe8c9ABDi/x6I+gF/w52wXKzmn5cJ/7w4z0rN63Kc4x79Kx9/pRNTH8Rpa3EtERLRZ8KxGHZEPNvBqtIParsnRdFnUZylrY2sbozyBUtTHZPvn1/KYHE0hmbAwkkogE3XhP7+GR2wZbmtBctR2mcJ/NV+M+7Ra+JsrGbUW+LrFDby6t3Nvp35eiYiIOoFnNeqIXDAJp1NF7LbRVDHqo6pYzNjF7H+rtowkkbSkLOqzPbgKMJJKRD7V5+hcpqcW9gJ+rEkEmFnOFeM+LXf8g6+t1gLf4lSfeh3/OHfuZcefiIj6DM9q1BGd7vhvG00Vp/qs5B04nrY91UdEsH0sXez4z64WsCPYsXYkHU/h30sLewF/M6sdY0NBx9+f57+rxcJ/NJ3Eromh4lqGSo3t3Bvffo0Fl4U/EVG/yDsubj863+3D6Dqe1agjiuM8O9bxT2MxmONvFo9ubbPjDwS794Y6/jvCHf8Ioz6reQezq/liDr6XTE0M4exy+1EfwF/gW6vwL2X864zzjHsDrx7ZMZmIiNrznq/ch1/9yC14aGa124fSVTyrUUfkHQ+WAMkOTT3ZNpIqZvxN57/djj9gdu8Nxnmu5ouTfkbTyUg7/sfne2+ijzE1MVTM+FtSWvvQit3bhnF6OVv1PhPnqbeBV8GJb6oPoz5ERP3jvtPLAMpHfQ8intWoI8yGTyKdKfz9xb0FeJ4W/5O3m/EH/Iz7/FoBjuthMWtje1D0Dkfc8T8WFP69lvEH/GjP2eU8zq7ksWN8CIk2Xsylk1bVqT1AaHFv3ahP3It7e2dhNRERUbtY+FNHNDMTPQrbRlPw1M/3m47/tkiiPn7GfyFjQ7U04nMkbTXd8T8+n8H/uO6Oqi8Yjvdw4T81MYTZ1TzOLOcwNd56tx/w1wxUW7wLlLr61e7vRNSHHX8ios1BNb71Xv1mw7OaiHxCRM6KyD2h2z4tIncGf6ZF5M7g9n0ikg3d99E4D542j7zjdrR7anbpXcwUih3/dnfuBfxCfzXv4PSSH08pi/o02fG/5fAcvnLXKRw8sbjuvmPzGUwMJ7G1zU3H4rBrYgiOp3jw7Cp2bWmz8LekuIi3kluc6tOdjn+ehT8RUU9rNUXwW/9yK/7yq/dFfDSbQ7KBx1wN4B8BfNLcoKq/Yd4WkQ8AWAo9/iFVvSyqA6T+kLM9DHdg8y5jsrh7r10s/KMoos0UnwfP+IuDzDjP4WCcp+dpw7u3ruYcAMD03BqeduGOsvuOz2dw/uRox6JRzTCLeU8sZPFzFcfdrKRVO+rTExt4cXEvEVHf+f6hOXz/0Fy3D6MrNjyrqer3AFSdfyR+VfIqANdFfFzUZ7rV8V/IFLCQKWBiKIlUBEWcKfQfPOsX/juDFwKjaf9ryzdRiK4Ehf+R2fVTbY7N994MfyMc72lnog8ApBICu1bH3+3uBl551+vITtNERNQaRnya1+5Z7ZkAzqjqg6HbHikid4jIjSLyzDY/P/WJTnf8t5V1/AvYNhZNZGZnEO158MwKAJRt4AUAmYLT8OdazftXIo7Mlo8W8zzF8YVsT47yBIBdW4ZLb7dZ+CcsKUZ6KpmOf6Fqxz8o/GPK+Kuqv7iXHX8iop7Xi1fHe1UjUZ96XoPybv9pAHtVdU5EngzgiyJyiaouV36giFwB4AoA2Lt3b5uHQb2u0x3/yXDGP2tj20j7+X4AxSk+Pz27ApHS84wEHf9mFviajv90Rcd/ZjWPguPh/MmRKA45cuEu/9TEcJ1Hbsws7lXVdb+460/18e9r5gpLM8znZ8afiIj6SctnNRFJAvgVAJ82t6lqXlXngrdvB/AQgEdX+3hVvUpV96vq/qmpqVYPgzaJvNPZ2ITJ8y9kbCxk7Egm+gClxbzH57OYHE0XR1majn+umcI/X8r4e6GutxnleX6PRn3Gh5LFaFO7i3tTwfevWtffRHyqZfzjXtybD/YHYOFPRNQfmArytXNWez6AB1T1hLlBRKZEJBG8fSGAiwEcbu8QqR/k7M7ORE9Ygi3DSSxlbSxlCpFM9AEQrBXwi1UT8wHCUZ/mO/55x8PDy7ni7cfmeneUp2G6/u2O80wE30unovD3PIW5qdoaADsU9Ykj42leUHBxLxFRfxn0VFAj4zyvA3ALgMeIyAkReVNw16uxflHvswAcFJG7AHwWwFtUterCYBosecft+ELJybF0sLjXjmTzLsDPEZqdaneECn/TAW9mpOdqzi52lI/MrhVvP76QgQiwp0ejPkCp4G97ca/lf/2Vhb8bKuarjfM0j1dd/7FRMGsHhlLcwIuIiPrHhhl/VX1NjdtfX+W2zwH4XPuHRf0mb3sY7vAuqNtGUphbLWA5Z0fW8Qf8Tv/Dy7li7AcAhoPCP9Nkxv+x527BXccXcWR2Dc941E4AftTnEVuGe3rX2F1bhjCWTmBsqL1lQiYq5VbEecLxnqpRn1DEp+B4kUxsCmPHn4iI+lG7i3uJGtKNjv+20TSOzK5B1X8REBVT8JvOP1Dq+Oea6fjnHTzx/G144PQypsMd//lMz+b7jZc9YXckx2hiU5VxnvAIz6obeIW6/AXHw1h7Fx7WMYuGmfEnIqJ+wrMadUSuCx3/ydEUTiz4efnJiMZ5AqWIT7WMfzNTfVZzDrYMp7Bvxxim58KFfxbnT/Z24f+Sx5+LP3nxY9v+PMmgo17Z1Q8v9q0W5bFdD8ngakEcIz0LLPyJiPrSoC/y5VmNOqJbHX9TM0Yb9fHbyzvHW1/c63mK1YKD8eEk9u0cxeGg45+zXTy8nOvphb1RMlGfyk26wsV+rZ17Tcwojsk+puM/xMKfiKgvDPqiXoNnNYqd6ylsVzuf8Q8t6I0j6rM9lC8xc/wbHee5VnCg6k8JeuTOcRyfz8BxPZxYyAIA9u7o3YW9UTJRn3od/2od/YLrYSz4nrPjT0RE1Bhm/Cl2ZiZ6x6f6hLr8kxF2/OtFfRrt+JtRnhPDSWwZScJ2FacWczi+0PujPKOULE71qd7xH0paVTfwclwv1o5/caoPC38iop4Xx1jnfsXCn2KXt7tTRJV1/CMa5wkAjz9vK87ZMoRH7Rov3pZMWEgnrIYz/qvB5l3jw8niaMwjc2s4bjbv6vGMf1SSVvU5/mbKz0g6UfVqgKfAaJxRn+DfMZ3o3clKRESDrnLH93r42sDHdhbFLhd0/Ic7PBPd5PotAbZqcnqsAAAgAElEQVQMR1f4X7J7K2770+evm2E/nLIanuO/krMBABPDKTxy5xgA4MjMKo7NZTCUtNqej79Z1Frca64ADCcT6yb+mCk/40MxRn1cRn2IiDazHxyaxQMPL6+7fdCz/uz4U+y61fE3m3ZtHUnBsuL/nz6STjRR+Acd/6Ekpib8mfjTcxmcXspi7/bRproYm1nNjn/w/nDKgr1afp8p/EfTMUZ9uLiXiGhT+81/uQ0AMP2+l3b5SHoLC3+KXdc6/iN+xz/KiT71jKaTTUd9tgwnISLYt3MMR2bXcHYl3/Mz/KOULC7urZ7xH04l4HoKVS2+GDJXB4qLe2Ms/NnxJyKifsKzGsWuaxn/YHZ/lPn+eoZTiaYX944P+6+9TeF/fD4zMAt7gdLiXrvGVB/zYjF8f7HjH2T88zEu7mXhT0RE/YRnNYpdaSZ6Zzv+E0NJJCyJdJRnPaPpRMPjPFdDUR8AeOSOMRybz2A17wxkx9/1KjP+paiP/35oJ1+vouMfQ8bfvFhl4U9E1P8+cfMRvPTDNwEAbnloDmeXc10+ovgw6kOxM8XwcIfHeYoIJkfTZfP24zSSSiBTcBp67ErOhggwFuTUzQJfADh/cjBm+AOljH/lAl4T/Sl2/B0FgsSWHbyQ5DhPIiKKwnu+el/x7dd87FbsHB/CgXc+v4tHFB8W/hS7bnX8AeADr3oi9mzrTCE9nEpgbq3Q0GNX8g7G08niouN9ocJ/747B6finak71CTr+wc9M+IWB6f6Pxbi41/zMphMs/ImIBs3sar7bhxAbFv4Uu251/AHg2Y+e6thzNRP1Wck5mBgu/fcr7/gPTuGfsEzUp7x4dyujPqEXBgUniPoUO/6Nfc+bUXA8pBPWwExXIiKiwcDCn2LXzY5/JzUT9VnNOcWFvYA/enTLcBKphFUsaAdBKsj4Vy7uNR3/kbRZ3Ful4x/jHP+84zLfT0REfWdwKgzqmm52/DupmTn+q3kHE6FNxUQEj9o1PnAd5kQw1adyca+5AlCa6hNa3OvGH/UpOB4LfyIi6jss/Cl2A9PxTycanuO/krPX7S/w/l97QhyH1dOKi3sr5/i75eM8wxt8masDozHP8efCXiIi6jcs/Cl2+SCDPdTnHf/RVAK2q7Bdr7hotZaVKmM7H7VrIs7D60nFxb21du5N1u74p5IW0gkL+RiiPgWXHX8iIuo/PLNR7HJd2sCr00wevZEFvpWLeweVWdxbWfiXMv7rF/eat1MJC+mk5Y/6jJhZ3EtERL0v+rNA/+KZjWJnFkr2e37dxFIayfmv5pzi5l2DzCzuddxaU33Wd/zNYt6kJUgnLRTc6Kf65JnxJyLqef1dVcSDZzaKXd72MDwARZTJnG+U87ddD1nbLVvcO6iSjc7xr9LxTwdRH2b8iYiIGsMzG8Uu77gYSvX3wl7AH+cJAJkNOv5reX/kJzv+pcW96zP+QTzMzPH31mf8ix1/TvUhIiJqCM9sFLu87fX9KE+glPHfqOO/kvMLf2b8Q4V/RdTHdPjNi6nwFYHi4t4g4x/LHH/XQ7rPp1AREdHg6f9qjLou57h9P8oTKBWpuQ06/iz8S2ot7q3M+BfKNvAKLe6NKeqTt10u7iUi6nHNLOrlAmDfhmc2EfmEiJwVkXtCt71bRE6KyJ3Bn5eE7vsTETkkIj8RkRfGdeC0eeTtwchLm47/RlGf1bwp/JnxFxEkLSmL8gChjH/djr8f9cnHEfVxB+NnloioHzSzyHfQFwQ3cma7GsCLqtz+96p6WfDn6wAgIo8D8GoAlwQf888i0v+tXqor57jFAq6fNbq4dyVnA2DG30gmZN3iXpPxL0Z9yjL+GnwcF/cSERE1Y8Mzm6p+D8B8g5/v5QA+pap5VT0C4BCAp7ZxfNQHBqXj3+g4z1LHn4U/ACQta8M5/naVjn86xow/F/cSEVE/aufM9nsicjCIAk0Gt+0BcDz0mBPBbeuIyBUickBEDszMzLRxGNTrBqfj7xfyG3X8l4OM/zgLfwCm418xxz8o9Ieq7NxrHptMxDjVhzv3NoWRUCLqB/uu/Fq3DyF2rZ7ZPgLgIgCXATgN4APB7dWiU1XXU6jqVaq6X1X3T01NtXgYtBkMSse/0XGeq2Zx7xAz/oDf8bdrdPxLGf/wBl5B1MeSGBf3cufeJl0NRkKJaBMY9EW+LZ3ZVPWMqrqq6gH4GEpxnhMAzg899DwAp9o7RNrs8o43EB1/8+KmkYx/0pKBGHHaiKQlxQ6/4XqKhCXFnX3LN/DykEoIRCS+qI/rFfcQoI0xEkpEvW7QF/UaLZ3ZROTc0LuvBGAu734ZwKtFZEhEHgngYgA/bO8QabPL2e5AdPwtSzCSSiC3QeG/mncwPpyECH8NAX5kx64y1SdhSWln34oNvFLB7XFEfVxP4XqKdKL/X6x2QFuRUCIiilYj4zyvA3ALgMeIyAkReROA94vI3SJyEMBzAPwRAKjqvQA+A+A+AN8A8DZVrV8FUd/LD9CElJF0ApmCU/cxqzmHC3tDklb1qT7JGh1/29Xixl9xFP7m8zHj37a2I6FcC0ZEFK0Nqw9VfU2Vmz9e5/HvBfDedg6K+kvOHozFvYCf888W6heiyzkH48z3FyUTVnHDLsN0/FOWmepT3vE3RXkcGf+84/cqWPi3R1XPmLdF5GMAvhq823AkVFWvAnAVAOzfv3/Qo7lEVEMjvxz4C8THMxvFSlUHruOftTfo+OdtdvxDkpaUFfaAv2FX0hJYlsCS8g28/Pv8n6ehpIV8xBl/dvyjwUgoEcWtlcDsoIdsWX1QrMyuqkMD1fHfaHGvg0dsGe7QEfW+ZEJqdPyt4H6rbA2A7XpIJUtRH9v1oKqRrZko/syy8G9YEAm9HMBOETkB4F0ALheRy+A32qYBvBnwI6EiYiKhDhgJJSLqGBb+FKtBK6L8jv/Gi3vZ8S+pNs7TZPwBP85jO6GMv6fFCFA6YUHVf6Fg1gO0y0wJGpSf2SgwEkpEtDnwzEaxMnlpdvxLVnION+8KSVXZwMtk/IFgg69wx98pn+oDINKcfzHqwzn+RETUZ3hmo1jlbb+IGh6Q7uloIx1/Lu4tk7CkuGGX4YY6+EnLKp/j73lIJkpRHyDawj/PjD8RUU/55C3T2Hfl14rNRGodz2wUq0Hs+NfbuTfvuCi4HqM+IamEVbfjX3lFoODq+o5/hAt8C8V42mD8zBIR9boPX/8gAGA5W394Bm2MhT/FKjdgHf+RdP0NvFZy/i8tFv4lVTv+ock9qYRVNvXH7NwLlOI4sUR9BuRnloiIBgfPbBSrQez418v4r7LwXydpWes28KrM+IcX/1bu3AuU4jlRKLic409ERP2JZzaK1aBl/EfSCWRsF6rVtwoxHX9m/EtSFYt3gWCqT9DVT1nlUSDbVSQTpak+QMQZf5uLe4mI+k6N8/Kg4ZmNYpUbtI5/OgHV2h3olbwNgB3/sGpRn3VTfdzyjn+6cnFvlBl/l1EfIiLqTzyzUaxM93RQZqKPBC9wasV9Sh1/Fv6Gv7h3/VSfZHFxb/mc//DOvXFO9RmUn1kios1kenat5lX1uiLa5HGz45mNYmU6/sMD0vEfTQeFf40Fvibjv2WYUR8jaVWZ4++WT/Wxncqde+OL+hRY+BMR9aS7ji/i8r+9Adf8YLrbh7Jp8cxGsRq0jr95gVOz8M8HHX9GfYr8Dboqoz5esauftKzyDbw8DymrMuoT3WxnTvUhIupN03NrAIA7ji92+Ug2L57ZKFZmtOWgdPw3jvr4GX9GfUr8wn591Kdsqk844+9UmePPDbyIiAYW1+02jmc2itWg5aVH035BX6vjv5J3MJS0WFSGJCwpm9MP+It7TcY/nSjv+Id37h2KY5ynw6k+RES9qLLAZ2y/eTyzUawGrfAfSftfZ63de1dyDif6VEglBG6Vjr8p7pMJge2U7i84oTn+iUTxtqgUXBcJS4ojQ4mIqNtY4UeFZzaKVc52kRygImokFXT8axT+qzkHE1zYWyZZZaqP3/G3ivfbZR1/Le3cG7ygtN3orvMWHI/dfiIi6ks8u1Gs8o43MN1+wJ/jDwCZglP1/pWczXx/haRVbQOv0FQfa/0c/+S6jH90i3vzjscoFhFRv+FCAAAs/ClmOdsdmIW9APCILcMQAY7NZ6rev5pn1KdS0rLgKeCFZ/V7XjHj718R8F8YqCpst8ri3ig38GLhT0TUt2TAFwbw7EaxGsSO//mTozh0drXq/Ss5hx3/CibLH47zuGVz/C0Ugo6/mf6TCi38BaKf4z9IP7NERJsNm/et49mNYjVoHX8AeNSu8bqFPzP+5Uxn3y3r+JcW96YSpSiQifyYDbxM1j/ScZ4uO/5ERL1owJv1keDZjWI1iHnpi3eN4/DM2rrdaFUVy1mbUZ8KJq8fXqBbNsffKi3+NZEe82JBRJBOWshHHfXh4l4iop7Hzn/zeHajWOUdbyA7/gXXw/GFbNntp5ZyWMk7uGhqrEtH1ptMER9+oWS7pZ17U4nSnH/zmPCLyaGEFfkGXoz6EBFtHs1cCdABf7Ww4dlNRD4hImdF5J7QbX8jIg+IyEER+YKIbAtu3yciWRG5M/jz0TgPnnpfznYHroh61K5xAMCDZ1bKbr/n5BIA4NI9Wzt+TL3MRHrCUZ+yqT4Jq1j4m6sC5kUB4L8IiDbj7w7cVSoior7HnBCAxjr+VwN4UcVt3wZwqao+AcBPAfxJ6L6HVPWy4M9bojlM2qzyjoehAez4A8CDFTn/e04uIWEJHnvulm4cVs9KBUW8XZnxt0obeJmpP+YFgMn2+29HXfh7GEoO1s8sEdFmMODN+khsWPir6vcAzFfc9i1VNYPKbwVwXgzHRn0gb7sYHrDu6cRwCuduHcZDVQr/i3eND1z0aSOms+/WyPib0Z2255Wm+iQqOv5RZvy5uJeIqKewWR+dKM5ubwTwn6H3Hykid4jIjSLyzAg+P21ig9jxB/yuf7jjr6q4++QyLtnNmE+lynGeqhpM9Ql27i2uAQh3/OOL+uRtLu4lItpMeCWgcW2d3UTkHQAcAP8R3HQawF5V/VkA/xPAtSJSNdcgIleIyAEROTAzM9POYVAPG8SOP1Aa6Wk2pTq7ksfsah6P38OYTyVTxJvJPSbxk6zs+LtesfBPhqI+6aijPuz4ExH1NFPnN3UlgK8OALRR+IvI6wC8DMBvabBEWlXzqjoXvH07gIcAPLrax6vqVaq6X1X3T01NtXoY1ONyjoeh1OAVURfvmkDWdnFqyZ/sc/cJLuytxUR6irP6g79LUZ/gioCrxcW96TijPpzqQ0TUk+oV+meXc9h35ddw+9H52g8Cd+5t6ewmIi8C8HYAv6yqmdDtUyKSCN6+EMDFAA5HcaC0OeVtdyAXSlYu8L3n1BJEgMftZse/kinsTcffTPcpLe4Nrgh4XnGcZ1nHP2khH/HiXnb8iYh6RyPN+lsOzwEAPnT9oZiPZnNrZJzndQBuAfAYETkhIm8C8I8AJgB8u2Js57MAHBSRuwB8FsBbVLX+Sy/qaznHw/BAdvz9wv/QmaDwP7mEi6bGMZrm5l2VElapsPf/1uB2s4FX6YVBoUrGfyjycZ4s/ImINqvv/XQGmYKz8QMH1IZViKq+psrNH6/x2M8B+Fy7B0X9wXE9uJ4OZMd/ciyNneNpHDId/5PL+LmLdnT5qHpTyqro+LvlHX9ThBdcr/iYVIwZ/0HcbZqIaLMLR3jW8m4Xj6S38exGsckFxdggdvwB4KKpcTx4dgUzK3k8vJzDJYz5VFWK8mjZ34niVJ/S4t9aU33siDL+qv5VhSFO9SEioj7EsxvFJm/7r7gHseMPABef44/0NDv2Pp4Le6sqLe41hX+Q4w9t4AWYqT41du6NqPA3n2cQR9ASEfU6k/Wvtjx3sJfsNo6FP8UmP+Ad/4t3TWAl5+A7D5wFwIW9tZQW9wYZf7c841+8P7RzbzoZT9THfB7O8Sci6h2NDOIJP2YpW4jvYDY5nt0oNrkB7/ibyT5fPXgKF+4cw8RwqstH1JtM996uNdXHKs3xL10NiGcDL/NilRl/IqLetdGQn7/6+gMdOY7NiGc3io0pogZ1JrqZ7LOQsXEJYz41mSiPW5nxr7aBlxMs7k3GU/gXWPgTEfWsep1/CYV9zHkEAO46vtjWc95+dB6nFrNtfY5ewrMbxcZ0/IcHNC89NTGELcP+4Czu2FtbsmIDr1LH3//1FJ7zbwePMZOAgGCOf1QZ/wF/sUpE1G/e/ZV72/r4X/3ILbj8b26I5mB6AM9uFJtB7/iLSDHuc+ludvxrqYz6FOM8ifUbeNlOlTn+QcZfI9iOveCy409EtBmFrwacmC/uLYuzy/m2P3fB9XBrsEHYZsezG8WmmPEf0I4/4C/wBcCoTx2lqE9lx798A6+Co8XLt+Gde0tRoAgKfy7uJSLalMIpoMOza5F//ldfdWvkn7MbuI0oxWbQO/4A8NqfvwCPecQEto5wYW8tpXGd1TP+pvvueF7VnXvDG3y126nPO27Z5yQiou768l2nMLNSv2uvGy73JYOFP8Vm0DP+AHDJ7q24hDGfukobdFXP+CdDO/uWdu6tUvg7HjDU3rFwqg8RUW/5/evuqHlfeEFvIyM/iVEfihE7/tSIZKJiA691c/xDU31cD5aU7gMqCv82lRb3Du6LVSIi6l+syCg2eXb8qQEp0/H3Kub4Fxf3lqJAtqvFxb6GyeNHW/jzVyMRUa+qPsyhsZb/YmawN/fi2Y1iU+z4D+jOvdQY0703Bb8Z2VnZ8Xc8v+NfufC2lPF32z4WRn2IiPrLycUsvn3fmeL7r//XH3XxaLqPZzeKTTHjz9gE1WEy/LbJ+LvlU31SoXGfjuuVTfQBSt35fIQdf071ISLaXOpl/H/3kwciW/57cpNv5sWzG8Um73gQKW3ARFSNZQksKWX7K6f6FNcAuB4KrpYt7AUizvi7vEpFRLQZdaLS+PyPT+AZ7/sOfnhkvgPPFg+e3Sg2OdvFcDIB4VJ72kAyYa3P+JupPonSFQHH9cp27QWAdMK/ohRlxp8dfyKiwXZgeh5LGbvsttuPLgAAfnJmpRuHFAmO86TY5B0Pw+ycUgNSlhTHeTqVGf9Q1Md2PaSS1Tv+kW7gxYw/EdGmslGTsdnN3X/to7eUTZDrFzy7UWxytsuxiNSQhCXrOv4mImZZEtzvwfa0mP03ol3cyw28iIj60d0nl5r+GHM+6ic8u1Fs2PGnRqUSVrHTX5nxB/yFvo6rsB1vfcY/hnGejPoQEVE/4tmNYsOOPzUqmZDi4t7KjD/gvzCwXYXj1V7cG8VUn7zrIZ20uC6FiKiHtT7Fn1j4U2xyNjv+1JikVVrcW63jn0pIcefeyilRQxHv3DvEbj8R0aagkQ3pbM3hmVUcn8909RiaxTMcxSbvsONPjfE7/maOv/93OMufDKJAtuut37m3mPGPoOPveMz3t0BEPiEiZ0XkntBt20Xk2yLyYPD3ZHC7iMiHReSQiBwUkSd178iJaDMxC3Qru/sCqTvHPy7P/cCNeOb7v9v5J24Dz3AUm5ztcR46NSRhCezKjn+os5+yJJjqo+s6/lFn/Fn4t+RqAC+quO1KANer6sUArg/eB4AXA7g4+HMFgI906BiJaJOrVdx3vPOviud+4IbOPmdEeIaj2OQdjx1/akjKsoo79jpe+c69QNDxN3P8Kzr+qaijPiz8m6aq3wNQuaPNywFcE7x9DYBXhG7/pPpuBbBNRM7tzJESUT9YyBQAAHceXyze1omOf/g5Ds+sxf+EMWjoDMfLuNSKvO0y408NSSakONXHrTbVJ+F3/Auuli36BaLt+Ocdlx3/6JyjqqcBIPh7V3D7HgDHQ487EdxGRH3unV+8G+/+8r11H+N5irW8U/cx3z80BwD4g0/dWbxNuLy3IY2e4a4GL+NSk9jxp0YlgygPgOJ0n3CBn05YxZ1708nyX+4m+hNFxv/0Ug7nbBlu+/NQXdXOzlWv04vIFSJyQEQOzMzMxHxYRBS3f7/1GK7+wXTdx/z1Nx7AJe/65obFf6XZ1XwbRzY4Gir8eRmXWpFjx58alExYxU6/G3T+w/t0+VcE/J17Kzv+IoJ00mq746+qODKzhn07xtr6PFR0xvzuD/4+G9x+AsD5ocedB+BUtU+gqlep6n5V3T81NRXrwRJRb/j8HScBAJe865v44ZHK0rO2//XZg3EdUl9ppypr6zIuOzn9jx1/apTf8S9t4JW0pGyWftKygnGe6+f4A8BQwmp7jv/cWgEreQeP3MnCPyJfBvC64O3XAfhS6PbXBrHQpwNYMucSIqKwq39wpOZ907ObM2PfbXG0Yxu6jMtOTv9jx58alUxIqOOvSCbWx3ls14PjrZ/jD/gjPduN+piTCAv/5onIdQBuAfAYETkhIm8C8D4ALxCRBwG8IHgfAL4O4DCAQwA+BuCtXThkItrkLv/bG7p9CJtSso2PPSMi56rq6VYv41L/clwPjqfs+FNDkpaFVc8FYDr+FZN7En6Up1bHP4qoz5Gg8N/Hwr9pqvqaGnc9r8pjFcDb4j0iIqL4dHfbsPa0047lZVyqycQu2PGnRiSt0AZenpZN9AH8NQC2yfjX6vhHUPgnLMF5kyNtfR4iIuqs4/PZjjxPP0wOaqjjH1zGvRzAThE5AeBd8C/bfia4pHsMwK8HD/86gJfAv4ybAfCGiI+ZNoFS4c+OP20sHPVxPK9shj/gb+DluP7OvelqHf9E+4X/9Nwa9m4frXpFgYiIqB80VPjzMi41K2f7sQ1uhkSNSAbjOoFaHf8g4++uz/8DfsffbjPjf2Q2g307Rtv6HERE1L86vkNwDFiVUSzY8admJC0p7tjruLq+45+wYLsKx6uT8W+j8FdVTM+uMd9PRNRluglq680c+GHhT7Fgx5+akbSs4sZdjqdIrJvqYyFbcItvV0q3Oc7zzHIeWdvlRB8iIuprrMooFuz4UzNSCYHjhef4l/9qSlqCTMEpPrZSu4t7j3CUJxFRz+mHxbS9hoU/xYIdf2pGwpKynXurTfXJBj9TlS8KAP/nrJ3Cf3ouGOXJXXuJiGgDf/ale7t9CC1jVUaxMB3/IXb8qQEmww9Uz/inE1K8P1XlxWQ6aSHvuC0///TsGtIJC7u3cZQnERH1Lxb+FAt2/KkZiQbm+Bspa/2l360jKSxl7Zaf//DsGvbuGF33vEREREY/RI9YlVVx6OwqljKtFxFUKvyZ8adGJBOhqT6elhX65n6j2uLeneNDmF8rFONCzZqeXWO+n4gIwLG5DP7wU3e0vTcK9SYW/lX85sduxUdufKjbh7GpFaM+7PhTA1KWVSz8Xa/KOM9Qrr/aHP+d40PwFFjIFJp+bs9THJ3PsPAnIgJw5ecP4ot3nsKPpue7fSibylrewZnlXLcPY0OsyqpYyBSw2EIBQSV5dvypCWZxr6rCqbK4N9zlr7Zz747xNABgdjXf9HOfWsqi4Hhc2EtEA+Ob9z6My97zreLV+V5VuWFWL8/4f+U/fx9P+6vru30YG2LhX8FxPdiu9vx/hl5XWtzLHzHamBnR6XhateMf7vJXxoAAv+MPALMrzb9gN6M89+3krr1ENBje+7X7sZixN0WHerP46ZnVbh9CQ1iVVcgFBWuWhX9bihn/JDv+tDFTzDvB7rzrO/5S9W2jWPi30PGfDgr/C3eON/2xREQUn35YTNtrWPhXMAVrzuailnbkHQ8i1Ys0okqmw+94XvWOfyjjX21x71Qbhf+R2QxGUgmcs2Wo6Y8lIqLOEZYUbWPhXyFbMIU/O/7tyNkuhpMJCP+XUgOKhb+rcFxFomKTrtQGU322jCSRTliYaaXjP7eGC3aM8meViKjDTi5msZzjFMVOYuFfwWwClOMYq7bkHQ/DzPdTgxJBMW97HhzPWz/VJ1F/qo+IYMd4GnOrrWX8OdGHiPrBn37hbuy78mvdPoyGPeN938GLP3hTtw+jYf3QH2JlVsFEfPLs+LclZ7sYYr6fGmQ25XK9IOOfqL2BV7WpPoA/2afZqI/jejjOUZ5E1Ceuve1Ytw+haScXsw0/9ot3nozxSAYDC/8KJuLDxb3tYcefmhFe3Ft1jn/ZVJ/qLZed40NNF/4nFrJwPMU+Fv5ENEAqx2RuFvecXO7q86/lN39tyMqsQtZmxj8K7PhTM0yhb7tekPFvbnEvEBT+TY7zPL6QAQDs3c5RnkQ0eDg1pzmf+/GJbh9C21j4VzBRH071aQ87/tQM08V3gzn+qXqLe63ahf/cWh7axA4va3kHALBlONXsIRMRbSo3PziL+093t2PemM15NWKzYGVWIceOfyTY8admmI6+7VbP+Ie7/KlkrahPGrarWMo2PiHCXOEbSfNnlYj6229//Da8+EObZyEtAPCCRPRY+FcwhUDe8ZrqHFK5vONx115qWPkc//VTfcp27q3R8Z+aMLP8G4/7ZILxvaMs/ImIyrAE6k+szCqEp/nkOdKzZTnbY8efGmYKe8ervnNvuNivOdVnrPlNvMy+Hez4ExH5+mFkZTedCNaO9SoW/hXC2X5TFFDz8o7LjD81zBT2tab6pJMNTPWZSANosfBPsfAnosFRr5vPTn97fv+6OwAAn739BM6u5Lp8NOuxMqsQHuOZc1j4tyrPjj81odTx94KOf/mvpkan+gDA7EoThb/tIpWQmp+TiKif1evu90TnfxO+CHEVOLuSw//zf+7Cf7vmQLcPZ51kqx8oIo8B8OnQTRcC+HMA2wD8LoCZ4PY/VdWvt3yEHRZe1MvJPq1jx5+aYab21Or4h7v8qRod/8nRNCxpPuM/zG4/EQ0odvfj4bj+N3amiUZUp7Rc+KvqTwBcBgAikgBwEsAXALwBwN+r6t9GcoQdFi72Odmndcz4UzMSxak+HtwqGX/TkU9aAqnRhkpYgu1jzW3ilbNdLuwlooHTE9186oqoWrLPA/CQqh6N6PN1TQA2nmsAACAASURBVFnUh4V/y3I2O/7UONPhNwvq1+/cGxT+Nbr9xs7xdNMdf+b7iYh6FF+gRC6qyuzVAK4Lvf97InJQRD4hIpMRPUdHhKf6ZFn4t8Rx/Zw2O/7UKFPYmxfblXP8zQuBjbL4O8eb6/hnCi5G0i1f+CQiIirX4/mptgt/EUkD+GUA/ye46SMALoIfAzoN4AM1Pu4KETkgIgdmZmaqPaQrwgt688z4t8R0bdnxp0aZaI+J2tXq+G9c+KcZ9SEi6nGDkqjoxdcAUVRmLwbwY1U9AwCqekZVXVX1AHwMwFOrfZCqXqWq+1V1/9TUVASHEY1swS0uHhyUH8yomcJ/KMnCnxpj/s/lgxfelZt0mYhPrYW9hun4N7r5XqbgMOpDRAOn2wXpH3/mru4eQMx6eQ1FFJXZaxCK+YjIuaH7Xgngngieo2NytoetI/48cI7zbI15wcRpKdSodR3/igI/ZZnFvRt0/CeGkLM9rDW4B0fW9rh5FxENrG4VqLcdmevOE1PrU30AQERGAbwAwJtDN79fRC6DP311uuK+npdzXEyOpjC7muc4zxaVoj4sqKgxJsJjOv7rpvokG+/4A/4s//GhjX+9ZdnxJyKiAdJW4a+qGQA7Km77nbaOqMuyBRfbRlMAGPVplfm+MepDjUpukPE3nf5GMv4AMLeWx76dYxs+b5YZfyIiitBdJ5a6HqWqh5VZhbzjYduoXzxwqk9r2PGnZpnCvjjVpyLSYzr9yQam+gDAzEpjIz25gRcRUXW9XLz2usMzawAA7cGth1n4V8jZLraOmI4/oz6tYMefmpVM1J/jLyJIWIJ0o1GfBif7ZAvs+BMRhZnc///+7F342sHTsTxHoy8qeniNbF1fvyee71sUWJlVyNn+hj5DSatspj81rjjVh51UalCx8LerZ/wBv+u/Ucd/RxD1aaTwt4P9JpjxJ6JB00jhfWoph7dd+2Psu/JrePDMSqzH47ilRutmvNJgu+WN4mtvO9alI9kYC/8K2WDH2eFUghn/FrHjT80qRn2K4zyrFP6WteHi3lTCwrZgcf5GMsHkH071IaJBctODMzi5mAUAPPP938U37314w495wd9/L9ZjetGHbor188ftFf/0/aq3Sw9es2BlFqKqyNkeRlIJjKQSjPq0iBl/albCEoiU4nXVOv7JhGy4uBcIZvk3kPE3L1BHuXMvEQ2Qq753uOz9d3/53rL3O9Vxv/VwaaTnobOruPnBWXz3gbOdefKI3XtquduH0DCe8ULCEZXhlMXFvS1ix59akbSk+LNTOcffv81qsPBPY26tmY4/f06JaPPKFlys5G3smhhed1/OdvH8v7sR7/uVJ9T8+NNLueLbjuvhBw9FP2M/U3Bw3Q+PF9+fWyvg1VfdWvaY3/74bQCAHWPp4m2bMPXT83jGCwlvPMWoT+vY8adWJC2r+LNTOdUHANIJq2oEqJK/e+/GHf+sKfxT7H8QUe+bWcnj2tuO4Znv/w6Oz2cAAHOrebzq/78FT33v9VU/5l+/P40TC9liUQ0AC5n1vx/vPrEEALjuh/Fk0x/359/EX3z1vlg+dy/rxak+POOFmJjBSCqBoVQCOYdRn1aYBZpDKb6upMYlE6WOf6pW1KeBq0h+1Gfjjn/WdgAw409Em8NT3vtfxbdf8uGb8KFXX4Y3Xn2g7sf89TceWHfbPSfXx1IWs/6LgeWc0+ZRUq9j4R9S6vhbGE5a7Pi3qNjxT7KgosaFoz7VMv6//9yLce629ZeyK+0cT2Ml7yBn15/Rny34P6cc50nUG7568BSGkgm84HHndPtQet5KzsHvXXtHtw8jFt5mHOuzibDwD8lWRH0Wq1wOo43lbBci2HACC1FYMlGK+lTL+P/qk89r6POEZ/mfNzla83GZQtDxZySNqCeYQnb6fS/t8pFQNy1k7OLbrCKixyxGiOk2mqk+XNzbmrzjYTiZgAj/y1LjUpaEpvq0/qupVPjXf+Fu/n8z6kNERIOChX+IKTqGUhaGUxbHeVbheorf/NituPGnMzUfk7Nd5vupaYmEFNeHNLKIt5adE37hP7fBLH+zuJdRHyIadEzXxKMXv6+szkI41Wdj82sF/OChOdx2uPa4r5ztMt9PTUuVTfVpo/BvcPfe4jhPRn2IaBOK45q69lilOtfAhDZqDgv/kHDUh4V/dWbdQ7VxYEbe8TDMjj81KWEJCsG25+10/HeMBR3/NUZ9iIg2s1vqNBmpNazOQnJOqeM/lLI4zrMKs+hmvk5RlbNdDLHjT01KhjbnaqfjP5JOYChpYTG0QKyabMGFJf7+AETUPWeWcxs/iIgiwTNeiBnvN5yyMJJKoOB48LzeuuzVbabTv7BWu6hix59aEZ4ClWxjcS8AbBtNbTiVK2u7GE0nuQidqItuPzqPp/3V9fj8j090+1CIIteLpxeO8wypjPoA/lWA0TS/TYYppubrFFXs+FMrwl3+RJujYCdH02Uj4arJFOrP+adoiMg0gBUALgBHVfeLyHYAnwawD8A0gFep6kK3jpG654GHVwAAP5rmP3+z2LSgVrAtGxKO+gwHO4Rysk+5+bWNoz55x+NUH2paKtTlbyfjD/gd/6UNoz4OJ/p0znNU9TJV3R+8fyWA61X1YgDXB+8TEfWVHlsrDYCFf5lcMOVjKGmVOv5c4FvGdPwXMwW4NWJQOdtjx5+aliyL+rRZ+I+k6y5AB0zUhz+nXfJyANcEb18D4BVdPBaiTSmOCTy9WKhStFj4h+SCbLqIsPCvwRRTngLL2eod1bzjMuNPTQtHfdrN+E+OpRj16R0K4FsicruIXBHcdo6qngaA4O9dXTs6IqIBwvB6SM4uFQKmcGXUp1y4mJrPFDA5ll73mDw7/tSCVHiqT5sZ/60jaSxlC1DVmjnYHDv+nfIMVT0lIrsAfFtEHmj0A4MXClcAwN69e+M6PqKBx0b/4GBbNiS88ZR5AZBlx7/MYqZQXKVeK+fPjj+1orzj3+7i3hRsV7FWqP3/N1NwuXlXB6jqqeDvswC+AOCpAM6IyLkAEPx9tsbHXqWq+1V1/9TUVKcOmWhT4OJeagWrs5Cs7RU38zGFf56Ff5mFjI3zJ0cB1C78mfGnVoTHebYzxx/wp/oAwEKdRehZ2+XmXTETkTERmTBvA/hFAPcA+DKA1wUPex2AL3XnCIkojJ3/aPXi95NRnxB/DKX/Wig8zpNKFjMFXLpnK47NZ2oWVez4UyvCuf5Em52sraMpAMBS1sb5NR6TZce/E84B8IWgM5kEcK2qfkNEfgTgMyLyJgDHAPx6F4+RekIvlkhE7enFvaDaLvz7aUYzM/71qSoWMjYu3DmOG34yU3WWv+spbFfZ8aemmXiPJYAVVce/zmSfTIEZ/7ip6mEAT6xy+xyA53X+iKgTVBW/+8nb8dtP34vLH8N1251y7W3Hun0IVGGuzlXnbomqLdsXM5pzdqkDaLL+nOpTspxz4HqKc7cOYySVwPzq+h/ofHEvBHb8qTlmnGe7E30AP+MPAIt1JvtkbRfDLPyJIucp8F/3n8Ebr/5REx/FvHqzKr9j7/9mw+vmO8LzFJ/64THYLhuovSSuqM/LAVwevH0NgBsAvD2m54pMzvawc9z/lpjsLzv+JWaG/+RYGtvH0lU7/ub7ZSJTRI1KBlN92s33A6Woz2KNjr/rKQqOh9EU045Em9XppSweOL2C5/wMryoAvTODX1VxYiGLWx6aw5Wfv7snu96DLIrqrKUZzSJyhYgcEJEDMzMzERxG+8qiPklO9alkRnlOjqawfSxdNeOfD+1+TNQME/Vpd6IP4G/gBaDmLH/z/5pRH6LN65f+4ft4Q1NXFXpHwfHwnq/ct+EO4/X0SJ2/zsduOoxnvv+7uO3IPID6QxYGwcxKHicXs90+jKIo2l0tzWhW1asAXAUA+/fv74mf32yo8B8qZvxZ+BsmL71tNI3JsTTmq/zCMh1/Fv7ULBPxaXeGPwCkkxbG0omaUZ9MwQEARn2IYhT3iX12NR/zM8Tni3ecxCe+fwR5x8V7X/n4SD5nFDv5RnHV4LbDfsF/YiEDAMg7g52ceMp7/wsAMP2+l3b5SHxtd/zbmdHca3K2Vyr8kxZEOM4zrBj1GU1h+2gK82vrf+majj+jPtSsKDP+gP8CtVbUJ1fwT0SjfIFKFDmm9TfmBhW247ZeaVd+n3uig1rFv916tNuHQCFtnWH7bUZz3i6NoRQRDCUt5Ab8lWrYwpqJ+vgdf/N+GDv+1Koooz4AMDmWwmK2Rsff9jv+nONPRN1gfstpj5TrUVwtKH6uyD5Tf/m7b/8UqoqP3vgQPnnLNPZd+TUs1ThHxandqE9fzWgOR30Av3hl1KfE7Nq7ZSSF7aNprOYd5B23bHSnuULCjj81K8rFvYCf8681zjMb7OjLwp8oeqbwUwWOzWWwd4e/6eNKzkYqYdVoDA1Wudirm+5G+UKkV7/Gbvnw9Q/i1598Ht73n6VE/KGzq3jyBZMdPY62qjNVPayqTwz+XKKq7w1un1PV56nqxcHf89Ecbnwc14PjadmGPiOpRLFAIH+h5NaRFBKWYPu4v3iyMkNtrpAMseNPTSp2/CPI+APAttFUzYx/sfDnzylRrK74twPFtx//7m/hxR+6qex+GfBgUK9M4qHO8HrgH5xt2YApWMPz54dTCUZ9QuYzheLGSNuDv+crVuvn2PGnFpmCP6qO/2SdjH+mwKk+RJ1QOcP9yOxa2fu9EnXpNPOCJ9Kvvke+lVHGhvrdfaeWcM/JpY4+J6uzgOkAhi9BDiUtRn1CFjOF4sZIk2PVC/+8w4w/tSYVLOqNKuO/bTSFpaxddct0jvMk6p4//sxdVW4d7M5/SyL8lglzOV3xZ1+6Fy/7h5s7+pws/AOmwB/UjP9SxsZa3qn7mIU1u9Txr1H4s+NPrTKd/kSEU308BZZz6+M+1V7oE1H0qvV+P/fjEx0/jn4U5VSfKLv0fBFRWy9E21idBaptPDWcspAfkJ1733jNj/BnX7yn7mMWMwVsqyj8KxdPsuNPrUolop3qs23E7N5bpfAvdvy5cy9R1G49PNftQ+h9wa+5durtyg9lxGbzyjtux9aUsvAPZIO53sOhTvVIKjEQO/eqKu47tYz7Ti/XfdxC5v+2d9/hcVVn4se/Z5o06sVyt9ywjU01OMYG0yGAQyC7IQmBHyVAWNIWSJ7fxrAsZZNNSMKyIQlL6AECwZRQjQnGGDC4N9yNbdmyJNuyeps+c/aPuTOakUZl5LGm6P08jx5Jd+6Mzr1zde87577nPd5wqk8oqOqW6hPq8bfKoSXiE+rpT1iOf27wGI1V2cchg3uFOGaufWp1n+ss2nyoX691+8sbeWp5xdE2KeWkWjnPkER8dpAPIPG74KFPmH7v+4PytyQ6M7h83cv7DZVUn7o2N06vn+omZ4//sC6vH6fXH87tt5hNFNqtPef4WySgEvGxJLrH37g7FauWvzOc2ienQCGS4UcvbcAfY/xNV29tOsgvF+2IWrZuf2O/PzikqmORDuOJGEg9YcEiXllXlfC/0Zsdh1qZsGARy3bVAamR1pIuapqdg/a35D63occcf1/mB/77G4LTare7fTQ5vOE0nkihdIkio8cfguk+sXL8lepM2xCiv6wJrurTmerTvcff6fFht5olF1WIJIr+7+t/L/FVf16Z6KYkTwI7x71dZgFeuLb/gf/RNqOh3d2tVKtITdLdZQjPOGuJzvF3DYEc/8qGzvJqBxodMdcJpUuEBvdCMPCPleOfbZGASsQvlOpjNSfmtBQ6VmPNMO3w+KWijxCDoKKugzX7Yk/lo1TvvcLr9nc+r6bZyU/+trFbOdCQH7+0gWufWnV0jR1EmXaFPP2XHya7CWkhFUKjjAr8r35i5YBzAUO3/u22zl2SZRkaqT6RwX5VH4F/ZI9/cY6Nxi5Blcvrl/x+MSBWU2J7/AvsVpTqOdVHBqALMTi+/XjPPfR3v7Gl2zKtNVrrqJ79a59cxTtfHOT8hz6O+Trvbj7E53vSb1Dx0fS0t7l6r8S3vrIp7teMtz13/X0LL6yqjPlYKgS5oruMSfXpcPtYVdFIfraVW86eFPfzO8tQRqf6DIWqPvsbHJTlZ1HX5u65x98I8CPTgEpyrWypaY5az+0NSH6/GBCLObF1/M0mRUG2tYdUH+nxF2KgPL4AFpPCZFIcbnFRkmvDdpQlnCPz/Sfe9V63xxvaY0/Gl67iCYoPNjtjpuAmw+vrq/nVezt49ntf4UcvbaCqsefcdBnjm5oypms2dPuvpmlgAyTcMXL87VYzHn+gXwOQ0tmBhg6OH5nPsDxbnz3+kak+xbk2mjq8UQOCXT6/DJgUA2JJcI8/QHGOlaYeynlK4C/EwEy9ZzE/fWUTbp+fOb9eyr+9FmtCrvi8sq732v6ZehXuTwWcMx/8iNv+uv6YtkN1+R5Lq8vLz179goYOD1f86fNeg36RujImQttb1w4MfGR0Z6pPdI4/kPHpPvsbHIwvzWFcSU6PPf7NMVJ9SnNtePwBOiJqz7q9gai7JkL0V7iqTwIHhhfm2GL2+Ds8kuojRLwm3rWIq58Ipt+8uelgeDDpku21yWwWy3YdSerfH4hQj39/P9B8bFTKOVZ0l++xnHz/B3G9pqT6pKaMCfwr6oI9/i1OL+19zEAbS+fg3s5dEgoMMjnwb3F4aXF6GV+SS3lJDlVNPfX4e8mxmaOC+s7Bk52BlfT4i4FK9My9EOzxjzWBl0t6/IWIm9awqiL2QN2BiKcIRG/X9e89uzYRzRlUsQY1f7Szlq01LUBwsPIPjnEvvxiaMiZCC/X4QzAfLl4urx+LSYXzjCGix9+XuXn+lY3BD0zlpTmMK87hYLMLr7/79jY5PFFpPtCZ7x9Z0lN6/MVAWROc4w/BD6fNztg9/nYJ/IXo0faDrdz71la01nS4fUxYsCjZTUpbh1tcrN3fGDOt561NB/loZ/COyU1/Wcflf/wMCA5WXrz18KC2s67NTWuMYggis2TM4N6Kug6KjN69miYnU0fkx/V8p9ffbRbPodDjH6rhP6E0lxaHF39Ac6jZRXlpTtR6zQ5vVJoPEJ7Mq7FLj39eVsYcVmIQHYsc/0K7leYY5TydHj92qxynQvTk+mdWU9/u4ScXTInrLrrM2trdnF8vBeDmeROpbnKQn21lxqiC8OO3/20TWx64JFnNA2DX4baE3zmRVJ/UlBFXvkBAs6++gwumD2fR5kNUD6jHP0BWD4G/05O5gf8Bo4Z/eUlOOICvanJ0C/xj9vjnxO7xL82VnlQRP4vp2PT4t7l9eP2BqPkBnF5/VOleIURsOs5htT0Vw1i7P3EpQunq6c/2JbsJPXpw8c5kN0EMkoy48h1qdeH0+pkzsQSrWQ2oso/b2z03PRT4uzN49t7KBgcjCrKw28zhYD/WAN9YPf4leUaOvzF40usPcKTNRX52RnyeFIPMkuCZe6FzMHpLl9vXDo+PHJscp0LEsnJvA/Wh8pk6vsmmfvTShpjLv5VJs+2KfpGbP92lwl2QjLjyVRj5/ccNz2dUoX1AOf4xU30soao+GZzj3+BgfEkuACMLsrGaVczAv7Gje49/fpYFi0mFe/zf+eIg9e0erjhl9LFvuMg41lBVn2MQ+Dc7PAzLywKCdwhd3kC3/3chRNB3n4x/BtyDzU7OfPCjY9AaIUQiZUSPf6iiz+SyXMYU2QdU0tMVYybPoZDjX9nYEe7pN5sUY4rs3QJ/f0DT6vKGc/pDlFIU59po7PCgtebxTyqYNiKf86aVDVr7ReYIVfOJHGB/tEIfViMr+7h83Uv3CiEGrsPj58Zn1yS7GSLFpELvtuguQwL/dvKyLJTlZzG6yD6gVB+XN9Bjqk+m9vg7PX5qW91MiMjnH1eSQ3WXwL/F6UXrYGnErkpygoH/x1/Wsau2jVvPmRRXiTYhQkI9/ceixz9yEq/QmB0p5ylE33rL1qhvc4d//rK2vZc1xVB0uMWV7CaIGDIi8N9b18HkslyUUowptlPb5sITZwlOZ8wef1P4sUwU6tkvL80NL4s1iVesWXtDSnJtNDk8PP7JXkYVZvN1SfMRA3QscvzDc01ETOLl8HSfpVsIEVtvedrnPfTxoLVjqDorjdOn9hrZGCK1ZETgX1HXzqSyPADGFtnRGmpb4/ukGSvVx57hqT6VRkWf8SWdPf7lJTk0Oby0uTp7SGPN2htSkmtja00rqyoauXneRGyWjDikRBIci6o+4cG9kak+XunxF0Kkh4GkLgvRm7SP0hweHwdbXEwaFuy1HlNsB6A6znQfty/QLfDPyvjAv7OGf0i58SGgqrFz/zUZddBj9fgX51pxev3kZ1u4enb5sWyuyHBWc+Jn7s0zBqDH6vGXwb1CCCEGUyqkQg/4CquUGqeUWqaU2qGU2qaUut1Yfr9SqkYptcn4mp+45nYXHtg7PNjjP7ooGPjH+yk5OKFP1xz/4O/uDJ25t7Kxg0K7lcKInvxQ4B+Z7tNrqo+x7Lo542XiLnFUQik+oZSfRFBKUZRjjcrxDwf+0uMvRJ+anR52Hm5NdjOEEAlyNJGaD/iZ1nqDUiofWK+UWmI89j9a64eOvnl9q6gPBv6TyoK91qMKswHiHuDr8nVP9bGZTSiV2T3+47tM1DWuONTj3xn4hyqiFOV2T/WZOjKfohwrN5454dg1VAwJdquZkQXZ4Q+fiVJot9LijJhdOpzqIx9UhejLN/93BR1pNonlkVYXC9dW8eMLjkuJHlYhUsmAr3xa60PAIePnNqXUDmBMohrWXxV17SjVma6SbTVTlp8Vdy3/WDn+SimyLeaMDvxPGVcUtawwx0pBtiWqx7/R4cFiUuTH6NG//OTRXHrCyISWYBRDk8VsYtXdFyb8dYtzbOF0NZBUHyFiOdDgoN3tY8bogqjl6Rb0A9yxcBMr9jZw7rQyTh5b1PcTkk0+m4hBlJBoTSk1AZgJrDYW/VgptVkp9YxSqjgRf6Mne+s6GFtsjwraY9Xyf/bzfTy6bE/M19A6OKFPdoyBqXabOSOr+nj9AWqanVEDe0PKS3OoauoM/A80OijKsfbYcyJBv0hlRTk2miNm7nXK4F4hujnnd8uY/4flUYUd0lXow4ovkCZTx3Zp5hsbq5PTDjEkHHXEppTKA14H7tBatwKPAZOBUwneEfjvHp53q1JqnVJqXV1d3YD/fkVdO5OG5UUtG1McHfj7A5o/LN3Nnz/Ziz/GiSCUw58dIxDItpgyso5/TZMTf0CHJ++KNK64s6Tncyv2s2jzIeafNGqwmyhEQhTlWMOVqQCcHh8g5TyFiOWk+z9IdhOOWl8d6AcaHGyubmZrTUtC/+77Ww+xvrIRgIZ2Ny0OLx9ur437de5c+EVC2yVEpKNKclVKWQkG/S9qrf8OoLWujXj8SeDdWM/VWj8BPAEwa9asAX0s11qzr76D2RNLopaPKbKzZHstgYDGZFKsr2wKD+7bcaiVE8cURq0fSuXJtsQI/K2ZmepT2di9ok9IeUkOS3cc4c2NNdz39jYumj6Cey+fMdhNFCIhinOsUVV9pMdfiNQwYcGihL9mU4eHTVXNva5zzu+WhX+u+NV8TAkoIXz3G1t4afUBAD77+fnM+82yPp7Rqc3tiyo5LDJXKmR1HU1VHwU8DezQWj8csTyya/ifgK0Db17vDre6cHj8TC7r0uNfZMfjC1DfEZxV8MMdteHa4KsqGrq9TqhHP1YPYJbVnJE9/nuOBGdZnBCrx78kB48/wE9f2cQZE0v40zUzJZ1HpK3SvCxc3gBHjLk9ZAIvMdTc/Je1nHT/P3p8PPKOWLr72audveWRk485PX4eeGcbDuOOX8jbXxxMyN8NBf0AB5vjn7H2mqdWJaQdQvTlaKK5s4DrgAu6lO78rVJqi1JqM3A+cGciGhrL3iPRFX1CxhglPQ82u9Bas2R7LWceN4yJw3JZubd74B/qAbTbuu+ObKsJt6+zx//BxTv55mMreG7Ffhra3d3WTxdr9jUwttjO8ILsbo+FqqpMH1XAkzfMkgBJpLWLZ4wA4LUNwbxZp9dPlsWU0BmChUiWNfsaWbbzSK/rLN15hDaXr8fH0yUVvj8+irEvPL4Az3y+j2c/388Nz6yJeuzzPfUAdLh9TFiwiKc/23fUbRhIIaFtB6VkqhgcR1PV5zNi37V4b+DNiU9FfbDXuluPvzGJV02Tk7wsM/vqO7hp3kS2H2zh3S8O4Q/oqIt+r6k+FjNOo4fQ5fXz/Mr9KGB9ZRO/eHc7500bzm+vOpmS3O417lNVIKBZva+Ri6ePiPn47Ikl3HnRVK6dU05BdvcSnkKkk8llecyeWMLCtVXcds7k4JwdkuYjMsS3H18JwP4Hv5bklqSeqkYHVrPiij99Hl62dn9T1Dqvrq/md986hRPuC94ReWp5BTfPmzio7RRiMKV1/obL62d0YTbD87OilndO4uVgyfbgp/+Lpg9nzqRS2tw+tnf5ZB0O/GP0bNttZlxGj/+KvfU4PH4evfY0Ft9+NjfNm8iHO2r525oD3Z6XynbVttHs8DJnUmnMx7OtZm6/aArD8rJiPi5EurlmdjmVDQ5WVTTg8PjJkbtYQoRpnUFd/hHuWLgpKujvj0ydsFOIkLQO/G89ZzIr7rqwW5nJQruV/CwLNU1Olmw/zEljChlVaGeuEeiurKiPWt/ZS+Cfbe2s6rNk+xFybWbmTi5l+qgC7p4/nRNGF/DJlwOvSpQMoXSnOZNjB/5CZJpLTxxJod3KS2sO4PRKj7/IPDf/ZS2HW3rPLb//7W38dVUl837zERMWLOLhD3YNUutSW+QHn8aOox/vIEmEIpWldeDfmzHFdr6obmFjVTMXGSktwwuymVSWy6qKxqh13eHBvTFy/I0JvAIBzYc7ajl3WhlZESlBZ08pY0NlU1rVPl5V0UB5SU54LIQQmS7bauafZo7hg2211DQ5JfAXGWfp0q2JuwAAEwNJREFUziM8snR3r+v8ZcV+7nlzK9XGzPZ/+GgPExYsirtXXAgxMKkwkXTmBv5FdjZVNaN15+A+gDmTSlm7rxGfv/N2Xm+pPqGqPl9UN1PX5o56LYBzpg7DF9AxBw2nolB+/5xJJX2vLEQG+e7scjz+AJuqmsmxHlUlYyFSlj+gOdLmorHDQ7u75wG9kbpOeDnUdM10Wl/ZROtRdOalQnAnRE8y9uoXyvMfU2Rn+qj88PI5k0p5afUBth1s5ZRxwam8w1V9ekj1cXv9fLijFrNJcf604VGPzxpfQo7NzPLd9Xz1hJEJ3w6tNct313PimMK4BxAfbHZyqMXF6eM7J0/ecbiVFqeXuZLmI4aYaSPzmVlexMYDzTEn6xMi3Xn9AU7/5RKaI2rCSydP/L752AoAdv7iUqlqJzJO5vb4G5V9Lp4xImoMQOgkGFnPv7c6/narGafXz5LttXxlQjFFOdHBt81iYu6kUj7dHV+evz+go+46xNLi8PLDFzdw/TNr+Nkrm+J6fYCfv76Z7z65iipjsi4gnOZ0xkQJ/MXQ893Z5QAyuFekrI921vLYx3v7ta63yzXktfXVUUE/0C21VXTX09Dmf/7fFQN8RenyF7GpFDg2MjbwD01M9dUuqTnD87OZXJbbJfAPpfrEquNvxhfQfFnbzsUzYvfonzO1jMoGB5UNHf1qW2OHh/mPLOeaJ1f3GPyv2dfIZY98ypLttZw9ZRjLdtWF6w33R1Wjg+W76/H4AjwUMYBrVUUD40tzwndEhBhKLj95FPlZFgrtUqZWpKab/rKO37y/M2qZP6DZX9/BvW9t5f2thznQ4OCOlzcy5d8XJ6mVmaWnqkbbD0ltfZFYusePmYMnY1N9Lpo+gudvmh0zpWXOpFLe2nQQnz+AxWzqs6pPSE9178+eMgyAT7+s47q5uTHXCWl3+/jes2vYW9eOL6B57OO9/OTCKVHrPLdiPw+8s41xJTm8/oMzmTYyn4se/oT/WrSDd38yr1/Tiy9cW4VJwTdmjuHvG2q4ed5EThhdyOqKBuafNKrP5wuRiXJsFhb+y9y0mndDpLd2t489R9o51UgtDfEHNCYFSikCAU2ryxsedAuw4PXNfPsr47r1Oj+/snJQ2i2CVlU0hEtfu7x+KhscrNxbz41nSa1/kZ4ytsffYjZxztSybqU+AeZOLqXd7eOBd7bz8Ae7+OTLOpSCLEvsHn+AaSPyKTfuInQ1cVguY4vtfPJl7z3ybp+f215Yz9aDrTz2/07nilNG88jS3Wypbgmv89r6au57exsXTh/Bon89m1PGFZFtNfP/L5nG9kOtvLGxps9t9/kDvLq+inOnlvHAFSdQmmvjvxbtYMehVlpdvh7r9wsxFMwYXcDIwu4zVgsRrz1H2qlr630G9x/8dT3fePRzHJ7gQFuPL8DqigYm3/0e1xuzyN739jZO/c8lXP7Hz8LPe3lt1VGkmoh47K3r+W797S9v5Eiri1ueW8vx//E+l/z+U+5/ZzsvrJIPYCJ+P3pxQ7KbkLmBf2/OnDyMYXk2/rq6kj8u28Pa/Y3MGFUQ80NCaDbfrtV8IimlOGdqGSv31nfLuQzx+gPcuXATn+2p5zffPJmLZ4zgF1eeyLC8LO58ZRMuYxzBz1/fzLzjhvGna2aSl9V5Q+brJ4/mlLGFPPTBrnBqEkBdm7tbKdFlu+qobXVz9exy8rOt3HHRFFbva+TXi3cASOAvhBAJcNHDn3Dmg0tpdnjYcKCJafcs5ocvrsfl9VPV6ODk+//B8t3BDiGPL0Czw8PUexbznSdWAbB8dz1LttdKEJlkl/z+0x4fq211M/tXS/lwx5Go5f/x5tYenyNVfURPNhxoTnYTMjfVpzcluTbW3XNxv9YtzQumBFzSR8Wec6aU8dLqA2yobOIMI7DWWrOpqpk3N9bwzuZDNHZ4uOdr07nq9LEAFOZY+d23Tua6p9fwoxc3sHxPsHrP49edHjVXAIDJpLh7/nS+88QqHl22hwmlubyxsYbP99YzoTSXV2+bG55p9+U1ByjLz+KC44MViK6eXc6zK/bz+Z4GJg7Lld5OIYQYoEBA49caqznYb+b1a079zyXhx9/bcpj3trzf7Xn3vLmVdzcf6rb8+8+vO3aNFUKILoZk4B+P86YN54M7z2HqiPxe1zvzuFLMJsWnu+sYVWjnjY01vLmphn31HdgsJi6eMYJvnT6W87qUAz17Shk3zB3PcysrOW54Hs/e+BVys2K/LWdMKuXiGSP440d7ACgvyeGmsyby4upKbnhmDS/fOod2t49lu45w27mTwxcmq9nEgkuP59YX1ktpNyFEylBKXQo8ApiBp7TWDya5SVEmLFgEwLYHLsFsUvgCmhPv+wcA15xRHtdrxQr6RWaSDn+RyiTw74PZpPoM+gEKsq3MHFfEU8v38eiyYCm2OZNKuO3cSVx20igKsnuuIrLgsumMKMzmn2eO7XPQ4QNXnMDUEXmcP204p48vRinFvCnD+P5z67jluXXMmlBMQMN3vjIu6nkXzxjBvZfP4NxpZf3YaiGEOLaUUmbgUeBioBpYq5R6W2u9PZF/Z+mOWvbWtXPB8SPYX9/BRTNGUFHXTmWjg/OnDWdVRQOf7a7nXy+cwr+8sI5lu7qXZj7BCPYjvbT6QCKbKdLQjc+u4bITR/LtWdHX21hpwyL9XX7yqGP+Ab6xw3PMi0+onspYDaZZs2bpdevS/3bnGxurefbz/Vxywki+MXMMYwaxZOZbm2q4Y+EmtIazjivlxVvmDNrfFkIcW0qp9VrrWcluRyIppeYC92utLzF+vwtAa/3rWOsP9Dpx71tb+euqSgLGpe6bp43l9Q3VA222EH36+imjeeeLg8luhujDWceVclp5MbedO5k1+xuZNiKf0UV2altdfPV/PqXFGRw/+dT1szh5bCHDC4Jp0lWNDhZvPcT3z57EB9trmTGqgLHFdi57ZDmXnjiS33+4e8Bt+v13TuUbM8fE/bx4rhES+GeQ51bs5763t/H4daf3OSZBCJE+MjTwvwq4VGt9i/H7dcAZWusfR6xzK3ArQHl5+emVlfEPgt11uI2v//EzPH1MmCiEyEzzTxrJKWOLmH/SKMYW2/t9R8bl9ZNlMQ3oDk5tq4vh+VlU1Hdw4X9/wrC8LEYVZrOlpqXX56286wJGFcbfaRzPNUJSfTLIDWdO4IpTRlMsNcqFEKkv1tU0qidKa/0E8AQEO4gG8kemjcxn5V0XcLjVBUBZfhZOj5+qRicmE7Q6fYwszGZiaS71HW6Kc2wooKHDQ4HdQkmOjQONDswmxYiCbMwmRUBrHG4/Tq8fm8WE1sFZ3LMsJtzeAIU5Vnz+AC5fgDaXlxH52XR4fNgsJhSKNpcXpRQ2i4lmh4c2l4/xpTnUtrqxmBRef4AOt5+GDjdjiuxYzSZ8gQBji3OobnLS5PBQaLcyttiO2aQ40uqmor6D6aPyKci2EtCadpcPk0nR5vJxsNnJ8SPzcfsCOL1+rCYTAa2pb3djMimG52fR4faHi1nYLCbysyzhgMfl9YdLWwcCOjyXjNY6aqbg/GwLDuP1AbyBALk2C00ODxaTIi/LgsUYe+bxBWhyeDjc4mJSWS65NgvNTi/+gCY/24IvoMnLsuAPaBTg8Qdod/vItpqxW814/QE8/gA2swmr2YTW2nhvIDTVjVIKnz+A2aRw+wK4vQGyrMH3yGYxkW0NzuPjC2jybBZaXV5ybBYsJoXHeF6r04vD48diVnS4/djMJvKyLTS0u3H7AuHHPL4AgYDGF9CMLMym2RHcFofHR0mujaIcG/XtbkpzbRTarZiUot3tIzfLQpvLS7vbR02TkwnDcmlz+RhXbKfd7WNUoZ0DjQ5sFhNtLi+ji+xYTIrN1S2cMq6IHJsZk1KYTQqnx4/ZFDyufP4AJqXYfaSd0UXZ+AOaQrsVpRRaaxwePzm24ASlHl8gfFybTcFtyc+2EgholDHXhNY6fDwEjNtnJmNdXyBgvAfBYye0jlLBwe9Wc3D/m5TC5fOTb4xj7PAE96fF+Nu+gI4ZaLt9/m4FTwZDrLmd+muEcXdgclke+x/8WqKalBDS4y+EECkuQ3v8ByXVRwghMl0814ghWcdfCCFE0q0FpiilJiqlbMDVwNtJbpMQQmQ0SfURQggx6LTWPqXUj4F/ECzn+YzWeluSmyWEEBlNAn8hhBBJobV+D3gv2e0QQoihQlJ9hBBCCCGEGAIk8BdCCCGEEGIIkMBfCCGEEEKIIUACfyGEEEIIIYYACfyFEEIIIYQYAiTwF0IIIYQQYgiQwF8IIYQQQoghQGmtk90GlFJ1QOUAnz4MqE9gczKN7J/eyf7pneyf3g3W/hmvtS4bhL+TsuQ6ERfZ3swm25vZBrK9/b5GpETgfzSUUuu01rOS3Y5UJfund7J/eif7p3eyf9LDUHufZHszm2xvZjvW2yupPkIIIYQQQgwBEvgLIYQQQggxBGRC4P9EshuQ4mT/9E72T+9k//RO9k96GGrvk2xvZpPtzWzHdHvTPsdfCCGEEEII0bdM6PEXQgghhBBC9CGtA3+l1KVKqV1KqT1KqQXJbk+yKaXGKaWWKaV2KKW2KaVuN5aXKKWWKKV2G9+Lk93WZFFKmZVSG5VS7xq/T1RKrTb2zUKllC3ZbUwWpVSRUuo1pdRO4xiaK8dOJ6XUncb/1Val1N+UUtly/KS+TLhOxHtuV0F/MLZ5s1LqtIjXusFYf7dS6oZkbVN/9Pd8rZTKMn7fYzw+IeI17jKW71JKXZKcLelbPOffTHh/4zmfpuP7q5R6Ril1RCm1NWJZwt5PpdTpSqktxnP+oJRS/W6c1jotvwAzsBeYBNiAL4AZyW5XkvfJKOA04+d84EtgBvBbYIGxfAHwm2S3NYn76KfAS8C7xu+vAFcbP/8Z+EGy25jEffMccIvxsw0okmMnvG/GAPsAe8Rxc6McP6n9lSnXiXjP7cB8YDGggDnAamN5CVBhfC82fi5O9vb1st39Ol8DPwT+bPx8NbDQ+HmG8Z5nARONY8Gc7O3qYVv7ff5N9/c33vNpOr6/wDnAacDWiGUJez+BNcBc4zmLgcv627Z07vGfDezRWldorT3Ay8CVSW5TUmmtD2mtNxg/twE7CP6DXUnwpILx/RvJaWFyKaXGAl8DnjJ+V8AFwGvGKkN53xQQPFE9DaC19mitm5FjJ5IFsCulLEAOcAg5flJdRlwnBnBuvxJ4XgetAoqUUqOAS4AlWutGrXUTsAS4dBA3pd/iPF9H7ofXgAuN9a8EXtZau7XW+4A9BI+JlDKA82/av7/Edz5Nu/dXa/0p0NhlcULeT+OxAq31Sh38FPA8cVx70jnwHwNURfxebSwTgHErbCawGhihtT4EwQsIMDx5LUuq3wP/BgSM30uBZq21z/h9KB9Dk4A64Fnj1vpTSqlc5NgBQGtdAzwEHCB4gWoB1iPHT6rLuOtEP8/tPW13Ou2PeM7X4e0yHm8x1k+X7Y33/JvW7+8Azqfp/v6GJOr9HGP83HV5v6Rz4B8rn0lKFAFKqTzgdeAOrXVrstuTCpRSlwNHtNbrIxfHWHWoHkMWgrclH9NazwQ6CN6KFICRi3klwdvJo4Fc4LIYqw7V4ydVZdT/eBzn9p62Oy32xwDO12m9vcR//k3r7R3A+TStt7cf4t2+o9rudA78q4FxEb+PBQ4mqS0pQyllJXhheFFr/Xdjca1xawjj+5FktS+JzgKuUErtJ3i7/wKCPUpFxq1GGNrHUDVQrbVebfz+GsELkRw7QRcB+7TWdVprL/B34Ezk+El1GXOdiPPc3tN2p8v+iPd8Hd4u4/FCgmkW6bK98Z5/0/39jfd8mu7vb0ii3s9q4+euy/slnQP/tcAUYxS4jeCAj7eT3KakMnLengZ2aK0fjnjobSA0GvwG4K3Bbluyaa3v0lqP1VpPIHisfKS1vhZYBlxlrDYk9w2A1vowUKWUmmYsuhDYjhw7IQeAOUqpHOP/LLR/5PhJbRlxnRjAuf1t4HqjWsgcoMVILfgH8FWlVLHR6/pVY1lKGcD5OnI/XGWsr43lVxtVYSYCUwgOikwpAzj/pvX7S/zn07R+fyMk5P00HmtTSs0x9t/1xHPtOZpRy8n+IjgS+kuCI7n/PdntSfYXMI/g7Z7NwCbjaz7BXLilwG7je0my25rk/XQenVUiJhE8UewBXgWykt2+JO6XU4F1xvHzJsEqAnLsdO6fB4CdwFbgBYKVJOT4SfGvTLhOxHtuJ5gK8KixzVuAWRGvdZNxvO4BvpfsbevHtvd5vgayjd/3GI9Pinj+vxv7YRdxVD5Jwnb2+/ybCe9vPOfTdHx/gb8RHL/gJdhDf3Mi309glrHv9gJ/wpiQtz9fMnOvEEIIIYQQQ0A6p/oIIYQQQggh+kkCfyGEEEIIIYYACfyFEEIIIYQYAiTwF0IIIYQQYgiQwF8IIYQQQoghQAJ/IYQQQgghhgAJ/IUQQgghhBgCJPAXQgghhBBiCPg/KzhEO4SH6eYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "num_frames = 10000\n",
    "batch_size = 32\n",
    "gamma      = 0.99\n",
    "\n",
    "losses = []\n",
    "all_rewards = []\n",
    "episode_reward = 0\n",
    "\n",
    "state = env.reset()\n",
    "for frame_idx in range(1, num_frames + 1):\n",
    "    epsilon = epsilon_by_frame(frame_idx)\n",
    "    action = current_model.act(state, epsilon)\n",
    "    \n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    replay_buffer.push(state, action, reward, next_state, done)\n",
    "    \n",
    "    state = next_state\n",
    "    episode_reward += reward\n",
    "    \n",
    "    if done:\n",
    "        state = env.reset()\n",
    "        all_rewards.append(episode_reward)\n",
    "        episode_reward = 0\n",
    "        \n",
    "    if len(replay_buffer) > batch_size:\n",
    "        loss = compute_td_loss(batch_size)\n",
    "        losses.append(loss.data[0])\n",
    "        \n",
    "    if frame_idx % 200 == 0:\n",
    "        plot(frame_idx, all_rewards, losses)\n",
    "        \n",
    "    if frame_idx % 100 == 0:\n",
    "        update_target(current_model, target_model)\n",
    "        "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}