{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Policy Evaluation\n",
    "\n",
    "Let's try to evaluate a policy using Monte Carlo and TD(0).\n",
    "\n",
    "The world and the policy are the following:\n",
    "\n",
    "<img src=\"http://www.cs.upc.edu/~mmartin/Grid+world.jpg\" width=\"350\">\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "State Matrix:\n",
      "[[ 0.  0.  0.  1.]\n",
      " [ 0. -1.  0.  1.]\n",
      " [ 0.  0.  0.  0.]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "#Define the state matrix\n",
    "state_matrix = np.zeros((3,4))\n",
    "state_matrix[0, 3] = 1\n",
    "state_matrix[1, 3] = 1\n",
    "state_matrix[1, 1] = -1\n",
    "print(\"State Matrix:\")\n",
    "print(state_matrix)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reward Matrix:\n",
      "[[-0.04 -0.04 -0.04  1.  ]\n",
      " [-0.04 -0.04 -0.04 -1.  ]\n",
      " [-0.04 -0.04 -0.04 -0.04]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUgAAAEECAYAAABKjq0kAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAACQFJREFUeJzt3VGIpXd5x/Hfs9kdFSzkYksqs1kii4SGQBVLEAJeBKGrluamFwbqlbAUVGJYG9Ir8b4YBb1ZTFCoRAQVRCwSaEoo2NaYRkm6FYKUZFcxlLSNChriPr2YuYjJPp53zZmc48znAwNzZv7n5eG/w3fec868Z6u7A8CrHdv0AADbSiABBgIJMBBIgIFAAgwEEmCw1kBW1dmq+mFVPV1V963z2IdJVT1YVc9V1ZObnmWbVdWNVfVIVV2sqqeq6u5Nz7SNquqNVfVvVfX9/X365KZn2mZVdV1V/XtVfXPV2rUFsqquS/K5JO9NckuSu6rqlnUd/5D5QpKzmx7i98BLSc539x8neVeSD/uZuqpfJbmju/8kyduTnK2qd214pm12d5KLSxau8wzytiRPd/ePuvvFJF9Ocucaj39odPejSZ7f9Bzbrrt/0t2P73/+s+z9UO9udqrt03t+vn/zxP6HK0CuoqpOJXl/ks8vWb/OQO4mefZlty/FDzNrUlU3JXlHkn/d7CTbaf9h4xNJnkvycHfbp6v7dJJ7k1xZsnidgayrfM1vMV6zqnpzkq8m+Vh3v7DpebZRd/+6u9+e5FSS26rq1k3PtG2q6s+TPNfd31t6n3UG8lKSG192+1SSH6/x+BxBVXUie3H8Und/bdPzbLvu/t8k/xTPcV/N7Un+oqr+K3tPAd5RVX//2+6wzkB+N8nbquqtVbWT5ANJvrHG43PEVFUleSDJxe7+1Kbn2VZV9YdVdf3+529K8p4k/7nZqbZPd/9td5/q7puy16d/7O6/+m33WVsgu/ulJB9J8u3sPZn+le5+al3HP0yq6qEk30lyc1VdqqoPbXqmLXV7kg9m7zf9E/sf79v0UFvoLUkeqaofZO9E5eHuXvknLKxW3u4M4OpcSQMwEEiAgUACDAQSYLAykC6EB46qJWeQ13whfFWdW8dwh519WsY+LWevllm6TysD+TteCO8faRn7tIx9Ws5eLbOeQCYuhAeOpmv6Q/H9y5m+nuSj3f3kK753LvtV3nnDG955w6438lnl2JUruXLM62Sr2Kfljvevc+L4ojeqOdIuX76Un//iytXeYOc3XPOVNFX1iSS/6O6/m9acPnOmT/z1h6/puEfRPad3c/8zlzc9xtazT8s98I6dvPvWz2x6jK132589m8e+/8uVgVzyKrYL4YEj6fiCNW9J8sX9/1LhWPbehMKF8MChtzKQ3f2D7L2TM8CR4plvgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgxWBrKqbqyqR6rqYlU9VVV3vx6DAWza8QVrXkpyvrsfr6o/SPK9qnq4u//jgGcD2KiVZ5Dd/ZPufnz/858luZhk96AHA9i06u7li6tuSvJoklu7+4VXfO9cknNJcvLkyXd+4rOfXd+Uh9QNOzv56YsvbnqMrWeflvujHM//XPq/TY+x9c5//ON5oZ+vVeuWPMROklTVm5N8NcnHXhnHJOnuC0kuJMnpM2f6/mcuX8O4R9M9p3djn1azT8vdd+z6fOVv/mHTYxwai17FrqoT2Yvjl7r7awc7EsB2WPIqdiV5IMnF7v7UwY8EsB2WnEHenuSDSe6oqif2P953wHMBbNzK5yC7+5+TrHwyE+CwcSUNwEAgAQYCCTAQSICBQAIMBBJgIJAAA4EEGAgkwEAgAQYCCTAQSICBQAIMBBJgIJAAA4EEGAgkwEAgAQYCCTAQSICBQAIMBBJgIJAAA4EEGAgkwEAgAQYCCTAQSICBQAIMBBJgIJAAA4EEGAgkwEAgAQYCCTAQSICBQAIMBBJgIJAAA4EEGAgkwEAgAQYCCTAQSICBQAIMBBJgIJAAA4EEGAgkwEAgAQYCCTAQSICBQAIMBBJgIJAAA4EEGAgkwEAgAQYCCTAQSICBQAIMBBJgIJAAA4EEGAgkwGBlIKvqwap6rqqefD0GAtgWS84gv5Dk7AHPAbB1Vgayux9N8vzrMAvAVjm+rgNV1bkk55Lk5MmTuff07roOfWjdsLOTe+zTSvZpuet3dnLnQ3+56TG23qPn/2XRurUFsrsvJLmQJKfPnOn7n7m8rkMfWvec3o19Ws0+LWev1sur2AADgQQYLPkzn4eSfCfJzVV1qao+dPBjAWzeyucgu/uu12MQgG3jITbAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAYFEgq+psVf2wqp6uqvsOeiiAbbAykFV1XZLPJXlvkluS3FVVtxz0YACbtuQM8rYkT3f3j7r7xSRfTnLnwY4FsHlLArmb5NmX3b60/zWAQ+34gjV1la/1qxZVnUtybv/mr3Lv+Sdfy2BHwUeTk0n+e9NzbDv7tJy9WuzmJYuWBPJSkhtfdvtUkh+/clF3X0hyIUmq6rHu/tMlAxxl9mkZ+7ScvVqmqh5bsm7JQ+zvJnlbVb21qnaSfCDJN17LcAC/D1aeQXb3S1X1kSTfTnJdkge7+6kDnwxgw5Y8xE53fyvJt67huBd+t3GOHPu0jH1azl4ts2ifqvtVr7cAEJcaAowEEmAgkAADgQQYCCTAQCABBgIJMPh/9EW6o+bXfk0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 384x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Define the reward matrix\n",
    "reward_matrix = np.full((3,4), -0.04)\n",
    "reward_matrix[0, 3] = 1\n",
    "reward_matrix[1, 3] = -1\n",
    "print(\"Reward Matrix:\")\n",
    "print(reward_matrix)\n",
    "plt.matshow(reward_matrix,extent=[0, 4, 0, 3])\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAADMCAYAAABTJB73AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADo5JREFUeJzt3XlwnPV9x/H3V2vJumJbPjCOD2BI7MYxgxs7LsEEggOOiZXQI9MBJsdM3bowMTXXUNI0pWGY6bRMSD2JZxgPRxrqQiCkBSTAlrBB4cYGQwQOxEDAsjEK8aldHavVt39IBJtg7Vrso2ef/X1eMx4k/Gjn+3j3vc+zq+fZx9wdEQlLRdwDiMjoU/giAVL4IgFS+CIBUvgiAVL4IgEqavhmtszMXjGzHWZ2TTFvO25mdquZdZpZe9yzRMHMZprZZjPbbmYvmdnquGcqJjOrNrNnzOyFofX7ftwzFZuZpczseTNryrds0cI3sxSwFjgPmAtcaGZzi3X7JeAnwLK4h4hQP3Clu38KOA34dpndf73AEnc/FZgPLDOz02KeqdhWA9sLWbCYW/xFwA53f93d+4A7gfOLePuxcvc2YG/cc0TF3d929+eGvj7E4ANoerxTFY8P6hr6tnLoT9kcvWZmM4DlwM2FLF/M8KcDOw/7voMyeuCExMxOBP4UeDreSYpraFd4G9AJtLh7Oa3ffwJXAwOFLFzM8O1D/l/ZPKOGwszqgXuAy9z9YNzzFJO759x9PjADWGRm8+KeqRjMrBHodPethf5MMcPvAGYe9v0MYHcRb18iZmaVDEa/3t1/Efc8UXH3/cAjlM97NouBr5rZbxl8ib3EzP57uB8oZvjPAp80s5PMrAq4ALiviLcvETIzA24Btrv7jXHPU2xmNsXMJgx9XQOcA/w63qmKw92/4+4z3P1EBrvb5O5fH+5niha+u/cDq4ANDL4xdJe7v1Ss24+bmd0BPAnMMbMOM1sR90xFthj4BoNbi21Df74c91BFNA3YbGYvMriRanH3vL/2Klem03JFwqMj90QCpPBFAqTwRQKk8EUClDf8kZzcYGYrizNe6SnndQOtX9IVun6FbPFHcnJDOf/jlvO6gdYv6QpavzH5FvDB3/eV7ckNIiEq6Pf4Q6fcbgU+Aax193/8kGVWMvRsM3Zs1YITZk0t8qilIdtfQeWYgs6DSCStX7Lt2tVBV3rgw86bOcIxHcAzdMjj/wKXuvtRP5BizuxZvr1tbMG3myRt7as5c96auMeIjNYv2RZ9aSdbXujJG/4xvatfhic3iASpkHf1y/bkBpFQ5X1zj8GTG/5r6HV+BYMn3wR7coNIOSjkXf0XGfw0FhEpEzpyTyRACl8kQApfJEAKXyRACl8kQApfJEAKXyRACl8kQApfJEAKXyRACl8kQApfJEAKXyRACl8kQApfJEAKXyRACl8kQApfJEAKXyRAZR9+V3qAbLZ8L/yz/0COY7k2QtLs25+Le4SyVPbh9/U5sz/3JhdevIf19xxi777yeiC98lqWExb8lkuu7qSpJU13d3ldJebOe7uYd+abXHP9uzz2dDe5XPk+yY2mQj5eu6Q9tCnNI090D7tMXa1x171d3HVvF6kULP5sNY1L61h+Th1zPlGJWd4Lj8Tm5vUH2PFGdthlMt3OutsPsu72g9RUG188s4bGc+toPLeOaVNL+y6+7gd7yQzzZNXb62z/TZbtv9nPDWv3M7GhgvOW1NG4tJYvfaGW8eNSozjtsXloU5rp08ZwyqdK76pSpf2oKMAvn+rhhrX7C14+l4O2p3p4/NkemjamufziBr6ytLZk47/7vi5a24Z/Yjtcd4/TtDHDw23dPNCa4Z8vb2DBqdURTvjR/OiW/ezdV/heyt59A6y/5xAPbUpz/nn1/MsVDcycXhnhhCN37X/s5QuLa/j37yn8orvi4gn8zUXjhl3mm6v28NTWXsaPq2DZkloaz61j2dm1TGwo3a3Fe25bM5XunqPv3vb2Omf9RQd79w0wfVqK5efU0bi0jiWLa6ipKf1Xcs88NJOBYbp//c0syy7YDcDc2VU0Lq1l+Tl1fG5hNalUaT5Zv+eJ5hlUlOhdkPjwJ01MMWni0QPeuSvLaQuquf47kzhjUQ2VlaX9YPmgjx8//F306BPd/MPfTqDx3Drmz6sq2T2Xozlp1vBb68ee6eaH101m+bl1nHxiaW7Zj6aUn5gSH34+M6dX8oPvT4l7jMicdXoNZ51eE/cYkfnWXw+/NycjU6I7IiISJYUvEiCFLxIghS8SIIUvEiCFLxIghS8SIIUvEiCFLxIghS8SIIUvEiCFLxIghS8SIIUvEiCFLxIghS8SIIUvEiCFLxIghS8SIIUvEiCFLxKgvOGb2Uwz22xm283sJTNbPRqDiUh0Cvl47X7gSnd/zsw+Bmw1sxZ3fzni2UQkInm3+O7+trs/N/T1IWA7MD3qwUQkOnYsl1g2sxOBNmCeux/8wN+tBFYCTJkyecFd/3Nt8aYsIV3dU6mveSfuMSKj9Uu2q668ii0v9OS9hE/BV9Ixs3rgHuCyD0YP4O7rgHUAc2bP8jPnrTmGcZOjrX015bpuoPULRUHv6ptZJYPRr3f3X0Q7kohErZB39Q24Bdju7jdGP5KIRK2QLf5i4BvAEjPbNvTnyxHPJSIRyvsa390fA0r3er8icsx05J5IgBS+SIAUvkiAFL5IgBS+SIASH/6Bg7m4R5ARGhhwDh4aiHuMICU+/H+9YW/cI8gIbXmhl/s3puMeI0iJDr9jdz9rbzvA3n3a6idRc0ua5laFH4dEh9/UkiaXgwc3ZeIeRUagqSXNQ5syZLOFnyEqxZH48A//ryTHzl1ZtrX3ceDgAI890x33OMFJbPjpzACbHht8wGzYrK1G0jS1vr+X1qTX+aMuseG3tmXo7R2MXVuN5Dk89qaWDMfygTDy0SU2/OaWI1/Xa6uRHOnMAJsff/+JescbWV59LRvjROFJZPjuTk2NcfWqCQB874oGBj82QJJgW3sv1145kc+cMpZJDRXcdMMUXtzeF/dYQSn4o7dKiZmx5vop3H734CeAzZ83lj8/rz7mqaRQixfVsHhRDQ+0pkmljL/7+vi4RwpOIrf4IvLRKHyRACl8kQApfJEAKXyRACl8kQAp/BK2ZVsPV1/3btxjyAjdeNO+kj37MJG/x3/PXy2v54ufr6VhfHk+f+3a089zL/aSzTqVleV3gNI9t06jP1e+h+pua++jckxp3m+JDr+2toLa2vKMHuD8ZfWcv6x8D0yaPCkV9wiR+umPp8Y9wlGVbzUiclQKXyRACl8kQApfJEAKXyRACl8kQApfJEAKXyRACl8kQApfJEAKXyRACl8kQApfJEAKXyRACl8kQApfJEAKXyRACl8kQApfJEAKXyRACl8kQApfJEAKXyRAecM3s1vNrNPM2kdjIBGJXiFb/J8AyyKeQ0RGUd7w3b0N2DsKs4jIKCnaJbTMbCWwEmDKlMm0tV9drJsuKV3dU2lrXx33GJHR+iXdVQUtVbTw3X0dsA5gzuxZfua8NcW66ZLS1r6acl030PqFQu/qiwRI4YsEqJBf590BPAnMMbMOM1sR/VgiEqW8r/Hd/cLRGERERo929UUCpPBFAqTwRQKk8EUCpPBFApT48N/cmY17BJHESXT47s4V174b9xgiiZPo8F9+tY//ezDNWx3a6osci0SH37QxA0BzaybmSUSSJdHhN7ekAbh/YzrmSUSSJbHhv/v7HE9u7QFg8+MZutIDMU8kkhyJDf+Bh9MMDLXe1wetbdrdFylUYsNvakljNvi12eD3IlKYRIbv7px9Ri0/+rcpANz8w+P47PzqmKcSSY5Ehm9mXPKt8dTXDm7yJ4yr4O+/OT7mqUSSI5Hhi8hHo/BFAqTwRQKk8EUCpPBj8sZbWW5ef4BczuMepejcned/1cvP7j0U9yiRyGadR57IJPrYkaJdUEOGl8s5T23toaklTdPGDC+/2sdlK8eTSlncoxVFd/cAmx7vprklTXNrho7d/fz0x1PjHqto9u3P8eCmDE0taTZsznDg4AAvbJ4Z91gjlujwa2oqmDY1RfXY0oynr8+5d0Oapo1pHnw4ze/3HXlY8brbD3LbncNvFX9+y/EsOaM2yjFHbP+BHD9v6qKpJcPDbRky3UfuvXz7mk4u/affDXsbO546gYkNqSjHHLG3OrLcfX8XTRvTPP5sD7nckX//+a/uGvbnZ00fw7ZNsyKccOQSHf7XGuv5WmN93GMcVVWVsfDUsbzT2c+ezn4efbKb7GFnEJ80q5KpU4Z/0I+rL91XY+PHVbDw1Gre6czxTmc/zzzfe8Tff3pOFbU1w8+fKs3mAfj48WNYeGo1ezpz7Pldjldfe//OS6XgM6eM/cPRox/muDz3bZwSHX4SnDSrklUrJrBqxQQOHhpg46MZmlvSPNCa5tN/UsUdNx0f94gjZmbMnzeW+fPG8t3LJ7Kns5/m1gxNG9O0tmX4y+X1XHlJQ9xjjtiYMcZZp9dw1uk13HDtZF59rY/mljRNLRl++XQ33728gbMXl+beWD4KfxSN+1jFH/ZScjnn2W295HJeNq/zjz9uDCsuGseKi8bR0zPAr37dF/dIRTX75Cpmn1zF5Rc3sG9/jo63++MeacQUfkxSKeO0BeV7fkF1dUVZnz/RMCFFw4TS3ZXPp3RfQIpIZBS+SIAUvkiAFL5IgBS+SIAUvkiAFL5IgBS+SIAUvkiAFL5IgBS+SIAUvkiAFL5IgBS+SIAUvkiAFL5IgBS+SIAUvkiAFL5IgBS+SIAUvkiACgrfzJaZ2StmtsPMrol6KBGJVt7wzSwFrAXOA+YCF5rZ3KgHE5HoFLLFXwTscPfX3b0PuBM4P9qxRCRKhVxQYzqw87DvO4A/++BCZrYSWDn0bW9qGu0ffbxSdOlk4N24p4iO1i/h5hSyUCHhf9j1nf7oou7uvg5YB2BmW9x9YSEDJE05rxto/ZLOzLYUslwhu/odwOEXAp8B7B7JUCJSGgoJ/1ngk2Z2kplVARcA90U7lohEKe+uvrv3m9kqYAOQAm5195fy/Ni6YgxXosp53UDrl3QFrZ+5/9HLdREpczpyTyRACl8kQApfJEAKXyRACl8kQApfJEAKXyRA/w9xvy3WLtAcEwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def print_policy(policy_matrix2, V=np.array([]),lstates=[]):\n",
    "    policy_matrix = policy_matrix2[::-1]\n",
    "    shape = policy_matrix.shape\n",
    "    U=np.zeros(policy_matrix.shape)\n",
    "    R=np.zeros(policy_matrix.shape)\n",
    "    for row in range(shape[0]):\n",
    "        for col in range(shape[1]):\n",
    "            if(policy_matrix[row,col] == -1): \n",
    "                U[row,col]=0    \n",
    "                R[row,col]=0  \n",
    "            elif(policy_matrix[row,col] == 0): \n",
    "                U[row,col]=0    \n",
    "                R[row,col]=1 \n",
    "                #policy_string += \" ^  \"\n",
    "            elif(policy_matrix[row,col] == 1): \n",
    "                U[row,col]=1   \n",
    "                R[row,col]=0 \n",
    "                #policy_string += \" >  \"\n",
    "            elif(policy_matrix[row,col] == 2): \n",
    "                U[row,col]=0    \n",
    "                R[row,col]=-1 \n",
    "                #policy_string += \" v  \"           \n",
    "            elif(policy_matrix[row,col] == 3):\n",
    "                U[row,col]=-1   \n",
    "                R[row,col]=0                 \n",
    "                #policy_string += \" <  \"\n",
    "            elif(np.isnan(policy_matrix[row,col])): \n",
    "                U[row,col]=0    \n",
    "                R[row,col]=0  \n",
    "                #policy_string += \" #  \"\n",
    "    plt.rcParams['figure.figsize'] = (4,3)\n",
    "    if V.size==0:\n",
    "        V=np.ones(policy_matrix.shape)\n",
    "        for x in lstates:\n",
    "            V[x[0],x[1]]=0.7\n",
    "        plt.matshow(V,extent=[0, 4, 0, 3],vmin=0, vmax=1)\n",
    "    else:\n",
    "        plt.matshow(V,extent=[0, 4, 0, 3])\n",
    "    plt.grid()\n",
    "    X, Y = np.meshgrid(np.arange(0.5, 4.5, 1), np.arange(0.5, 3.5, 1))\n",
    "    Q = plt.quiver(X, Y,U,R)\n",
    "\n",
    "    plt.show()\n",
    "\n",
    "policy_matrix = np.array([[1,      1,  1,  -1],\n",
    "                          [0, np.NaN,  0,  -1],\n",
    "                          [0,      3,  3,   3]])\n",
    "\n",
    "print_policy(policy_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "transition_matrix = np.array([[0.8, 0.1, 0.0, 0.1],\n",
    "                              [0.1, 0.8, 0.1, 0.0],\n",
    "                              [0.0, 0.1, 0.8, 0.1],\n",
    "                              [0.1, 0.0, 0.1, 0.8]])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "gamma = 0.999\n",
    "\n",
    "def execute_action(s1,a):\n",
    "    a=np.random.choice([0,1,2,3],p=transition_matrix[int(a)])    \n",
    "    row = s1[0]\n",
    "    col = s1[1]\n",
    "    if(a == 0): \n",
    "        new_col = col + 0\n",
    "        new_row =row - 1\n",
    "    elif(a == 1): \n",
    "        new_col =col + 1\n",
    "        new_row =row + 0\n",
    "    elif(a == 2): \n",
    "        new_col =col + 0\n",
    "        new_row =row + 1      \n",
    "    elif(a == 3):\n",
    "        new_col =col - 1\n",
    "        new_row =row + 0              \n",
    "    new_col = np.clip(new_col, 0, 3)\n",
    "    new_row = np.clip(new_row, 0, 2)\n",
    "    if state_matrix[new_row,new_col]==-1: \n",
    "        new_col = col\n",
    "        new_row = row\n",
    "    return (new_row, new_col), reward_matrix[new_row,new_col],a\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([(2, 2), (2, 1), (2, 1), (2, 0), (1, 0), (0, 0), (0, 1), (0, 2)],\n",
       " [-0.04, -0.04, -0.04, -0.04, -0.04, -0.04, -0.04, 1.0],\n",
       " [3, 0, 3, 0, 0, 1, 1, 1])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def gen_trial():\n",
    "    lS=[]\n",
    "    lr=[]\n",
    "    la=[]\n",
    "    valid_start_states = [(r,c) for c in range(4) for r in range(3) if state_matrix[r][c]==0]\n",
    "    state = valid_start_states[np.random.choice(len(valid_start_states))]\n",
    "    #print_policy(policy_matrix,lstates=[state])\n",
    "    while state_matrix[state]!=1:\n",
    "        lS.append(state)\n",
    "        state,r,a=execute_action(state,policy_matrix[state])\n",
    "        lr.append(r)\n",
    "        la.append(a)\n",
    "        #print_policy(policy_matrix,lstates=[state])\n",
    "    return lS,lr,la\n",
    "gen_trial()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sum_returns(lr,i):\n",
    "    s=0\n",
    "    g=1\n",
    "    for r in range(i,len(lr)):\n",
    "        s = s+ lr[r]*g\n",
    "        g = g*gamma\n",
    "    return s\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQwAAADTCAYAAAB0ky3dAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAH3lJREFUeJzt3XmcHGW97/HPr3uWnj2TzGQySTCTBAybYZkgRFAT5CCboKgsinoUb5SjR44oQrj4Qj3Hl3hAruLCcgXlIgeUTSEICDIDAgHJQgIhhIQsJGSZTNbZp2f6d/+omqETMt3PZKaqesjv/XrVa7p7qqu/qaR/qXrqeZ4SVcUYY1zEog5gjBk5rGAYY5xZwTDGOLOCYYxxZgXDGOPMCoYxxpkVDGOMMysYxhhnVjCMMc7yog6QrqqqSuvq6rKu19bWRklJSfCB9lOu5wPLOBxyPR+4Z1y4cGGzqlZnXVFVc2apr69XFw0NDU7rRSXX86laxuGQ6/lU3TMCC9ThO2qnJMYYZ1YwjDHOrGAYY5xZwTDGOBtRBaO7J8Xl9y7h5aaeqKMYc0AaUQVDUe5duIH1LamooxhzQAq8YIhIXEQWi8i8oW4rJgKAzRFmTDTCOMK4FFg+HBuK9xUMqxjGRCLQgiEiE4Ezgd8Oz/a8nykrGMZEIuiu4T8HvgeUDbSCiMwB5gDU1NTQ2NiYcYMCdHV3Z10vSq2trTmdDyzjcMj1fBBARpfuoPuzAGcBv/EfzwLmZXuPS9fwqXMf0W/c8rhTd9eovJe6DEcp1zPmej7VkdU1/ETgbBFZC9wDnCwifxjqRmMxsVMSYyISWMFQ1bmqOlFV64ALgKdU9aKhbjcm1oZhTFRGVD8M8K6UqF1YNSYSocyHoaqNQONwbCsmdkpiTFSyHmGIyKUiUi6e20RkkYicGka4fYnFxPphGBMRl1OSr6jqbuBUoBr4MnBtoKkyiAlYx3BjouFSMPzuUpwB/E5Vl6S9Frq4HWEYExmXgrFQRP6GVzAeF5EyIvxPXqwNw5jIuDR6XgwcDaxW1XYRGYN3WhIJ7yqJMSYKLkcYChwOfMt/XgIkAkuUhfXDMCY6LgXjN8BM4EL/eQvw68ASZSFibRjGRMXllOR4VT1WRBYDqOoOESkIONeA4jEhZSclxkTC5QgjKSJx/HlrRKSaCBs9Y2LzYRgTFZeCcSPwIDBWRH4MPAv8JNBUGdjgM2Oik/WURFXvEpGFwMfw+l98UlWHZQat/WFdw42JTtaCISJ3quoXgNf38Vro7LKqMdFxOSU5Iv2J355RH0yc7MQuqxoTmQELhojMFZEWYLqI7BaRFv95E/CX0BLuxbqGGxOdAQuGqv5EVcuA61S1XFXL/GWMqs4NMeMeYiI2+MyYiLg0es4VkUrgENJ6eKrqM0EGG4hdJTEmOi6Nnl/Fu7fIROBl4ARgPnBysNH2LSbQY+ckxkTCpdHzUuA4YJ2qzgaOAbYGmioDu0piTHRcCkanqnYCiEihqr4OTAs21sCsH4Yx0XEZS7JBREYBfwaeEJEdwMZgYw0sFrOu4cZExaXR81P+wx+ISANQATwaaKoM7AjDmOi4TAJ8Z99jVX1aVR8Cbg80VQYxa8MwJjIjrqenXVY1JjqD6em5Oxd6etrwdmOiM5ienuW50NMzbj09jYmMyynJPBEpARCRi0TkBhGZFHCuAdms4cZEx6Vg3AS0i8hRwPeAdcD/CzRVBvEYqJ2TGBMJl4LRo9439BzgF6r6C6As2FgDs8FnxkTHpeNWi4jMBS4CPuJfJcnP9iYRSQDPAIX+59ynqtcMJSzYvVWNiZLLEcb5QBdwsapuBiYA1zm8rws4WVWPwrsR0mkicsJ+J/VZxy1jouPS03MzcEPa87dwaMPwT2Na/af5/jLkr3pchmEjxpj9IkE2IPqnLwuBg4Ffq+oV+1hnDjAHoKampv6ee+7JuM1bl3bx/MYezp9WwOmTs54ZRaK1tZXS0tKoY2RkGYcu1/OBe8bZs2cvVNUZWVdU1cAXYBTQAByZab36+nrN5o7n1+ikK+bppCvmZV03Kg0NDVFHyMoyDl2u51N1zwgsUIfvsksbxpCp6k6gEThtqNv64sw6plTEqCotHHIuY8zgDNiGISKvsO/mAsFropieacP+HdKSqrpTRIqAU4CfDiVsn0nlMV7ZYS0ZxoQtU6PnWUPcdi1wh9+OEQP+pKrzhrhNwG41YExUBiwYqrpuKBtW1aV40/kNOwFS1hnDmNBlOiVpIfMpSXlgqbKICfTaIYYxoct0hBFZ9+9sYtgQd2Oi4NI1HAARGcue9yV5K5BETlnslMSYKLhM0Xe2iKwE1gBPA2uJcE5PP5MVDGMi4NIP4z/xbl70hqpOBj4GPBdoqiy8Rs8oExhzYHIpGElV3QbERCSmqg14g8kiExNIWcUwJnQubRg7RaQUb6j6XSLSBPQEGysza8MwJhouRxjnAO3At4HHgDeBTwQZKpsYdkpiTBRchre3+Q9TwB3BxnEj4v1UVaTviTEmcKEMPhtufSXCOm8ZE66RWTD8imH1wphwZSwYIhIXkT+EFcZVrL9gWMUwJkwZC4aq9gLVIlIQUh4nfaGtXhgTLpfLqmuB50TkIaCvARRVvWHAdwSsr6HTjjCMCZdLwdjoLzEivB9Juv5GTysYxoTK5bLqDwFEpMx7qq1Z3hK4vjYMtTsaGRMql8FnR4rIYuBVYJmILBSRI4KPlimT99NOSYwJl8tl1VuBy1R1kqpOAr4D/N9gY2XWd0piBcOYcLkUjBJ/wBkAqtoIlASWyEGen/r5N7dFGcOYA45LwVgtIt8XkTp/uRpvbozIHF/rNb2saoq8OcWYA4pLwfgKUA084C9VwJeDDJVNUZ53UmInJMaEK9MkwHeq6heAL6rqt0LM5ES8qYijjmHMASXTEUa9iEwCviIilSIyOn0JK+BAbNYtY8KXqR/GzXjzX0zBu6Fy+jhy9V+PjIigdlJiTKgGPMJQ1RtV9TDgdlWdoqqT05ZIiwX03Rwl6hTGHFiyNnqq6iVhBBksEWv0NCZsI3I+DABB7AjDmJCN2IKBYG0YxoTMZSxJiYjE/Mfv929slO/wvoNEpEFElovIMhG5dDgC928f7JzEmJC5HGE8AyREZALwd7xOW793eF8P8B2/4fQE4Bsicvj+Bt2btWEYEz6XgiGq2g6cC/xSVT8FZP3iq+omVV3kP24BlgMThhJ2j1AIao0YxoTKZQIdEZGZwOeBiwfxvvQN1AHHAC/u43dzgDkANTU1NDY2Zt1ea2srqZTw1vr1NDY2DSZKKFpbW53+HFGyjEOX6/kggIyqmnEBPgI8BFzhP58C3JjtfWnvL8Xr+HVutnXr6+vVRUNDgx7+/Uf1Rw8vc1o/bA0NDVFHyMoyDl2u51N1zwgsUIfvc8YjBRGJA59Q1bPTCsxqwGlsid84ej9wl6o+MNhilklM7LKqMWFzmTW8fn82LN5MvbcByzWICYPt/qrGhM6lLWKxP2P4vew5a3i2I4YTgS8Ar4jIy/5rV6nqX/cr6V7sBonGhM+lYIwGtgEnp72meHNjDEhVnyXA77WIXSUxJmwus4ZHOlnOQKwfhjHhc+npOVFEHhSRJhHZIiL3i8jEMMJlzIWNVjUmbC4dt36Hd1l1PF7Hq4f91yJl82EYEz6XglGtqr9T1R5/+T3eHJ+RsiMMY8LnUjCaReQi/07ucRG5CK8RNFLWhmFM+FxnDT8P2AxsAj7jvxYx67hlTNhcenp+Or2nZ64QG99uTOhcenqeE1KWQYmJtWEYEzaXjlvPicivgD+yZ0/PRYGlciCIdQ03JmQuBeND/s8fpb2m7NnzM3RiRxjGhC5bG0YMuElV/xRSHmeCtWAYE7ZsbRgp4JshZRkUseHtxoTO5bLqEyLyXX9S35y5VSLYrOHGhM2lDaOvz8U30l7LgVslYuckxoTMZbTq5DCCDJb19DQmfC6jVYtF5GoRudV/foiInBV8tCy5bNZwY0LnOlq1m3cur24A/iuwRI7sCMOY8LkUjKmq+t9AEkBVO8iBGfJstKox4XMpGN0iUoT/H7qITAW6Ak3lwJsPwxgTJperJNcAjwEHichdeJP7/muQoVx4PT2tZBgTJperJE+IyCK8+6MKcKmqNgeeLAs7JTEmfE63PFTVbcAjAWcZFJuiz5jwubRh5CQ7wjAmfAMWDBHJyQ5bfWy0qjHhy3SEcR+AiPw9pCyDItgpiTFhy9SGERORa4D3i8hle/8ykPulDoKidCZTUUYw5oCT6QjjAqATr6iU7WOJ1BtbWnn6ja3sbO+OOooxB4wBjzBUdQXwUxFZqqqPhphpUG5/dg2XnTot6hjGHBBcrpI8LyI3iMgCf/mZiFRke5OI3O7fXvHVYcg5oBufWkXdlY9Qd+UjrNvWlv0Nxpj95lIwbgda8O5Nch6wG7dbJf4eOG2/k+2Hj17XyB3Prw3zI405oLgOPrtGVVf7yw9xmDxHVZ8Btg854QCuPvMwDq8t5/wZB+3x+jUPLQvqI4054Em28RgiMh+4XFWf9Z+fCFyvqjOzblykDpinqkdmWGcOMAegpqam/p577skaurW1ldLS0v7nX36srf8C6+iEcMOs4qzbCNLe+XKRZRy6XM8H7hlnz569UFVnZF1RVTMuwFHAEmCtvywGpmd7n//eOuBVl3VVlfr6enXR0NCwx/OnVzTppCvm9S+dyR5duWW3/rphpa7YvNtpm8Np73y5yDIOXa7nU3XPCCxQh++oy+CzJcBRIlLuP9+dtQqFrLqscI/n065+jER+jM5kihWbW/jFBcdElMyY9xbnsSSqujsXiwXA6JICAD5YN5qqUq949HXqauvqiSyXMe81TqNV94eI3A3MAqpEZANwjareFsRn1ZQnePKyjzKuIkFhXoyHl2wkJsItz6ymvbs3iI805oAUWMFQ1QuD2va+HDz2nYadc4+dCMD9izbQ0mlHGMYMl6wFQ0TiwJl4DZj962vEY0lcFOXH+cfKZm742wo+ecwEplTndou2MbnO5QjjYbwxJa8AI2q015nTa/nba1u48alV3PjUKn5y7gfY3tbtN4QejUjkcxkbM6K4FIyJqjo98CQBOOfoCRw5oYJf/n0lf355I3MfeKX/d+3dPdx0UT358RE7h5AxoXP5tjwqIqcGniQgU6tL+dl5R/PQN09kYmURRflxxpYV8uTyJi75w8Ko4xkzorgcYbwAPCgiMbx7k3iz46mWB5psGMVjwvSJo/jH92aTUu953ZWPMP/NbXT19FKYF486ojEjgssRxs+AmUCxqparatlIKhbpRIR4zGu3+OpJk2nr7uW8m+dHnMqYkcOlYKzE6979npoP75JZU0nkx1iyYRebdnVEHceYEcGlYGwCGkVkrohc1rcEHSxoY0oL+eWFxwLQ3GKzdhnjwqUNY42/FPjLe0Yi36uXnT3WG9QYFy6Dz34YRpAoJPK9xs7OpBUMY1y49PRsgHfP56+qJweSKESJvL6CMaL6oxkTGZdTku+mPU4AnwbeEwM0+k5JXli9jZMOrqKowC6vGpOJyynJ3r2bnhORpwPKE6pKf1j8bc+u4c4X1nFYbTlHT6zg67OmMq48YV3HjdmLyynJ6LSnMaAeGBdYohBVlRZy39dn8uyqZppbu1iyfhd3zF/HHfPXMb4iQcPls6xTlzFpXE5JFuK1YQjeqcga4OIgQ4VpRt1oZtS9UxPrrvRuUr9xVyfb27opzIvTm9J3zeplzIHI5ZQkp2/KPNxu+vyxPLF8Cw8sepsTr32KlELdmGIaL58ddTRjIpe145aIfFZEyvzHV4vIAyJybPDRonH6B2o5+6jxAKT8a0Nrt7Wzd0fXZO++r6w8u7KZ373axan/52kee3Xzu95nzEjmckryfVW9V0ROAj4OXA/cBBwfaLIITRtXxqQxxfzg7CNYtaWVH/91Obc8s5oxJQXct3ADJYV5PPV6E5XF+Xz8iHF87aNT+dVTq3j6jSaaW/t6jbby9T8spLYiQXt3L/GY8InptVx15mHWLmJGLJeC0der6UzgJlX9i4j8ILhI0autKOJp/xSkr6/GtY++/q71drQnueel9dzz0noAPjh5NF84oYpR7W9RVPt+Hl7qzS2aHxeeXN7EHfPXUV83uv8IxpiRxqVgvC0itwCn4N2cuZBBzDY+0s2cOoZff+5Ybn76TWZOHQPAk69t4T8/eSTvrynjU795jukTK7jwg+/jw4dUA9DY+DazjjuI8457565sa5vbmHV9I9+6ezGnHzmO/HiMzmQvuzuTtHT2UJgXo62rl7e2t3PyoWP7R9Uak0tcCsZ5ePdIvV5Vd4pILXB5sLFyy5nTazlzem3/86vOOKz/8bNXuHV4rasq4cOHVPGPlc2c9vNnaNrdRcsAt0C4/OPT+Mbsg4cW2pgAuFwlaQceSHu+CW8Eqxmkn356Olfcv5REfpwTD66ipjxBeVE+hXkx1ja3MbmqhMvvW8qzK5spKYhTUpjHZ+onWgcykzMCu82Aebfxo4q48+LMbcUPLn6b59/cxvzV2wA4fHw5R4yvCCOeMVlZwcgxN154DBt2dLBxZwf/dtcirvnLMu675EOD2samXR1c9cArbNndhQJTqkuYPW0s5xw93iY9NkNiBSPHVJUWUlVayGG1ZYwqzmfBuh08vGQjB48tZVpNGbGY0Jns5cU123lpzXa6e1MU5ceZNq6M940uZumGXVz1oDc7+rjyBOVFeTyydBOPLN3EVQ++wtTqUiqK8tixo4NHti4hHhPau3upLiukpCBO7agizpxeS3kiP+I9safOZC+FebEhn54t3bCTN7a08oEJFUypLiEuwu7OJIIgMUillJbOHooK4pQl8va4BN7Tm+Ll9TsRERL5MTa1ptje1t1/q84DgRWMHFWYF+dXFx7LRbe9yL/fvRiACaOKOOuoWppburl/0YaM7/9s/UR+dM6RFBXE6eju5eElG5m/ehubdnXQ06vs6FLuXbiB/LiQ7N2zc9m9C9bz+eMnMbGyiOKCPKaOLaG4IPs/lVRKiflXd3a2d9OTUsoT+RTkDXxU89rG3dy7cD2FeXHy40JxQR47O7qZOKqItzf1wIomigvyOO+W+cTEmwW+MD/GOUdN4IzptcRFqCkv7C8kvSklJtDa1cNzq7bRm1KeXL6F9dvb2d7Wzermtv7PFoFs/eoqi/OprSiiLJHH65tb2NWR3OP3c599gs8d/z52dyRRhcqSfCqLCyguyKOyOJ9YTOhK9pLsVWbUVbKrI8na5jaqSgsZV5Fg/KgiKory++dm2drSRUlh/F37e/32dppaOr39rN6fM6XKlt2dLN/UQrI3RVyEeFzIj8WIx4TigjhlrcM7dYPkUk/EGTNm6IIFC7Ku19jYyKxZs4IPtJ+GK5+qsn57B1tbu1i0bgePLdvM4rd29PdAvfL0Qzlrei1VpYW8tHY729u66e5JMaa0gJMPrcmacfpxH6I8kUc8JqjC7s4k//XIcu5b+O5ilBcTCvNiJPLjVJcVMqo4n47uXtbv6KCyOJ/Wrh66elLMnDKGt7a3s2zjO/ftrizOZ1xFEVWlBexo76als4dJY0oozo/z2LLNABTkxUj2pjJ+gY+aWEFNeYKGFU17FLkjxpdz0iFVrG1u47lV20j2pujqefcX5YQpoxlXnuDcYyeycWcHjy/bTMOKrUytLuFzx09CVRERygrz6Ej20tKZ5O2dnTTt7qS1q4f27l5KCuN87aNT6Ur28syCV7l3ZbI/y/iKBO3JXnb5xWMwxlckSBTEWb3VK2hTq0uoLiskkR+nM9nLC6u3Z3x/WWEeKVV6Ut7S6/8jmTO9kKs+d0rWzxeRhao6I9t6gR5hiMhpwC+AOPBbVb02yM97rxER3jemmPeNKaZ+UiX/6yNTaG7t4qnlTUwdW0L9pHcGzfX1ARmM9ENpERhVXMD1nz2KK08/lHXb2ulM9nr/K29to6unl66eFO3dvaze2sqa5jYK8+IUxGOUJfJp6eyhp1dZsbmFddvbKcyLcdUZh7G7I8nGXZ1sbemkqaWLVAqOnFDB2uY23mxPcui4Mr5y0mTOm3EQqZTS1t1DIj/OjrZunnzmeaZ94GjWb+8gkR/n40fUICJs3NnB3f98i7JEHtvbkjy+bDO3/WMNPSll0phiTj50LAXxGJUlBZx0cBUtnT0cUlNKVemeAwjPP+4g3trezoRRReTtR9tOonkFP/7SKf0TMPXNp6KqdCR72dGeJJVSEvlxtrd1s2TDTrqSvRw5oYKCvBgbd3bS1NLJjrZuVjW1sr09SU1ZguPqKlm2cTctnT3saOumM5li1rRqvnDCJPLjMUQgLtI/C/6RE8rfdUSiquzqSPLi888N+s+VSZB3b48Dvwb+BdgAvCQiD6nqa0F95oGgqrRwjw5hQX3G3l+uwejxx9kM9ksYiwllftvJ2PIE40tj1E8aTf2kPdcbP6qI75w6rf/5lacf2t/2UFHs3vYiIkwaUzKojPvaxt4TL4l4p1bpX+LqskKmjSvbY70gr36JCKOKCyjMG95L8kE2mX8QWKWqq1W1G7gHOCfAzzM5Ii8e26//sYciFpNBFQuzf4L8W50ArE97vsF/zRgzQgXZhrGvY6F3NQWJyBxgDkBNTQ2NjY1ZN9za2uq0XlRyPR9YxuGQ6/kggIyqGsiCd3vFx9OezwXmZnpPfX29umhoaHBaLyq5nk/VMg6HXM+n6p4RWKAO3+sgT0leAg4RkckiUgBcADwU4OcZYwIW2CmJqvaIyDeBx/Euq96uqsuC+jxjTPByquOWiGwF1jmsWgU0BxxnKHI9H1jG4ZDr+cA94yRVzdqZJ6cKhisRWaAOvdKikuv5wDIOh1zPB8Of0YYuGmOcWcEwxjgbqQXj1qgDZJHr+cAyDodczwfDnHFEtmEYY6IxUo8wjDERsIJhjHE2ogqGiJwmIitEZJWIXBlhjoNEpEFElovIMhG51H99tIg8ISIr/Z+V/usiIjf6uZeGdatJEYmLyGIRmec/nywiL/r5/uj3wEVECv3nq/zf14WUb5SI3Ccir/v7cmYu7UMR+bb/9/uqiNwtIomo96GI3C4iTSLyatprg95nIvIlf/2VIvIl5wAu/cdzYcHrLfomMAUoAJYAh0eUpRY41n9cBrwBHA78N3Cl//qVwE/9x2cAj+INyDsBeDGknJcB/wPM85//CbjAf3wzcIn/+N+Am/3HFwB/DCnfHcBX/ccFwKhc2Yd4I6vXAEVp++5fo96HwEeAY4FX014b1D4DRgOr/Z+V/uNKp88P4x/GMO2oQQ9mCzHbX/AmCloB1Pqv1QIr/Me3ABemrd+/XoCZJgJ/B04G5vn/aJqBvL33J173/Zn+4zx/PQk4X7n/hZS9Xs+Jfcg70zOM9vfJPLx7C0e+D4G6vQrGoPYZcCFwS9rre6yXaRlJpyQ5Ob+Gf+h5DPAiUKPejZ7wf471V4si+8+B7wF9k1uOAXaqat/t1tIz9Ofzf7/LXz9IU4CtwO/806bfikgJObIPVfVtvBuPv4V3465dwEJyax/2Gew+2+99OZIKhtP8GmESkVLgfuA/VHV3plX38Vpg2UXkLKBJVRc6Zohi3+bhHVrfpKrHAG14h9MDCXsfVuLNEDcZGA+UAKdnyJBz/z4ZONN+Zx1JBWMDkD6Z5URgY0RZEJF8vGJxl6r23Upyi3/vWfyfTf7rYWc/EThbRNbiTY14Mt4RxygR6RuhnJ6hP5//+wog8zTVQ7cB2KCqL/rP78MrILmyD08B1qjqVlVN4t0u9EPk1j7sM9h9tt/7ciQVjJyZX0NEBLgNWK6qN6T96iGgr8X5S3htG32vf9FvtT4B2NV3CBkEVZ2rqhNVtQ5vPz2lqp8HGoDPDJCvL/dn/PUD/d9RVTcD60WkbzbfjwGvkSP7EO9U5AQRKfb/vvvy5cw+TDPYffY4cKqIVPpHUqf6r2UXZMNWAI09Z+BdkXgT+N8R5jgJ7xBuKfCyv5yBd876d2Cl/3O0v77gzaD+JvAKMCPErLN45yrJFOCfwCrgXqDQfz3hP1/l/35KSNmOBhb4+/HPeC32ObMPgR8CrwOvAncChVHvQ+BuvDaVJN6RwsX7s8+Ar/hZVwFfdv186xpujHE2kk5JjDERs4JhjHFmBcMY48wKhjHGmRUMY4wzKxgHEBF53v9ZJyKfG+ZtX7WvzzLvLXZZ9QAkIrOA76rqWYN4T1xVezP8vlVVS4cjn8lddoRxABGRVv/htcCHReRlf86HuIhcJyIv+fMmfM1ff5Z48378D17HH0TkzyKy0J8nYo7/2rVAkb+9u9I/y+9leJ0/p8QrInJ+2rYb5Z35MO7ye1QiIteKyGt+luvD3Ecmi7B6HNoS/QK0+j9n4ff+9J/PAa72Hxfi9b6c7K/XBkxOW7evF2ERXg/IMenb3sdnfRp4Am8+kxq8Lte1/rZ34Y1jiAHz8XrQjsYbht139Dsq6v1myzuLHWEY8MYSfFFEXsYbpj8GOMT/3T9VdU3aut8SkSXAC3gDmA4hs5OAu1W1V1W3AE8Dx6Vte4OqpvC619cBu4FO4Lcici7QPuQ/nRk2VjAMeGMO/l1Vj/aXyar6N/93bf0reW0fp+BNFHMUsBhvDEW2bQ+kK+1xL97END3AB/FGAn8SeGxQfxITKCsYB6YWvKkF+zwOXOIP2UdE3u9PZrO3CmCHqraLyKF40771Sfa9fy/PAOf77STVeFPM/XOgYP4cIxWq+lfgP/AGqJkcEdjd201OWwr0+KcWvwd+gXc6sMhveNyK97/73h4Dvi4iS/HaGV5I+92twFIRWaTeUPo+D+JNZbcEb4Tv91R1s19w9qUM+IuIJPCOTr69f39EEwS7rGqMcWanJMYYZ1YwjDHOrGAYY5xZwTDGOLOCYYxxZgXDGOPMCoYxxtn/B8c0kdZEi64iAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "array([[0.84920848, 0.9076811 , 0.95399825, 0.        ],\n",
       "       [0.80315709, 0.        , 0.66316725, 0.        ],\n",
       "       [0.75474999, 0.7123851 , 0.66786456, 0.42979731]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# From previous notebooks\n",
    "TrueV =[[0.84881226, 0.90630541, 0.95748948, 0.],\n",
    "        [0.797764, 0., 0.69906187, 0.],\n",
    "        [0.7404234,  0.6895106,  0.64536512, 0.42192624]]\n",
    "\n",
    "# Exact Monte-Carlo First Visit\n",
    "lerror=[]\n",
    "ntrial = 1000\n",
    "V = np.zeros((3,4))\n",
    "N = np.zeros((3,4))\n",
    "for i in range(ntrial):\n",
    "    lS,lr,la = gen_trial()\n",
    "    visited = []\n",
    "    for i,s in enumerate(lS):\n",
    "        if s not in visited:\n",
    "            visited.append(s)\n",
    "            N[s] = N[s]+1\n",
    "            alpha=1/N[s]\n",
    "            V[s]=V[s]+alpha*(sum_returns(lr,i)-V[s])\n",
    "    lerror.append(np.sum(np.abs(TrueV - V)))\n",
    "\n",
    "plt.plot(lerror)\n",
    "plt.grid()\n",
    "plt.ylabel('sum of errors for all states')\n",
    "plt.xlabel('iterations')\n",
    "plt.show()\n",
    "V"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARYAAADTCAYAAABEHzz8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXl4XVW5/z/vOZmTNkmT0JZ0orQUaBk6UFoovS0zKKA4gTKKF/WqIIpir3oR1EcU5F75oSDIIFygDIJAmeRqQ4FCC50nOk8pHdM2YzO/vz/2PulJmuTsJOfss5u+n+fZz9l7nbXX/mY3ebvWu971LlFVDMMw4kko2QIMw+h9mGExDCPumGExDCPumGExDCPumGExDCPumGExDCPumGExDCPumGExDCPumGExDCPupCRbQFcpLCzUYcOGxaxXXV1NdnZ24gX1ANPYc4KuD4Kv0au+BQsW7FHVIk+NquphdYwfP169MHv2bE/1kolp7DlB16cafI1e9QEfq8e/UxsKGYYRd8ywGIYRd8ywGIYRd8ywGIYRd3qtYXl9Yz0lq3clW4ZhHJH0SsOiqjy3uoHrHvso2VIM44ikVxqW5qikeA1NzckTYhhHKAkzLCKSISLzRWSJiKwQkTvaqXOdiOwWkcXu8Y14PFuj0m3urqyLR5OGYXSBREbe1gFnq2qViKQC74nIG6r6YZt6z6rqd+P5YO3g3DAMf0iYYXEj9arcy1T38OXvPDo/eHTvxTAMf0joWiERCQMLgBHAH1V1XjvVviAiU4E1wC2qurWddm4EbgTo378/JSUlnT63IcrJ8sEHH1KUFUxXUlVVVcyfJdkEXWPQ9UHwNSZEn9fY/54cQB4wGxjTprwASHfPvwX8K1ZbXtYKHahv1KG3zdKht83SLWXVntZBJIOgryFRDb7GoOtTDb7Gw3atkKruB0qAC9uUl6lqxLv6MDA+Ps87eN5sQyHD8J1EzgoViUiee54JnAt80qbOwKjLS4FV8Xi2RrlyzK4Yhv8k0scyEPir62cJAc+p6iwRuROnS/UKcJOIXAo0AnuB6+Lx4FbO23g0aBhGl4hpWETkZuAxoBL4CzAW+Imq/qOz+1R1qVu3bfl/RZ3PAGZ0UXNMooc/NhQyDP/xMhT6uqpWAOcDRcD1wF0JVdVDWsWxmF0xDN/xYljE/bwYeExVl0SVBZLWxsQsi2H4jRfDskBE/oFjWN4SkT5AsBfgtAqQS54MwzhS8eK8vQE4FdigqjUiUoAzHAos0bNCzWZYDMN3vPRYFDgRuMm9zgYyEqYoDjS3mhUyy2IYfuPFsPwJmAxc6V5XAn9MmKI4oGpxLIaRTLwMhU5X1XEisghAVfeJSFqCdfUImxUyjOTipcfS4Aa5KTgRtQTceWsh/YaRXLwYlvuAl4CjROTXwHvAbxKqqoeoGRPDSCoxh0Kq+pSILADOwYlf+ZyqxmVNT6KwoZBhJBcvIf1PqurVRC0gjCoLJGqzQoaRVLwMhUZHX7j+lrikN0gUFsdiGMmlQ8MiIjNEpBI4WUQqRKTSvd4FvByrYY/JtNNF5FkRWSci80RkWA9+lhZaxbHYWMgwfKdDw6Kqv1HVPsDdqtpXVfu4R4G7KjkWkWTap+BE7l4oIpPa1LkB2KeqI4D/Bn7bzZ+jrfaD5/Fo0DCMLuHFeTtDRPKBkURF3KrqnBj3eUmmfRnwC/f8BeB+ERHtYTdDba2QYSQVL87bbwA3A4OAxcAk4APgbA/3xkqmXQxsBVDVRhEpx8mDu6dNO11Kpr275mCYzcKFC6ncGI4lNSkEPckyBF9j0PVB8DUmJZk2sAynp7LYvT4eZy+geCTTXgEMirpeDxR01paXZNqb91S3JNOev7EsZv1kEfQky6rB1xh0farB15isZNq1qloLjrNVVT8BRnXReLWbTBsoBQa7bacAuTgpKntEdLStDYUMw3+8GJZSNyn234G3ReRl4NNYN3lJpg28Alzrnn8RZ/uPHpuC6AYspN8w/MeL8/bz7ukvRGQ2Tq/iDQ9te0mm/QjwpIisw+mpXNGdH6IdzVHn8WjRMIyu0JXIW1T1nUgZ0GnkrXpLpl0LfKmLmmPSeu9msyyG4Te9M/LW9v8wjKTSlcjbiq5E3iaT1mkTkqfDMI5UuhJ5G4m+9Rp5mzRsKGQYycXLUGiWiGQDiMhVInKviAxNsK4eYZG3hpFcvBiWB4AaETkF+DGwGXgioap6SKu9m5OowzCOVLwYlkY3tuQy4A+q+gegT2Jl9YzmqMSZFsdiGP7jJZl2pYjMAK4CprqzQqmJldUzWvlVzK4Yhu946bF8BScFwg2qugNn4eDdCVXVQyyDnGEkFy+RtzuAe6OutxBwH0s0NhIyDP/x0mM57Ij2q1gci2H4T680LNG9lB3lB5InxDCOUHqnYYk6//nLK5KmwzCOVDr0sYjIMtqfUxGczJMnd9awiAzG8cUMwNk58SF3qjq6zjSc5QEb3aIXVfVOz+o7IHqtUFGf9J42ZxhGF+nMefvZHrbdCPxQVReKSB9ggYi8raor29R7V1V7+qxWRFvDY4uy49m0YRge6NCwqOrmnjSsqtuB7e55pYiswpmqbmtY4k50j6U50LtMG0bvRDpK2OauZO5sKNTX80Oc/YLm4OS8rYgqnwb8DSdF5afArap6iFOkTTLt8TNnzuz0eWv3NfHrebUAjMgL8bNJmV6l+kpVVRU5OTnJltEpQdcYdH0QfI1e9U2fPn2Bqk7w1KjX5LjdPYAcnEz9l7fzXV8gxz2/GFgbqz0vybTnbyzTobfN0uEzXtNL738vZv1kEfQky6rB1xh0farB15isZNoAiMhRIjIkcni8JxWnR/KUqr7YjlGrUNUq9/x1IFVECr1q6ohIJywcEpotkMUwfCemYRGRS0VkLc7MzTvAJjzkvBURwclpu0pV7+2gzgC3HiIy0dVT5ll9B0QC5FJDQpMZFsPwHS+LEH+Js0nZ/6nqWBGZDlzp4b4zcfLiLhORxW7ZfwJDAFT1QZzM/N8WkUbgAHCF2+XqEZEWUsIhMyyGkQS8GJYGVS0TkZCIhFR1tojE3GNZVd/DcfR2Vud+4H6PWj0TWXiYEhKabLGQYfiOF8OyX0RycGZ1nhKRXTgxKsHFfCyGkVS8OG8vA2qAW4A3cbZBvSSRonpKxJakhkM0mmExDN/xkjah2j1tBv6aWDnxoWUoFBYam8ywGIbf9M5FiNFDIfOxGIbv9E7D4n6mhkLU1DcRh4kmwzC6QKeGRUTCIvK/fomJFxFDIgLlBxr4U8n6JCsyjCOLTg2LqjYBRSKS5pOeuBDpoJw/egAA8zfuTaIawzjy8DLdvAl4X0ReASKOXDqKpg0CEeft2ccfxWPvb+SYQkudYBh+4sWwfOoeIQK+n1CESI9FiEw5W+4Ew/ATL9PNdwC4yZo0smgwyEQMS0iElJBNORuG33hZhDhGRBYBy4EVIrJAREYnXlr3aY5y3qaGQzSYYTEMX/Ey3fwQ8ANVHaqqQ4EfAg8nVlbPiDYjKWGxoZBh+IwXw5KtqrMjF6paAsT0horIYBGZLSKrRGSFiNzcTh0RkftEZJ2ILBWRcV1S3wEtPhZxFiJaWL9h+IsX5+0GEfk58KR7fRUHs+p3hpdk2hcBI93jdOAB97OHuEMhhJRQiMYm67EYhp946bF8HSgCXnSPQuD6WDep6nZVXeieVwKRZNrRXAY84Wa++xDIE5GBXdDfwbOdz1DI1gsZRjLobF+hJ1X1auAaVb2pJw9xk2mPBea1+aoY2Bp1XeqWbW9zf3QybUpKSjp93rIdTlaHjz/6mAPVdeysr455TzKoqqoKpK5ogq4x6Pog+BoToa+zodB4ERkKfF1EnqBN0iZV9RTO6uZy+RvwfY3K0B/5up1bDuleqOpDOE5kJkyYoNOmTev0mVVLP4XFi5g48TT+XrqMtJQQ06ZN8iLXV0pKSoj1sySboGsMuj4IvsZE6OvMsDyIk39lOE6W/WgjoG55p8RKpo3TQxkcdT0IJxivR0QHyNlQyDD8p0Mfi6rep6onAI+q6nBVPSbq8GJUYibTBl4BrnFnhyYB5epsdNYjImZERJw4FptuNgxf8RJ5++1utu0lmfbrOPsJrcPJUhfTKeyF6NXNKZap3zB8x8t0c7fwmExbge/E/9nOpzMUsshbw/CbXproKdJjEVLDYnEshuEzXtYKZYtIyD0/zt3ALDXx0rpPqx5LyBJqG4bfeOmxzAEyRKQY+CeOH+TxRIrqKW1XNzdYj8UwfMWLYRFVrQEuB/6fqn4eODGxsnpG9Opmm242DP/xZFhEZDLwNeA1tyxhTt940Hp1c4gdFbX8fdG2pOkxjCMNL4blZmAG8JKqrhCR4cDsGPckl6jVzWFnz3m+/+ziTm4wDCOedNrzEJEwcImqXhopU9UNQI/WDiWayKxQSIT6RvOvGIbfeMnSP94nLXGjObrHEu40lMYwjATgZSi0SEReEZGrReTyyJFwZT3g4HSz8L2zRwBOBK5hGP7gxbD0A8qAs3E2g78E+GwiRfWUgwFyMDA3kxunDifFei6G4Rte1grFZf2On0QHyIFjYGyXVcPwDy+Rt4NE5CUR2SUiO0XkbyIyyA9x3eWrE4fwyPlZFPVJB5whkRkWw/APL0Ohx3DSGxyNk93tVbesU0TkUdcYLe/g+2kiUi4ii93jv7oivDNCISEcEsSdag7JweGRYRiJx4thKVLVx1S10T0ex8mBG4vHgQtj1HlXVU91jzs9tNktRA7OFBmGkXi8GJY9InKViITd4yocZ26nqOocIBC7sYdEWnK0GIaReCTWH5yIDAHuBybjxLTOBW5W1c0xG3eSaM9S1THtfDcNJ21lKU46yltVdUUH7UQn0x4/c+bMWI+mqqqKnJwcAF5cW88r6xt4/MJgbQ4frTGoBF1j0PVB8DV61Td9+vQFqjrBU6Oq2uEBhIFbOqsT4/5hwPIOvusL5LjnFwNrvbQ5fvx49cLs2bNbzu/9x2odetssbW5u9nSvX0RrDCpB1xh0farB1+hVH/Cxevzb9xJ5e5knC9VFVLVC3Q3mVfV1IFVEChPxLNeHazNDhuETXlYpvy8i9wPPAtWRQnU3I+suIjIA2KmqKiITcfw9MX033SHkWhazK4bhD14MyxnuZ/SsjeJE4naIiDwDTAMKRaQUuB1IhZZE2l8Evi0ijcAB4Aq3uxV3IoFyzaqEO0/DaxhGHIi1ujkEPKCqz3W1YVW9Msb39+M4hRNOyF0nZEMhw/CHWD6WZuC7PmlJOM1mWQzDF7zEsbwtIreKyGAR6Rc5Eq4sjkR8LIZh+IMXH8vX3c/o/X88bbEaFCJ2xXoshuEPXlY3H+OHkEQSsulmw/AVL6ubs0TkZyLykHs9UkQCnY+lLeLOBFmPxTD8wevq5noOTjuXAr9KmKIE0BIgl1wZhnHE4MWwHKuqvwMaAFT1ADH2ZA4akfQJanm1DcMXvBiWehHJxP0PX0SOBeoSqirOtPhYrM9iGL7gZVboduBNYLCIPAWcCVyXSFHx5mDkbVJlGMYRg5dZobdFZCEwCedv9GZV3ZNwZXHkYOStWRbD8ANPW6WqahkHt1c97LAei2H4ixcfy2FPi/PWfCyG4QsdGhYR6VFgnIdk2iIi94nIOhFZKiLjevK8zrU4nzYSMgx/6KzH8gKAiPyzm20/TufJtC8CRrrHjcAD3XxOTCJrhSxAzjD8oTMfS0hEbgeOE5EftP1SVe/trGFVnePmvO2Iy4An3BwsH4pInogMVNXtHnR3CQvpNwx/6cywXAF8zq3TJwHPLga2Rl2XumWHGJY2ybQpKSmJ2XhVVVVLvdWlDQDM/eADCjOD41aK1hhUgq4x6Pog+BoToi9WUlzgIq8JdNu5dxgdJ9N+DZgSdf1PYHysNruTTPu5j7bo0Ntm6Zayak/3+kXQkyyrBl9j0PWpBl+j78m0XeaKyL0i8rF7/F5EcuNg00qBwVHXg3C2AYk7LbNCNhQyDF/wYlgeBSqBL7tHBR62WPXAK8A17uzQJKBcE+BfAQvpNwy/8RIgd6yqfiHq+g4RWRzrJg/JtF/H2U9oHVADXN816d6JTDc3WYScYfiCF8NyQESmqOp7ACJyJk5W/U7R2Mm0ldZZ6RJGJB/L2b9/h4nD+vHctyb78VjDOGLxYli+BTwR5VfZB1ybOEnxJyV8MMvD/E2B2E7aMHo1MX0sqrpEVU8BTgZOVtWxqro08dLix/knDuDeL58CwMijgruHrmH0FjwHdaizJWpFIsUkirSUEJePG8RFYwYkW4phHBEEJ1rMB1LCIXPgGoYPHFmGJSQ0NFt+SsNINDGdtyISBj6DE0XbUl9jrBUKIikhoanJeiyGkWi8zAq9CtQCy4DD+r/7lLDQYEMhw0g4XgzLIFU9OeFKfCAlFKKx6bC2jYZxWODFx/KGiJyfcCU+EA4J+2oaOP7nb3DPW6uZ/ckuPnPfu/zwuSXJlmYYvQovPZYPgZdEJISzt5DgBM72TaiyBNDoOm5rG5q5f/a6lvIVn1bwezfOxTCMnuPFsPwemAwsc8PwD1veWbMbgOP655CTnkJNfROD+2XxkRuN++h7GxlakMU5J/RPpkzDOOzxYljW4uRUOayNCsBvLz+Zh97dwINXjScjNQzAb15fxZw1u6lrbOLOWSsBWH7HBeSke9rAwDCMdvDy17MdKBGRN4jaAdHLdLOIXAj8AQgDf1HVu9p8fx1wN7DNLbpfVf/iTXrXOWNEIWeMKGxVlpkWpq6xmWfmbWkpW7B5H/92XFGiZBhGr8eLYdnoHmnu4Qk3/uWPwHk4SZ0+EpFXVHVlm6rPqup3vbYbb1LDjv/6F68elPXWih1mWAyjB3jZCfGObrY9EVinqhsARGQmTgLttoYlqVxy8tHsqapj0Zb95GWlsqO8lgWb9rGjvJZfvLKCy8cVc/5oW2NkGF1BYrlORGQ2HJp6TVXPjnHfF4ELVfUb7vXVwOnRvRN3KPQbYDewBrhFVbe201Z0Mu3xM2fO7PynwkkQnJPT9ZXML6+r56V1DQefDTx2YXaX2/FCdzX6SdA1Bl0fBF+jV33Tp09foKoTvLTpZSh0a9R5BvAFoNHDfdJOWVsD9SrwjKrWici3gL8ChxgsVX0IeAhgwoQJOm3atJgPLykpwUu9tgwbU81L95S0EjzlrKmkhOO/rKq7Gv0k6BqDrg+CrzER+rwMhRa0KXpfRN7x0HbMZNnq7Akd4WHgtx7aTSjDCrNZcvv5VNY28Mz8Lfxx9npqG5vJSYBhMYzeipdFiP2iLkPAeMCL0+EjYKS7Ves2nH2Kvtqm7egNyi4FVnkRnWhyM1PJzUylf98MAGobmmz62TC6gJe/lgU4IwLBGQJtBG6IdZOqNorId4G3cKabH1XVFSJyJ87+JK8AN4nIpW67e4HruvVTJIhIrMvbK3cyIDeDjJQwk48tSLIqwwg+XoZC3d4cXlVfx8nGH132X1HnM4AZ3W0/0UR6KTNeXNZStuBn51KQk54sSYZxWBDTcSAiXxKRPu75z0TkRREZl3hpyWfaqCJuv+REvjP9WKaMKEQELr3/fRqamqmpb6SitiF2I0ag2Vtdz0uLStlcVp1sKb0KL0Ohn6vq8yIyBbgAuAd4ADg9ocoCQFZaCtefebDD9vCcDfz69VWM/OkbAAzpl0XJrdMIhdqbADMOB56et5l7/rGGlJDw0n+cyUmDurfJ597qeg40NJGflUpmarhl980jFS+Gpcn9/AzwgKq+LCK/SJyk4HL9mcOYs3Y3767dA8CWvTVc9cg8br1gFOOG5CdZXXxRVa546EPWba/hkooVXD15KMMLs3vNH8zGPdWkhoUdFbWkpzi5kG+auYivnDaYkwflclJxLn0yUtu99501u9lSVs2Qgmx2lB+gf98Mvvf0IirrnCiMtJQQQ/plMaBvBrlZqTRV1LEuvIHCnHQKctIYnJ/FwLwM0lPCfv7IvuLFsGwTkT8D5wK/FZF0jrBcuRFSwiGevOF0dlbUsmjLPr71vwuZu76My/80l4evmcB5J3ZvVXRFbQMCHf4iJ4MPNpQxb6Oz6vvp+Vt4fO4mivMymXxsAZOHF/D5scWHXU/tzldXUlZdR9+MVJ78cHNLeXFeJhefNIAXF27jrjc+aSk/OjeDjDTnjz8lJIwfms+o/n1aLf+I5rozhjEgN4O91fVs3FNNWVUd2/YfYOveRt7cdOiEZ36WM/N4bFEOowb0ISRQkJPOgNwMBvR1jqz0MPWNzeSkpxxWRt2LYfkycCFwj6ruF5GBwI8SKyvY9O+bQWGUAzccEr755MfMvHEyE4/p18mdB2lsaubtTQ189/a3qKprZPqoIh67fmKiJHeZdbuqAPjVmZmcM3Uy/7dyJ3PXl/F/q3bywoJSbvvbUkYX5/L8NyeTlnLw/5mmZiUkJO2PoPxAA79+bSUNTYqqkpmWQn5WKrsq63hhQWmrut+cOpxmVcYU53LZqcX89DMnsr+mnkVb9rNyewVrdlZSuu8A+VlpNDU38/LiT6mpdzrw/3nx8Ywa0JfSfTUcW5TDvup6zjuxf7uBlLNnz2bcpCmUVdWxu7KOzXtr2FFey86KWnZW1LF4635eW9b5tuVp4RBDC7L489XjGZibSWZasHs7XmaFaoAXo66346x4PqI5dXAePzjvOE4/ph/DCrM57953+PKfP+CnF5/Av08d3qruul1VzJy/haWl5WyvOEBWagrrdlfR1KycVJzLsm3lzF69m3fX7ua9dXv4eNM+Hrp6fEJmnxqbmtlbXU9hTnq7PY61OytZs7OK5dvKARiQLQzMzeTqycO4evIwVJVH3tvIn+dsYMnW/Vz76HwmDS/guP45TBlZyIX/8y4Af/raOE4ZnNfSbrObaziRvRxVZc6a3Tz3sWNACrLTaGxWKmobyEwN0yc9hb/eMJERR+XQ0Njc7vvNy0pj+vFHMf34o9ptf3dVHXsq6zmuf47naGwRaYmNGl6Uw+nDDw1Z2Lb/ALUNTWSkhtlRXuscFbXU1DWSnhrizeU7WLhlP2f/3olNnTA0nzsvG8OJRwcz35pFfXWTlHCIm84Z2XL92k1ncdbvZvP68u2tDEtFbQOX/+l9KmobyctKZfyQfOqbmhld3Jdi3cMtXzqTX722ikff38jVj8xvuW/FpxVMdVdYr9tVxcY91Zw1srAltqar1Dc2869PdvKr11ZRuu8A557Qn79ce+iyj1ufX8KSUseo5GelktLGEIgI3zhrOF8YN4gfPr+EVdsr+HBjGW2XnN32t6U8/e+T6JftLIi/6pF5zF1fxrgheVx7xjDOPaE/2V0MOqxrbOJfq3ZR39RMUZ90ivMyqaxXmpuVUEi4c9ZKHnt/EwCLfn4e+e6zI9/3FBHhqD4ZHNUno8dttaU4L7Pd8wjfmDKcldsr+GRHJb//x2o+3ryPi+97l9OG5ZOXlUZeZio7K+uoqWtseTc1DU2s31VFWkqItHCI1HCIYYXZDC/KZkd5LanhEKlhYdPmBqbG6R1FMMMSJwb3y+LC0QPYsKeqpaypWZl2dwkVtY1cO3kod1w2ptU9JSUlhELCd6Yfy2nD8klPDRES4brHPuKaR+dz6uA8yg80sHGPMxX6owtG8Z3pI9p9/vrdVby5fAdfO30IeVmts1vsra7niw/OZcPuavKzHD/Omp2V7bazu7KOEwb25ccXjqIoJ509axe1Wy8/O41HrzsNgOq6RpZvK+fFhdvYW1NPalh4fdkOxv3ybaYeV8T0UUXMXV/WouXmmYsBOH5AHz4/tpjB/bIoyE5jSEEW/bLTOnRqvrRwGz+JiimKcEvJG+RlpbGnqo7j+ufwnekjWowKJLaX5BehkDCmOJcxxblcPraYuevLmPnRFvZU1bF1bw3Lahrol51G38wUVu+sZPbqXTQ0KcV5meRnp9HY1MyKTzveyPSXcX5FZljiSFZ6mDU7q1i3q5KquiaufmQelbWNFOdl8rPPntjhfQU56Vx00kDAGapcesrR7K2uJxQSCnPSOGtkIU98sJm731rNpacczeB+WagqLy/+lLdX7qR0/wGWbN0PQE19I1NGFLG3up6PN+/l6XlbqGtsJi0c4r+/cgqXnHw0v3h1BbOWbm8ZFvXNTG3pCVXVNXL+6AFMH+UMBUrWxv65s9NTOH14QUsXv7Gpma9O3MurSz5l1tJPmeOmBP3lZaP52ulDeWftbj7cUMY7q3fzmyhnKUB2WpjRxbn0zUghPyuNkAhLSveTkRpmb3U9AG/fMpUdFbVs3FPNJ6vXkj9wMHur69myt4YrThvCJacc3YV/tcOPUEiYMrKQKSMLO63X1KyEo4xqVV0j//pkF/37pHPyIGeYWt/UzDtz3ou7T8wMSxyJ/OKfe+8c8rJSqaxt5Jv/NpwfnT/K83g8JRzivivHHlK+blcVc9eX8cQHmzimMIfnF2xtySEz5uhcrpw4pGXR5B9nr2+574LR/RlzdC5TRhYy1p0S75edzv6aBka48Tj5Wal8MOMcUsMhKusa6ZvRs1+LlHCo5Rf/ri+cxO6qOjaX1XBScS6hkDB91FFMH3UUMy46gW37D7BpTzWVtQ2UVdezeMt+Nu+t4dP9tSwtLWdXpZO08KyRhTSrMmXkEEb278PI/n04a2QRJXWbmDbt+B7p7a2E2/TUctJTuLSN0c0kTN/0+PfozLDEkRunDmdHeS3rdlWxv6aBz548kBkXnRCXth++ZgKjb3+Lh9/dCMCwgix+dMEo/v2s4S2zMl+eMIhV2ysZmJvREicxrCDrkP+Nrp40lLSwUNvQzM6KWp5fUMq1j85HFVRhUH5WXDRDbL9EcV5mK5/C104f2ur7itoG0sKhbvuWjORghiWOnHFsIW9+fypb99bw90XbOLebcS3tkZ2ewgWj+/PPVbu4YMwA7r9y7CEGY+yQ/JZeSWcU9Unnu2c7jue91fUs21beErNy4sC+nDEiOAst+wYotsfwTkINi4dk2unAEzipGMqAr6jqpkRq8oPB/bL4XtSMUbx48KrxqMbXGdkvO403vz+V8poG+mYeXkFYRnBJWARtVDLti4ATgStFpK0H8wZgn6qOAP6bACR6CjIikrAZjtysVDM4q9osAAAG5UlEQVQqRtxIZGh+SzJtVa0HIsm0o7kMJx0lwAvAOWK/3YZx2JPIoVAxEJ0Yu5RDV0S31HETQ5UDBcCe6EptkmlTUlIS8+FVVVWe6iUT09hzgq4Pgq8xEfoSaVi8JNP2UsfXZNp+Yhp7TtD1QfA1JkJfIodCMZNpR9cRkRQgFydFpWEYhzGJNCwtybRFJA0nmfYrbeq8Alzrnn8R+Fdv2CPaMI50Ym5Y1qPGRS4G/oeDybR/HZ1MW0QygCeBsTg9lSsiOyd20uZuYHNndVwKaeOrCSCmsecEXR8EX6NXfUNV1dPewwk1LMlERD72umtbsjCNPSfo+iD4GhOh74jMBGcYRmIxw2IYRtzpzYbloWQL8IBp7DlB1wfB1xh3fb3Wx2IYRvLozT0WwzCShBkWwzDiTq80LCJyoYisFpF1IvKTJGkYLCKzRWSViKwQkZvd8n4i8raIrHU/891yEZH7XM1L/dzGVkTCIrJIRGa518eIyDxX47NugCMiku5er3O/H+aTvjwReUFEPnHf5+QgvUcRucX9N14uIs+ISEay36GIPCoiu0RkeVRZl9+ZiFzr1l8rIte296x2UdVedeAE460HhgNpwBLgxCToGAiMc8/7AGtw0kf8DviJW/4T4Lfu+cXAGzjrpyYB83zU+gPgaWCWe/0cTrAiwIPAt93z/wAedM+vAJ71Sd9fgW+452lAXlDeI85C2o1AZtS7uy7Z7xCYCowDlkeVdemdAf2ADe5nvnue7+n5fv3y+nUAk4G3oq5nADMCoOtl4DxgNTDQLRsIrHbP/wxcGVW/pV6CdQ0C/gmcDcxyf7n2AClt3yfwFjDZPU9x60mC9fV1/3ClTXkg3iMHV+j3c9/JLJw9zpP+DoFhbQxLl94ZcCXw56jyVvU6O3rjUKi9dA3FSdICgNvdHQvMA/qrs+kb7mdkZ6xk6f4f4MdAs3tdAOxX1cZ2dLRKcwFE0lwkkuHAbuAxd7j2FxHJJiDvUVW3AfcAW3A28isHFhCsdxihq++s2++yNxoWT6kY/EJEcoC/Ad9X1Y43dkmCbhH5LLBLVRd41JGMd5uC06V/QFXHAtU43fiO8FWj66e4DDgGOBrIxsma2JGGQP1+unSkqdtae6Nh8ZKuwRdEJBXHqDylqpFtaneKs/817ucutzwZus8ELhWRTTgZ/s7G6cHkuWks2upIRpqLUqBUVee51y/gGJqgvMdzgY2qultVG3C2Iz6DYL3DCF19Z91+l73RsHhJ15BwRESAR4BVqnpv1FfRqSKuxfG9RMqvcT30k4DySLc1UajqDFUdpKrDcN7Tv1T1a8BsnDQW7Wn0Nc2Fqu4AtorIKLfoHGAlwXmPW4BJIpLl/ptH9AXmHUbR1Xf2FnC+iOS7PbPz3bLYJNLxlqwDx8u9Bmd26KdJ0jAFp9u4FFjsHhfjjKf/Cax1P/u59QUn+fh6YBkwwWe90zg4KzQcmA+sA54H0t3yDPd6nfv9cJ+0nQp87L7Lv+PMUATmPQJ3AJ8Ay3HSgKQn+x0Cz+D4fBpweh43dOedAV93ta4Drvf6fAvpNwwj7vTGoZBhGEnGDIthGHHHDIthGHHHDIthGHHHDIthGHHHDItxCCIy1/0cJiJfjXPb/9nes4zehU03Gx0iItOAW1X1s124J6yqTZ18X6WqOfHQZwQX67EYhyAiVe7pXcBZIrLYzTkSFpG7ReQjN2/HN93608TJPfM0ToAVIvJ3EVng5im50S27C8h023sq+llu1Ofdbk6TZSLylai2S+RgPpan3AhXROQuEVnparnHz3dkxMDP6E47Do8DqHI/p+FG47rXNwI/c8/TcaJhj3HrVQPHRNWNRHVm4kSkFkS33c6zvgC8jZNPpz9OqPxAt+1ynHUqIeADnKjmfjjL+yO97rxkvzc7Dh7WYzG6wvk4a0oW46SAKABGut/NV9WNUXVvEpElwIc4C9lG0jlTgGdUtUlVdwLvAKdFtV2qqs04SyOGARVALfAXEbkcqOnxT2fEDTMsRlcQ4Huqeqp7HKOq/3C/q26p5PhmzsVJaHQKsAhnjUystjuiLuq8CSeBUiMwEWf1+OeAN7v0kxgJxQyL0RmVOGk1I7wFfNtNB4GIHOcmXWpLLrBPVWtE5HicdIcRGiL3t2EO8BXXj1OEk1pxfkfC3Dw3uar6OvB9nIWKRkBIiV3FOIJZCjS6Q5rHgT/gDEMWug7U3Ti9hba8CXxLRJbi+EE+jPruIWCpiCxUJ0VDhJdwUjguwVkV/mNV3eEapvboA7wsIhk4vZ1buvcjGonAppsNw4g7NhQyDCPumGExDCPumGExDCPumGExDCPumGExDCPumGExDCPumGExDCPu/H+yhFU7p/783AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "array([[0.83431145, 0.89172753, 0.94741184, 0.        ],\n",
       "       [0.7784706 , 0.        , 0.6691613 , 0.        ],\n",
       "       [0.71663981, 0.67644134, 0.66721498, 0.46171651]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Exact Monte-Carlo Every Visit\n",
    "lerror=[]\n",
    "ntrial = 1000\n",
    "V = np.zeros((3,4))\n",
    "N = np.zeros((3,4))\n",
    "for i in range(ntrial):\n",
    "    lS,lr,la = gen_trial()\n",
    "    for i,s in enumerate(lS):\n",
    "            N[s] = N[s]+1\n",
    "            alpha=1/N[s]\n",
    "            V[s]=V[s]+alpha*(sum_returns(lr,i)-V[s])\n",
    "    lerror.append(np.sum(np.abs(TrueV - V)))\n",
    "\n",
    "plt.plot(lerror)\n",
    "plt.grid()\n",
    "plt.ylabel('sum of errors for all states')\n",
    "plt.xlabel('iterations')\n",
    "plt.show()\n",
    "V"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.814236   0.88951396 0.96371352 0.        ]\n",
      " [0.79628186 0.         0.74524536 0.        ]\n",
      " [0.73744982 0.68197226 0.69329821 0.63981702]] 0.1\n",
      "[[0.877945   0.92243812 0.9763932  0.        ]\n",
      " [0.82094576 0.         0.82310518 0.        ]\n",
      " [0.7517182  0.70513165 0.6598951  0.46690713]] 0.05\n",
      "[[0.84919493 0.90940769 0.94686416 0.        ]\n",
      " [0.78770109 0.         0.56929002 0.        ]\n",
      " [0.72540503 0.64597438 0.55300864 0.23881913]] 0.01\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQwAAADUCAYAAABplh1lAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsnXd4FNX6xz9nW3pPIIRAIITehNA7CIIiYIGrWMCC/tSreAWvir13sF07ilcRvYoIiFSFiPReQofQQgghCaS33T2/P2Y2hYRkEnazCczneebZmdkzM99Mdt895533vK+QUqKjo6OjBYO7Bejo6NQfdIOho6OjGd1g6OjoaEY3GDo6OprRDYaOjo5mdIOho6OjGZcZDCFEayHEjlJLphDiX666no6OjusRtRGHIYQwAqeAnlLK4y6/oI6OjkuorSHJ1cAR3Vjo6NRvastg3Ar8UNEbQoj7hRBb1OX+WtKjo6NTA1w+JBFCWIAkoL2U8kxlbUNDQ2WzZs2qPGdOTg4+Pj7OEegC6ro+0DU6g7quD7Rr3Lp1a6qUMqzKhlJKly7AGGC5lraxsbFSC6tWrdLUzl3UdX1S6hqdQV3XJ6V2jcAWqeE7WhtDkvFcZDiio6NTv3CpwRBCeAPDgHmuvI6Ojk7tYHLlyaWUuUCIK6+ho6NTe+iRnjo6OpqpVwYjoyCDF9e9yJH8I+6WoqNzRVKvDIbFaGHVyVX8lfWXu6Xo6FyR1CuD4WXyonNYZ04UnsBmt7lbjo7OFUe9MhhWm51mnr1Js6ax4fQGd8vR0bniqF8Gwy75cJEFpGDF8RXulqOjc8VRrwyGh8mAkJ40sndnUcIisguz3S1JR+eKol4ZDCElzU1pNC3qRIGtgFl7Zrlbko7OFUW9MhhY81hpfIQRmfvp17gfcw/OJacox92qdHSuGOqXwTB7U4gJD1sOd7e/m/T8dFaeWOluVTo6Vwz1y2AIQZbww9ueTWzDWHzNvuxI2eFuVTo6Vwz1y2AAWcIPP3sGRoORTmGd2HFWNxg6OrVFlQZDCPGoEMJfKHwlhNgmhLimNsRVxClTUyJtiQBcFXYVh84d0p+W6OjUElp6GPdIKTOBa4Aw4G7gTZeqqoQ8kx+eMg+AzmGdkUh2p+52lxwdnSsKLQZDqK/XAbOklDtL7av8QCEChRBzhRD7hRD7hBC9ayrUgc3oiYUiADqGdUQg2J6y/VJPq6OjowEt+TC2CiGWA82BaUIIP8Cu8fwfAEullGPV3J7eNdRZjDR6YKEQAD+LH+1C2rE5efOlnlZHR0cDWgzGvcBVQIKUMlcIEYIyLKkUIYQ/MAC4C0BKWQjqN/0SkCZPLFjBbgeDgbYhbVlxfAVSSoTQ1PHR0dGpIVoMhgTaAdcDLwM+gKeG46KBs8AsIURnYCvwqJSyTKSVWlrgfoCGDRsSFxdX6UkzchWbs3rVCuxGD8hU8mQs/HMhAaYADbJcT3Z2dpV/h7vRNV46dV0fuEBjVVmCgU+Bj4F96nYQsFnDcd0AK0q1M1CGJ69UdoyWrOELP39Wyhf8pcxJk1JKuTFpo+zwTQe59tRaTdmRa4PLKZu0O6nrGuu6PindkzW8p5Tyn0C+amDOARYNxyUCiVLKjer2XKCrNjN2cexGtXNjLQAgJigGgMPnDl/qqXV0dKpAi8EoUmujSgAhRBganJ5SymTgpBCitbrramBvTYU6sBs9lBWr8mg12DOYYM9gDp/XDYaOjqvR4sP4EPgVaCCEeA0YCzyn8fyPAN+rT0gS0OAsrQppKtvDAGgZ2FI3GDo6tUCVBkNK+b0QYitKD0EAN0gp92k5uZRyB4ovw2lIRw+jKK94X0xQDPMOzcMu7RhEvYt219GpN2gJDf9OSrlfSvmxlPI/Usp9QojvakNcRTh6GNKaX7yvZWBL8qx57EvTZMd0dHRqiJaf4/alN1R/Rqxr5GhAdXraC0t6GFc3vZpgz2BeWv8SRbYidynT0bnsuajBEEJME0JkAZ2EEJlCiCx1OwVYUGsKLyDXOxwAe2qJzyLQM5BHujzCvvR9rE1a6y5pOjqXPRc1GFLKN6SUfsA7Ukp/KaWfuoRIKafVosYyFHhFUCDNyIyTZfZf2/xavExeLE5Y7CZlOjqXP1qcntOEEEFAS0pFeEopV7tS2MUwmQyk40dwTlqZ/T5mHwY3Gczy48tJzUsl1CvUHfJ0dC5rtDg9JwGrgWXAS+rri66VdXF8PEycl77YLzAYAA9d9RA2u425B+e6QZmOzuWPFqfno0B34LiUcjDQBWWOiFsI8DKThwVrQV6596L8o2gb0pYtZ7a4QZmOzuWPFoORL6XMBxBCeEgp9wOtqzjGZQR6mSmQFuxF5Q0GQNvgtuxL2+eYz6Kjo+NEtBiMRCFEIDAfWCGEWAAkuVbWxQnwNpOPGVmUX+H77ULakVmYSWJ2Yi0r09G5/NHi9LxRXX1RCLEKCACWuFRVJQR6WTiGBYrOVfh+lwZdANh4eiNN/JrUpjQdncseTZGejnUp5V9SyoXA1y5VVQkBXkoPQ1gr7mHEBMYQ4RPBX4l/1bIyHZ3Ln3oX6elpNpAjvQjIT4SsM+XeF0IwsMlANiRtILco1w0KdXQuX6oT6ZlZFyI9hRAk0FjZmN6qwjbXNb+OfFs+S48trUVlOjqXPxf1YUgp3wDeEEK8UdPITiHEMSALsAFWKaVTZq5Kg7HS9zuHdaZFQAu+3/c965PWM7rFaPpH9nfGpXV0rmi0DEkWCSF8AIQQdwghZgghoqpxjcFSyqucZSwAjFUk+xVCcEe7Ozh47iBLjy3loT8fIqswy1mX19G5YtFiMD4FctVEvk8Ax4FvXaqqCg6K5lW2ubnlzXibSqoarE9a70pJOjpXBFoMhlVNEjoG+EBK+QHgp/H8ElguhNiqZgd3CodMMfwReIuyUVixY1MIwc2tbi7e1p+a6OhcOlpS9GUJIaYBdwAD1KckZo3n7yulTBJCNEAJ+tp/4aS16pYZADBjZ3duMEOBDX8uJN8rvMJ2V9mvIj0gncSiRFYkrGBA4QA8DB4apdecKzL9vAuo6xrruj5wT5mBcGAK0F/dbgpM0JKS/ILzvAg8XlkbLWUGpJRy9LtL5LPvvKeUGzi+vsr265PWyw7fdJBvbnxT0/kvlcsp/bw7qesa67o+Kd1QZkBKmSylnCGl/FvdPiGlrNKHIYTwUcsqojpNrwHia2LULsTfIjhR4KNs5FQ9D65Xo170a9yPlSdWkn+RgC8dHZ2qcWXG3IbAGiHETmAT8LuU0imBEWYDZNjKJwOujPFtxpOUk8TX8W4LUtXRqfe4zGBIKROklJ3Vpb2U8jVnndtkgCy76kbRGM05IHIAfSP68vPBnymwFVR9gI6OTjnqZU5+k0GQaVX9tRp7GAA3t7qZ1LxUdqbsdJEyHZ3Lm4s+JRFC7EatdnbhW4CUUnZymaoqMBogy2ZW1FdjvkivRr3wMnnx6KpH6dWoF6/3fx0vk5frhOroXGZU9lj1+lpTUU1MAvKlCWkwI3JSNR/nZ/FjWNQwFh5ZyB8n/qDj/o7c0+EeFyrV0bm8qCxr+PHKltoUeSFmA4BARnaHk5uqdezUblOL1xclLHKuMB2dy5zKZqtmlZqlWnrJEkJk1qbICzEalLkkRSFtIO1QtY4N9gzmrvZ3EeIZwqFzh3SjoaNTDSrrYfhJpR7JhYuflNK/NkVeiElVbfUOg/wMsBZW6/ip3aayfOxyYgJj+HaPW6fF6OjUKzQ/JRFCNBBCNHUsrhRVFSZ1sqrVM0RZydXux3BgMVoY22os+9L3MX3LdCeq09G5fNGSom+0EOIQcBT4CziGG3N6QkkPo8ikdnTyazZCujFGSVf6zZ5vmLl7pjOk6ehc1mjpYbwC9AIOSimbA1cDbi1galJ9GIWO6euF2TU6j7fZm6+HK5GfH2z7gJNZJ6s4QkfnykaLwSiSUqYBBiGEQUq5CrjKxboqxdHDKDCq80kKau6D7R7enW+vVfwYK0+svFRpOjqXNVoMxnkhhC9KucTvhRAfAFbXyqoco+rDKCw2GJeWTatLgy60D2nP3INzsUv7JarT0bl80WIwxgC5wGPAUuAIMMqVoqrCrA5JCgzqkKSgZkOS0tze9naOZR5j65mtl3wuHZ3LFS3T23OklHYppVVK+V8p5YfqEMVtOIYk+UaHwbj0fJ1Do4biY/ZhwWG3JUTX0anz1MvJZ0ZVdR7qPBAnJPj1MnkxvNlwVhxfodcz0dG5CC43GEIIoxBiuxDCaSGVZofBsJvA6OGUHgbA6BajybXm8vvR351yPh2dy41KDYb6ZZ99idd4FNh3iecog6fq9cwrsoKHn1N8GABdG3SlXUg7vt3zrV79XUenAio1GFJKGxAmhLDU5ORCiEhgJODUqCgPdY5tToENPHyd1sMQQjC+zXiOZR5j4ZGFTjmnjs7lhJas4ceAtUKIhUCOY6eUcoaGY99HqWVy0bIENckabsvPBQS79x0kq0hQkHSMeCdlRvaTfoSZwnhnwzv4nfTDIKo/arsis0m7gLqusa7rA+dr1GIwktTFgPZ6JAghrgdSpJRbhRCDLtZOSvkF8AVAt27d5KBBF21azMpVq4BcwiOj8CMCP0DLcVopPFrIE6ufILNJJjfE3FDt4+Pi4pyqxxXoGi+duq4PnK+xSoMhpXwJQM0ALqWUWh0GfYHRQojrAE/AXwgxW0p5R43VqhiEwGI0UGSzg1cQpCdc6inLMCxqGM38m/Hhtg8ZGT0Ss0FrGRYdncsbLZPPOgghtqOUCNijVjFrX9VxUsppUspIKWUz4FZgpTOMhQOzUVBoVQ1G3jlnnRYAk8HElNgpnM07y+y9l+rz1dG5fNAyQP8CmCKljJJSRgFTgS9dK6tqzKZSPYzcdHDyU43BTQfTJ6IPs/fN1p+Y6OioaDEYPuqEMwCklHGAT3UuIqWMk1I6NUeoxWig0GYH72CwFVQre7hWro++npTcFOYemuv0c+vo1Ee0GIwEIcRzQohm6vIsSm4Mt2I2Gii0SvAKVnbkOj9afWT0SGICY3hn8ztkFro1K6GOTp1Ai8G4BwgD5qlLKHC3K0VpwcOk9jACmyg70o84/RoGYeDpnk+TZ83jjY1v6EMTnSueypIAf6euTpBSTpZSdlWXf0kpnetlrAFmo4Eiqx1CWyk7nPykxEH38O5M6jiJRQmL2Ji80SXX0NGpL1TWw4gVQkQB9wghgoQQwaWX2hJ4MTzMBvKtNvBtCAjIPO2ya93X8T58zb48s+YZfWKazhVNZQbjM5T8F22ArRcsW1wvrXICvMxk5BWB0awYjawkl13L2+zN9EHTSclNYUrcFKx2t+YP0tFxG5WVGfhQStkW+FpKGS2lbF5qia5FjRUS6G3hRFouczaeQPo3gkzXGQyAPhF9eCz2MdYmrdWzjOtcsWhJoPNgbQipLkHeZtJyCnn6192cszSCc8dcfs17OtxD/8b9mb1vNuuT1rv8ejo6dY16mUAHlB6Gg0zPxnD+pNODtyri3YHvEuoVylfxX+lPTXSuOOqtwQjyLpnfkWMKBHuR06a5V4a32ZuJ7Say8fRGJq+cTJ7V+QFjOjp1FS1zSXyEUOZ4CyFaqYWN3D4bK8TXo3g9xxigrJw/USvXntB+Are0voW4xDiWHl1aK9fU0akLaOlhrAY8hRCNgT9Rgra+caUoLYT7exavnwjoDsII+2on6Y1BGHim5zNE+kby04Gf9KcmOlcMWgyGkFLmAjcBH0kpbwTauVZW1QT7lHRy0k1hENwczh6otesLIbi7w93Ep8WzLmldrV1XR8edaDIYQojewO2AIzuulsQ7LsXPs8Rg5BbawL8xZJ6qVQ2jWowiwCOA97a+pxdA0rki0GIwHgWmAb9KKfcIIaKBVVUc43J8PUpsVl6RDXxClWnutYiXyYt/d/s3h88fZvmx5bV6bR0dd1Bl1nBglJRytJTyLQApZYKUcnJVJxZCeAohNgkhdgoh9gghXnKSZgC8Lcbi9YIiO3iHQG6qMy+hiZHRI2kb3Jbn1z3P0Qy3T+LV0XEpWrKGx9bw3AXAECllZ5TizSOEEL1qeK5yCCGK13MLrcqQJD/D6dm3qsJkMPF6v9cpshXx7Npn9dgMncsaLUOS7UKIhUKIO4UQNzmWqg6SCo78n2Z1ccm3KafQBmGtlY00509zr4qYoBj+3f3f7Dq7i58P/lzr19fRqS20OC+DgTRgSKl9EiU3RqWoQ5qtQAzwsZSy3PzwmpQZuDB1+omkFDb7n6E7sGfdMs42cE5ho+oQLsMJMYUwfdN0ngp86opLP+8K6rrGuq4PXKBRSunyBQhEcZR2qKxdbGys1MKqVauklFJGPblIRj25SN78yVopc89J+YK/lGs/1HQOV7Dy+ErZ4ZsOcvK8yW7ToBXHPazL1HWNdV2flNo1Alukhu+ylkjPSCHEr0KIFCHEGSHEL2pFs+oYpfNAHDCi+iatarILrOAZABY/yEh0xSU0MbjpYAY3GczKzJXsTdvrNh06Oq5Ciw9jFrAQiAAaA7+p+ypFCBEmhAhU172AocD+mku9ONkFVhACAiLdajAAXuv3Gl4GL2ZsnaE7QHUuO7QYjDAp5SwppVVdvkHJ8VkVjYBVQohdwGZghZTSaRXcQcnrCZBToIZm+0e4PC9GVfhZ/BjiN4SNpzfy5OonybPmUWAr4FR27QaV6ei4Ai1Oz1QhxB3AD+r2eBQnaKVIKXcBXS5BW5XE/XsQk3/Yzs6TGcoOz4BayYtRFdcEXENQRBCz983mZNZJekf05svdX/LbDb/RLKCZu+Xp6NQYrVnD/wEkA6eBseo+t9MowIuBrcIotNkpsNoUg1Hg/nIABmHgie5PMK3HNOLT4vlyt1L36e3Nb+vDFJ16jZZIz5ulEukZJqVsIKW8QUp5vJb0VYkjRDynwAae/pDvfoMBSmDZbW1v48aYG4v3/X3qb97c9KYbVenoXBpaIj3H1JKWGuFTbDCs4OGvVkHLd7OqEp7v/TwAjX0bM67VOObsn6M7RHXqLVp8GGuFEP8B/gfkOHZKKbe5TFU1cPQwsvLVR6ugVEELaOxGVSWYDCbW3LoGKSV+Fj/O5p5lVvwsGvs05pY2t7hbno5OtdDiw+gDtAdeBqary7uuFFUdfD3VHkZhKYPxxUA3KipPgEcAgZ6BGA1GPhjyAbENY3l789tsPK0XRtKpX1TlwzAAn0opB1+wDKnsuNrEMSRRgrcClZ05Z+vUsKQ0BmHgvUHvEeEbwQvrXtBrturUK6ryYdiBh2tJS41wDEmy860QPVBJ1QeQUncjLYM8g5jWcxqnsk8xadkkPUZDp96gZUiyQgjxuBCiSV0qlejAt7TT0+QBj8Urbxyv22nz+kT04Z0B73As8xgjfhnBb0d+c7ckHZ0q0eL0dMRc/LPUPgm4vfoZXDAkAfBrBCYvOLvPjaq0MaL5CCxGC4+uepSn1zxN+9D2RAfUiduqo1MhWiqfNa9gqTOfah8181ZOgU3ZIQQ06QGH/6yVwkaXypCmQ1h+s5Leb8z8Maw9tdbNinR0Lo6W2areQohnhRBfqNsthRDXu16aNkxGA55mA9kFRSU721wPWadLJqLVccPRyLdRcbzGi+tfpMheVMUROjruQets1UKUx6sAicCrLlNUA3w9zGQ7ehhQkn3r3FH4353wcoh7hFWDca3G8f6g90nOSebTHZ+6W46OToVoMRgtpJRvA0UAUso8QFR+SO3i62EsmbEK4K36ZFP2K8WNpA3stooPrkNcHXU1fSL68OXuL9mf7pJMADo6l4QWg1Go5rOQAEKIFigJfitFfaqySgixT80a/uglar0oPh6mEqcnlMRjLPl3yb5arllSU6bETgFg3G/j9ArxOnUOLQbjBWAp0EQI8T1KucQnNBxnBaZKKdsCvYB/CiFcUjHN90KD4duwfKP0BFdc2um0Dm7N9IHTAbh/xf3M2TdHn3eiU2fQ8pRkBUqZxLtQcmJ0k1LGaTjutGO+iZQyC9iHkrHL6fh6mMoOSUwW6Hxb2Ubn6swE2yq5ptk1rB2/lgGRA3hj0xvcueRO9qfvJzWv9uuu6OiURtTGr5cQohlKUecOUsrMC94rnTU89scff6zyfNnZ2fj6+hZvf7Yzn4QMO28P8C5pJCV91k3EUqQk18n0a8m22NqZAnOhvppil3bWZK/h5/SS0gXXBVxHP79++Bn9LuncztLoSuq6xrquD7RrHDx48FYpZbcqG2rJFHwpC+CLUmrgpqraVjdruIOnftklY19ZXr7hT3cpmcQdy9E1ms5/qTg7m/Sza56VHb7pULx0+m8nufDwwks65+WU8dpd1HV9Ujo/a/hFIz2FEM2llJdU+08IYQZ+Ab6XUlZZx6SmBHiZycgrwmaXGA2lHuA07w97Sl02ZS806+sqGS7jpT4vMbrFaAzCwLxD84hPjefpNU9TZC/ippZV1pTS0XEalYWGzwVihRB/Simvru6JhVLL8Ctgn5RyRk0FaqFpsDdFNklyZj6NA71KvaGGjjRopxgLN1RFcwYGYaB7eHcAYhvGkluUy/0r7ueFdS9QaCvk1ja3ulmhzpVCZU5PgxDiBaCVEGLKhYuGc/cF7gSGCCF2qMt1TlF9AQ39PQBIybxgSnuDNnDzVzBhAYR3gtSDrrh8reNt9ub9we/TOqg1r218jYlLJvLpjk8psukRojqupTKDcSuQj9IL8atgqRQp5RoppZBSdpJSXqUui50h+kJCfBWDkZ5TWP7NjmPBtwE07ABH/oTcdFdIqHVCvUKZM3IO93W8j20p2/hk5yfcsOAGDp67PIyiTt3kokMSKeUB4C0hxC4p5ZJa1FRtQnwsAKRVZDCKG7VQXt9uDk8cBa8gyEkFXy0lVuomFqOFyV0n80DnB1h8dDEztsxg/KLxTGw/kTvb3UmQZ5C7JepcZmgJ3FonhJghhNiiLtOFEAEuV1YNglWDUWEPo7hR85L17d/Bjjnwbgyc3ulida7HYrRwQ8wNfD7sczyMHny5+0uunXct8anxxW1S81K5Yf4NzIqvsmidjs5F0WIwvgayUGqT/APIREOpxNrE22LEw2QgNauSiPX2N8HYryGoGRxfX2IoEv6qFY21QduQtiwdu5RpPaZRaCtk/O/jeX7t86Tnp3Pw3EGOZBxhxtYZzNw9k3x7Pla7lZyinKpPrKOjoiWBTgsp5c2ltl8SQuxwlaCaIISgeagPc7clcm//5jQK8KqoEXS4GQ6vhB2z4arblf3p9fPJycXwt/hzW9vb6BXRi58P/MyP+39k+fHltAhUhmTeJm8+2PYBRozYvlMm5F3X/DqmdptKA+8G7pSuUw/Q0sPIE0L0c2wIIfoCea6TVDOah/pwPreI3m+srLzhgMeV18N/Kq+nd7lWmJuIDojmyR5P8suYX2gT3IZdZ5W/c/6Y+Xw9/Gv6+/Uvbrv46GLGLxrPTwd+UoLtMhIhpe5nLNOpfbT0MB4Avi3ltzgHTHSdpJoR5udRvJ5fZMPTbKy4YXBzCG5R0rM4X3/mmNSE6IBovhz2JT8d/ImE8wk08G5AI99G5ATn8NaotzAIA9vObOOtzW/xyoZXWHNqDfdvnkuHvBx4McPd8nXqGFUaDCnlTqCzEMJf3a6TefFbNSx50nsyPZeWDSt58ttuNKx5T1nPTYOsM+BXwQzXywSz0cztbW8vt9/brMy96dO4D/Mj5vPxjo/58cCPjA8P4aWzcFNhDlh8aluuTh1Gy5AEUAxFXTUWALFRJY8Qj5ytwpEXM6zs9o7vXaCoLLM3HCf2lRUuv05NEULwcJeHWTBmATGFhbweEsT+I8vcLYtzOYW8smivUmxbx+1oNhh1nbaN/Jn/z75YjAZWHzpbeeOGpdJyNGgH+xe5PO/ncwviScspJK+wbn/wQ7xC+Ew2wEtK7tr8Cj8f/Jmswiy36flw5SG+WnOU+dvrRwKky53LxmAAXNUkkJGdGvHbziRs9koMgFepgKYud8KprZDsWudnkLcaK5JbSaxIHaGhZzAzT6fgIyUvr3+ZGxfcyLok99R58Vazwp/JrDLJm04toCVruFEIMVoIMbmac0ncQrdmQWTlWzlz4bySC7n9FxjxFnS+FQxm2PxVlefOLrByLLXi4U5GXhGrD168ZxPoZQaULnadx2aldVERf6Tm822TMXhnpfB/K/6PiUsm1nqu0TA17P9sZTE2OrWGlh7GbyjZtkKoxlwSdxEVrDjpjqflVt6w5VDo9YCSMLjjWIifB3Z7pYfcM2szg96NqzBl3huL9zHh600cSK64+x7grRiMSsPX6wp2JXuZyDpNl9Uf8dPJEzze7m6OZBxh3G/j6PNDHz7d8Skns04CkFWY5bI0go77lppd/w3Ga7/vZdJ/N7tbxiWh5bFqpJSyk8uVOImoEMXz/+v2RHq30FheoGkv2PkDnNqiFEG6CJuOKRPXsgqs+Huay7yXV6T4JnYmnqei8CdHDyM9pw5/8O02mP8gJG0vs9tTSib6xHDt6Hl8vvNzfjr4E5/s/IRPdn5Cn4g+rEtaR9+Ivjze7XFOZZ+ib+O+mAxaPlpV47BDl4PB+PJvJb1Mubwt9QgtPYwlQohrXK7ESUQGeWEyCP4+VI38ly3VP++rYXDs4pXHgtRfu1PnysetRQUrhup4WsVDlgZ+ngAkna+bVeUByE6BXf8DWwE07Q1GS8l7Sdtp4N2A53o/x447dzB/zHyGRQ0r9m2sTVrLjQtv5OGVDzN6/mh+OvCTUyrTO1xRtTIk2bsAClzv4K3PwystBmMD8KsQIk8IkSmEyBJCVPlJEEJ8LYRIEULEV9XWmQgheHBQC85k5pORpzE/hH9Eyfqif0F+xQFLrcOVkdjuxPLve6iBYscuMhTyUp13Ry/iA9FKek4hn/11hDRX/OLaSyVS9gqG1teWbO9fVLxqNBhpEdiCGYNmsPG2jTzX6znmjZ7HUz2eYlLHSVgMFl7Z8ArDfh7Gr4d+vejlXt/4Oo+sfISjGWUTu+UXlTxJsqsWIzXbxUO5zCT4aQLMHuva6wBJGXUuUFozWgzGdKA34C2l9JdS+kkp/TUc9w0w4lLE1ZTYqCDsEvYmVeMXrs9k5TX1ILzLSfACAAAgAElEQVTZFLb+F86fLNPE4R/ZfvJ8ucMdH+wTFzEYdrVvffoSPyzfbzjOm0v28+DsbZd0ngopbTCykpS5Nw5SD0J6+YyN3mZv/tH6H7QMasntbW/n0a6PMm/MPL685ktaBbXi+XXP88CKB9h9dne5Y+cfnk/cyThuWXQL/7fi/1h1YhUn03Np89xS5m5Vylw67luZMhKuwKr2/E5ucNklPEzK1y3pfP01GFoGmoeAeFlNr5aUcrWaLbzWcXT/z1fnEebQl+Cq2+CTXsr2b6oBeXgrhMYAJR/e7SfOlTvc0XU+cCaLovaeFbyvGoxLGJJYbXamr1AS5CRX9RSoJpQ2GEnblWEJgMlT+UId/gN63FflaQzCQK9Gvegc1pmPtn/E7wm/c9vi2+gT0YdejXqRWZhJsGcwedY8RkWPwiAMbE7ezORVkxkcMRqjd0M+ifPk+VgjtlIfu0pD/i+V0pXxslOUpEtOpqG/JyfScy97g3EaiBNCLKFUxTNn5em8oMwAcXFxVR6TnZ1dabtz+crTji9W7MQr7UC19IS2f5J2e6djkMqXJ/WHBzgccx+FlkBOnVbaHEjOYukfq/A0lTiuEo4qxqnQaudYak45fYmnlFt3Mj2HlatWYRDVd3qdzy95ipOTm6fpXgFEnlxIvmcYqWG9i/dVdA99so/TXV3P9GvJti17CWv3BPmeDWi7bzp5G35gd27LamnuQQ86hHVgTdYa/jrzV7l4joCMAHr69qR/cH/mMpe/khbhHWXnrN2Dj5Oa41nUEaOPH7a8SH5b8Rdh3tULHdqZu5MwUxgRlohK23nnnMDh7j6w8H1OR1Tttqvqc3ghskgx8pv3HKaV/WQVrZ1DdTVWhRaDcVRdLOriVKSUXwBfAHTr1k0OGjSoymPi4uKorJ2UksfiFhMSEsKgQd0v2q5iBkH6rbB6OqTsJTRpM6FpyqOwwuDxrGMUEghr2ZnYqODio7YWHoAjhwFIKfLgvgv0LUvfRa9Ty0iSIbTodAtRIdWfo7E/ORPi/gYgq0gwcOBAhBbD8+IY9bXE91LhPTy9E7YAI2fg33Ecgzz9AbWN6SDe275jUN9eYC7fg6qK67gOu7STlpeGxWihwFZAnjWPSN9IjAal1zCc4aw5kshdP8zB6LePk8H7yGE/3k2Vc8zMbslznafQrWE3PIweFf7tr254lcyCTLo07EJ8ajwLzy4sfu+2NrfxZI8nMYgKjE5yPKhPPFvLw7Qe8CoYKjdOVX0OL8Rr+1+QlQ0+IQwaVHUJEK38b/MJ4g6c5dM7Yi9ZY1VomXz2ktOuVksIIejfMrTmjrLgaLjhY4j/BebeU7x7aPoPeBmvI89m5HBKdhmD4XhU5udp4tC58vEcdjv8aFGK3s/cM4BJA2KqLeuDPw4B0C0qiC3Hz7EnKZMOjdVJxLt+UibUPbD24h/0qiaT2dQhSUAkeF7gpmo+EDZ9Acm7oUl1jbCCQRgI8648JaLF4I01uz3W7PY8Eu3JKR8DX+38EYPBShKbefCPB4vbtg5qTfvQ9jzW9TECPQORUjL34FwAlhwrySo5ttVYfj30K3P2z2HD6Q1MbD+RUdGjmLBkAnbsPNLlEXrhrXwZGnaAQ8tgzXQY8G+ciSP6eENCGna7xOCkR6tP/qL4h5x5zotRpcEQQqxCLcRcGinlEJcochJhvh4kVDUJrSra3Qhd46BhR+VXdeEj9AtIJzHLCnuPQZuxxbNcbVIxGO0j/DmeUt7HUWYsfmAl1MBgOKbwPzGiDf/4fD1L4k+XGIxlz0BOiuK0i+pT8QkSt0D0wItfIFNxNFJRDEUjNRQneWeNDYYWrLYSY5ucKwkJakbBmdE0DfbmxIFr+e6hcBYl/MbJrJP4WfxYcHgBy48tZ2jUUNoEt8EmbUyNnUrPRj05l38Ofw9/OoR24Plez/NV/Fd8tfsrXlj3Ah9s+4D0fCWu5sE/HiTUEsD1QYGM6XkvLXb8hFj5GnS6BQKbOu1vc/i5svKtpGQVEB5Q/Z5aZfy5P4Vh7Vw761rLkOTxUuuewM0ohZYrRQjxA0p/NlQIkQi8IKWsOv7aSYT6eZCaXYCUUlu3vSIMBhj9kbJ+/gQAdxfOob15PwFHM2D60xASA406I72mYhSC9hEBrD2cxosL9/B/A6OLs3/ZpaQIE2as9Dw9G9VtUy38PE0YDYIezYNp6O/BV2uOEhnkzajOEfjGDIWdc2DDJ+UNhsNpmbipcoPx0wTl1Wgu/15AE/AMLJdw6LO/jrAkPpnXbuhQYrwuAWupOUCnc+wEqIa2dbgfJ9JzifLqylsDivM5sTl5Mx/v+Jg/jv/B/MPzFakeAbQNaVvmvEIIJnWcxL0d7mVRwiJe2/gaAC/2fpFAj0AWxv+X2QXn+Wb3e3QKimZghh837V9EaK+HLvlvcmCzS4J9LKTnFHIiPddpBsNsFBTZJPd9u4Ulj/anbSMtDzFrhpZizFtLLWullFOAnhqOGy+lbCSlNEspI2vTWAA0DvSiwGonKcNJTxMCmpBt8KOPdSMB9lJxGGmHIf4Xos+twSCgezNlmPLNumPc/+3W4mZSgl293d3tO8lITaq2hIIie/GjuXaN/MkvsjNt3m6++OsImNQEQvt+g6zksgda1NqaB5dX+5rFCAHhHctM0vtu/THeXLKfnSfPc/1Ha/hi9RFu+Xw9XV9ZUePIzNKTBk9ny+LtCb2jADiUUjawqnt4d74Z8Q3rxq/j/UHvc0+HexjY5OJGUQjBqBajmD9mPv9o9Q8GN+rN1cHt+aDDgyw7mcRT0TeTazDwUXAgww58ysN/Psys+FnsSLn0rJQ2uyQ6VBkSHk3NvuTzOYgMKqkpHH/KtUmPtEw+Cy61hAohhgPhLlXlBLo1U2akbkxIc84JheC18PeI8xiE1eDBnYVPkTxhDTyqfIE6pi3FYBAMaBVafMjuUv88u5QYsZEd3B6AYwcqyVYuJWz6EvLKDm0KbXYsJgPYiviw+Ubu76Fc68OVhzmbqQ6/DGZY+WrxMSlZ+eTkq1/epG1g1eDXqcgpCBDZTfFh5KSRkVfEO8sOENPAlynDWgHw+uL9bDyaTnpOId1e/YNdieXjVarC0cPw9zSRnGMv7sa3VhMiHTxT8RdNCMHVUVfzWOxjBHsGV9imNOE+4TzX+zmClz8P01tDYQ4NbDZujxzCr2N+5deQQdxxPpP9KTuYsXUGdy65kwlLJrA/fT8FtpoZQ7uUNAv1wc/TxM4Kgv8AcgutfBJ3mPhTGSzcmcSyPclMm7ebn7dc/KlKg1LZ5qoV4VwDtAxJtqL4MATKUOQocK8rRTmDNuH+NPDz4Mu/j3Jjl8Y1H5aUItHYlA8CniBwVHv+/ngt23JCuS66EXS7h/ZbvqY/vfAwDUdQ4vQ5nZFHowAv7HaJCRuWiI6QvoefFy+jeezQcnNSAI4f2kXU4sdh+2z4v5Ks5sU9jFNb8Yt7lqfbXI/vsBeZseIg57PzCAtoCi0GK87akdM5nWNn9H/WstJmJcMUTIAtXQnACu9Q8R9o8VNSGEaVrT9rt0se/mEbPuktecduZeH3HzE5QfFjfDmhAz2jQ/hx0wmSMvKZMqwVn/91hJxCG3/sS6FTZGC17rFNnQDYOtyPPYnniuNXArzNNA70YtX+FB4c1KJa56yUU2oA3MqXlVfVfxPT6XambvmWqefOc/7JoyxMWMTM3TMZ99s4TMLE6JjR/HXqL6KWRBEdGI2XyYtdZ3cT7h3OoKYDGdl8ZLnPnM0uMRkEVzUJZPuJio3ppP9uYd2RNN5eWjYc4IdNJxjeIbzM5yUlM58/9qVQYLXTv2Uovh4mdlQQVOhMtAxJmkspo9XXllLKa6SUa1yqygkYDYJrO4Sz73QmS+KTqz6gCpLO5/H3oVRMBkGbcD8MgpKZqX0fBeATXoef7+aOhiV5Qv+3Wf1lkEpgkCWsBWeMjXjVPIvcWWPLzZDNL7IxaZYabXh6BxSWRI7uTDyvTMZylEQ8uIxHhsTgYTJQUFiIVRh4L7ElFGbD0dW8uWQ/Z7MKMGFjW2ETAE7s23TxP1LaFB9HqQ/6lmPpTJy1icW7k/n5VCB77VFEJS4AYGxsJD2jlQl+/xwSw6jOEfxzcAx7Xh6Br4eJ7PzqR2c6ehitGvqRay0pf2kQgiFtGhCflOHcmbGO4DRH2QmhBoY17QXXKH6OwFPbmdB+Ar/d+Bt3tb+LFoEtWJywmDRrGttStjH34Fy+2/sdO8/uYOmxZUz7exoD/zeQqXFTWXNqTbFeu1SeYnRpGsSB5ExyLoheTc0uYN2RNEwGQYswH/w9y/6ed379ex5f+BtFNjtFNhu3zVzH07/uZsfJ8wR4menRPJgT6bkknHXecOdCtAxJxgkh/NT1Z4UQ84QQXV2myIk8pnaVH/p+G4dTajapSErJh38eos+bSjZyL4sJT7ORyCDvEoMR1Iz9fqqjcc88XsmYxsbYlXSODMBz88fw/TgMdvVLbjARPOJJAMLPxMHWr4uvNWfjCa778G/MpX3Kx9ZwIDmLdYdT2Z+cReMgr5KITHsRIieVEB8LR1MySM6y8tmJSPIM3rDje5Iz8gn2sWAUdg7JSAB8456/+B9rt5Z5QhJ/KoOxn60v082daxtAZ0MC/25xgrdu6gjnFON4e88oPhrfpXgWppfFWKMhicNn0UZ13Dl+MY1CEB7gSW6hjZl/lw9RrzkXGJ/ST4hi71KGZ1uU/1GARwBTu01l7ui5bLx9I1PCp7D6ltX8eP2PdJUfkrX/ZbL3v0pe0jiifDqxKXkTD/7xID2+78GoeTeR5f0L6bY9tI4wYZew64JhyZtLlFwj//u/3vw5dRC7XhzO9ueG8exIxYHr22I6y849TecvRtF19lUkBz6JR/ivmAI2c2c/f3qrxnvRrtNOvD9l0TIkeU5K+bNaamA48C7wKRocn+4m0NtCgJeZjLwihs5YzaHXrsVs1BYpuO5wKrtPZTCgVRgz1HDsa9o15LnrlfR+HRr7s3zPGTLyigjwMvNd01c5vX8jX48JhXn30XDPTOYaZmO258Mh6Oco1Wg0Y+5wA/z+L2X796lwZBXc+j1P/6o8T+8oSnodJ5a+x/Ckkkl0k4e0BHsp/8emL+jd4nqMu+3kFkkKsPBV4TAe3vMr9oKOtI/phznRTj5mEuzhRBuS2b9xKVKWjHuLKWUwimx2Jv+gTHN/Z2wnercIYe3hVGb9ZcZW9BsP+a1BHG8L346B0f+BrneWOdXZrALOZhUwe8Nx7ugVpemeA+w8qXyJYpsG4WGkeKwvBPSLCeWdZQd4bfE+JvVvXq7Lv/V4Ojd/uh6AZ0e25XRGPhN6R/HBH4fw9jDy8ugO5eMUbEXKE6AMtSdoKBV67uELbUfD3vmw7BmmpI3Gw9ObN27qyI+bEjl8OpIgzyCCPIPIK8ikdYMQWjTwYfFuI61Ec14fEcGWs6v4fvt69qZuwRJyiLU5a1m7BXxi/Hlygz/n1iQS5R9FVo6F5JSGdGzdksjQbuQW5bL02FIifCPIMB1ClIoqNnorT+wQNnxDdlFo38ikVb8Q6BFI47bhfLo7GL+wQXQJb0uB3bmTFLUYDEeQ/UjgUynlAiHEi05V4UJm39uTUf9RRlBfrE7gf5tP8q+hLYvH1jENfMsdk11g5baZGwH4boPyCzpnUk/6xJQ4NO/q05zFu5N5/4+D3NK9CVtP5XDe2Bo6Xc2aM970O/IW5lJPFMalfaqsGEzgFcTXzWcQnTCbQWKbMhP0ZMlQwdHD2GmPpmPaevy5i0x8S/SeVXsY/o1h85dMf+xfnDjrjU+eN9dHNOLTXaMZY1jHc+bvuOdUB4S08/CQ1qwJeZrgeYM5s+g1vgt/lsGDldPEHUjhv2sTmCXtzNqQyLAuuRw8k0VCag7Trm3DuG7KcOaW7k25pXtTWHobbJ4JzQcoJ1jzXjmDMb5HU37YdIJn58dzddsGFReXKsW5nEJeW7yPuVsT8TIbaRzkRbsQI9tTbAihODU7Nwnkmeva8trifTSftpgQHwteFiNv39yJPqoxcfDq7/sAyY61S/EXucyzd+HGLpFlkkWTmw7pCcpj51Yj4OBSrNbCMl+K3AHP4r13Pqz/D9mFZubZu/P6jR2Kjbvnor3c3rMpm46m0y8mlE9uj6XrKyv4as1RZq09yqjOzdiywwIMRJjO06u1lfYtT7JwVwIZWTn4BwSAzYuz2dmYg9dxTKxh6NzyhQV91Yh8v5yxnD7VGoMljSBja9Y+OZgT2QlsTt7MwXMH2Z68jwz7FmZsV0Lwbwm+heEMr/TeVwctBuOUEOJzYChKcWYP6lEu0I6RAfz3nh5M/HpT8Qfqt51JTPlJ+ZWOf2k4B89kUWi100vt0pWevp6o5r7wu8A52aN5MO0j/Jm19hiz1h4r857V7Av3x4HdysEfp7Fx/3HuNP1Rpk2bvmP4/MAZBllUp9tXw/A1fEO23YJJtdG/23rS2ZzAso5x9N59PQD+XuYSH0b/KUoPZdEUmgZ5gsmbh4fEcC63kI+O3sDb5i95vvEmOAkGo5kBnVuzYu0tDEv5midOptDsqd/Z+/Jw7pq1GRNW8ITUHBv93lpVrHNsbGQFN3WsEu+x80dlO/1IuQlbb9zUkVGdGnHbzI3FxaUmD4nhX0NbIYSSBiDQy0yB1c7MvxNYtOs0Z7LyubNXFA8PiSHAy0xjXwPbU2xl8jM7/kegZi/LgQU7kugTE0p6TiEDW4XxQOscmq35N7bcc0QKZTi11x5Fevr3nPWDsKBAWP8fWP6scqIG7cgf/RlfffgSH3+ZSvxLJRGT/1qeQWbhs/xoeZUvLO/xVNEkUjIHF2v4as1RvlqjDJHWHFaudVuPpvxn1WHsUtEW6mvhq4mKk7hJsDfBPhaGNkjj1i82kAE40hv/+XhPztuO8038N8QlxtHAqwHP9HqGAlsB5wvOcyrrFAPCb8JoD6J9hH/xRLyWQS1pGVQyx+faD/4i05rCMzcEkpfg3ET/WgzGP1Cmqb8rpTwvhGgEODdm1sW0a+RP02BvbHZJqK+FVQdKcm9+FneE/6xS5oAkvH4dBoNg3rZEhIDnr2/HS7/tBcDXs/yt+uDWqxg6Y3XFFzUYwWCk8bi3eev1PzlR1IBnzHOUpxQoBudhr+68EPA2w3N/p0/+X/RjB/1vvIeItBzYBMOHX8/KbYUMOTSHD66dyNKTZvw8TCU+jKi+yjh76zfg2xACImkT7s/3k3qBtStFM7cw+uS7JXqAYeMegI+/ZpnHk/Qs+Jhv1ys9KCPKMMhW6rfg3n7NCfGtYOgS0VUpBpVUaor9mvdhxOtlmvWJCeWRITF8tFK5vx+uPMyH6jpA+wh/9pRKQTB1WCseubrkgz8w0sSihCImGJfBN5/CgMfpGD2IqcNa8cXqBLIKrPiRi+18IsQfYlz2MoYX7qbpCaV3YW8xBHtAJIbt39LOcBwW9KNAmkkN60JoakmPzn72ADfM3M3+zGGAJPrpxcS/NJyzWQUs33sGaEuSKZIIayJvmmfCezP52xLG5z4P8kt2e/KKbESH+vDcKGW4OvWaVgR6m3n1933c2r0Jb95cPmFdr+gQOjcJZKfqo3lwUAtahIYCocQ2jCW3KBcvk1eNnu49NrQ193+XzUNfFjKtR/WeUlWFlrkkucC8UtunUWaw1hvC/DxY/cRgpJQ8MXdXmWfgDmMBEP30Yl65oQM/b01kXGwkd/dtTqC3me/WH6dRBVF5MQ382PrsUB77aedFEwD7eJi4uWtjvl1/DR09zjC61z8BMBkNPDa0Jc8tKGQBt7HGYyOfWd4nyzMWv1bBsAlimzeAjs/Dh78yJvNHxtzxvnJSh8EwmJVp+fHzIPtM2TBmkwXzrd/Ch12U9kXqlOqw1mSF9yAweROTTfN4c4mZRgFexE3uC+9A31bheDVuRUwDX67reJFwGyFgxJswZ5yy3Xm8MkTpMUmZh1OKh4fEYLVL/j50lvhTmfQ37CKYTFbau7KnVOzabT2b8s/BZcPl22WvZ3erbfid+AOOAcf+hkZX8cjIGTyS8QsFp/fgcTYeEoG5cB8oRTybD4CRMzCEKsbn1IC3+GP6BCaaVuAhivBI3USyXwd+DZnEg8f+hUHaOJaWw4uj2vHl30c5dT6PDi+U1GRZ8dhAIkJ3sn/Jf2iz5QUAmhjO8mrey7zaaxKYvWHdh3ByCrR+QYkq7R/NkDYNaFbJJMNv7+7BQ3O2MmVYK7o0CSrznqPIVE24pn04XZoGsuNEOgVW5+YREa5K3loTunXrJrds2VJlu0uZgXcup5AuFxQUemJE63LPvRf8sy+dm2i3zvlFNmx2iY+HqZy+42k5DHwnDoBjb44s3m+3S56ZH4+HycBQ32P03fQQIv+8YgjsRXDfKmjcFRb8UzEKU/aBVyDsmKPk3py8Q4mb+Hs6/KnGEVxY3jArGeLegO73lcRf2O2c/vhaGqVtYLXohu/t39K1sTe81UwxBL0eRBMJfymGqEFb+LgnWPPg0Z0Q1Kxc07OZ+fz+wUPcZfsFgO8ZwTP5E3h4cAyPD29d/twrX4PVbyvrUf2g1TWwovwTHunXiGXGgSSk5pIpvRk89Dp6DhxVZgJeRm4RnV9eTnNxmj8sj7NZtmFi4ZNIBAc9laqf5584S6C3hbxCG4PfjSM5M5/OkQE8NDiG4e0Vw7khQRlGgGTxLYG0WzCynB4MZrhznmK0jq6G/90BVz8P3Sdpu6fO4OQm7HPvxZ6fyd5Wj9Dp5serPEQIsVVKWeUUWudkaq1HBPlYaB/hT0ZeEb8/0r84K/XIjo1YsCOJL1cn0DM6mI7VnBdRWWKXqBAfRnWOINi7rB/EYBC8cVNHdas9dFgGn/RUjAWUzOnoPkkJ4lryBNz0RYkPw/F+vylw7hj4V+Bv8AuHUR+U3WcwcKLFRBoWJTIgcwv8PhLGfaO8J6qRoKb0vJTRH8G8SYrh6D9VycaeuEVJrtx/KmGnd3KXrcSPcztLud7jb4raliqeLaUSe/LfMVCQwdnQnoRN+hk81f9F9/sU42nyhKufA58wBIJrhJGJszbRpUkgPQa1KhNHAkrQ17Rr25BTEIN9cAo//7qPgm3KRLtCSyCWlkMIVOvGeFmMrHlS8VGYLnii5mNRvi7Rob6069KPjac/oWeDQmjWD7xD4LsblHiO/45SCmSlKMNZfp8KB5fBLbNLQvirw8xhStrIW+cUJ3MqQ2EOHP0bNn2uvNqLMBgtGFoNx2pybqnLK85gAPw+uX+5fVEhPky+uiWTS42hnclH47tU3ahBG3hoo2I0ShPRBXo+ABs/A7NXydwQR8yAECWT5DSS5x2BYcoeJYx89Tvwhfrlr2jimRY6jVNyo875B6x6rex7f09XXq9+XjFuCx+G7bMJEDmQEQ8ZfuqXqmRKOj4N2Nd2CmGepQy3xRvGlX+CYAC+u7fyp/z/N7AkOvSpa9uw6VgafVuEYrnhSLlQ+AsNhYPW4X48MiSGW3soQ78878bQbVBJg0l/KrlBN30BCXEl+xvHwqHl8HEPeHiLMnw8uBSMHkpdnN1zlV5Z8m4oyFT+1xYf5f9akKVMGgT4fAA8vBkCGivb1kJY+Ajs+rGs0OBouHsJ+IWT6cTkOXCFGow6TYM2MOwVZYgRUKrHMOxlJUXe1m9K9hmdkM9o0NNKr8JepBigtqNqfq5mfeHJ47DyFUhYpThGQ1sqT1D8Gik9D4CR78E1r8J7HeDPl4pnAhM9SLl+zFAIaobdyR92B2F+Hvz9RPWzM1hMBqZeU8HwyYHRDEFRMFw1mJ/1VybrTVgIs0YoBuGb65VKe45e5MKHy59n5SvKa1Q/OK4GVcfeBdu+UwzrLbMVIzdnXIlhGv2RkoM1K1kxGE6YClERLjUYQogRwAeAEZgppXzTlde7bOg7WVlKY/JQkuOc3qnUTzF7Kd3+S8VggMHTLv08DowmGPYSUEneJZNFWa57F+Y/oOyrIPir3nPvciW038MXJq1UhiqOJMNjPlYC9jISlZD89jcqvYr9v0NummJUjq9RHKpdJyg/GAazEnX6VjNw1LttcbXiM3EQ4sR5NhXgMoMhhDACHwPDUPzYm4UQC6WUe111zcsesyc07akslwNXjYfWIyBxK8Rc7W41zsfspSygGMg7f1WGbFIqBqLLHeWPib1LebUWwJk90OiqEgfu4KeV1+PrIGUPxAyD8T+WP4cLcWUPowdwWEqZACCE+BEYA+gGQ6cEryClbOWVgMUb7lpUdTtQepSNL5iy5R0MI9W4GmuBMoQ0VMNJ7QRcGbHZGCg9iT9R3aejo3OpmDxq3ViAa3sYFXldygV9uKLMgLup6/pA1+gM6ro+cIFGKaVLFpRqactKbU8DplV2TGxsrNTCqlWrNLVzF3Vdn5S6RmdQ1/VJqV0jsEVq+F67ckiyGWgphGguhLAAtwILqzhGR0enDuOyIYmU0iqEeBhYhvJY9Wsp5R5XXU9HR8f1uDQOQ0q5GFjsymvo6OjUHnVq8pkQ4ixwvMqGEAq4Nj3ypVHX9YGu0RnUdX2gXWOUlLLysnTUMYOhFSHEFqlhZp27qOv6QNfoDOq6PnC+xnqTOUtHR8f96AZDR0dHM/XVYHzhbgFVUNf1ga7RGdR1feBkjfXSh6Gjo+Me6msPQ0dHxw3oBkNHR0cz9cpgCCFGCCEOCCEOCyGecqOOJkKIVUKIfUKIPUKIR9X9wUKIFUKIQ+prkLpfCCE+VHXvqq1Sk0IIoxBiuxBikbrdXAixUdX3PzVkHyGEh7p9WH2/WS3pCxRCzBVC7FfvZe+6dA+FEI+p/994IcQPQghPd99DIcTXQogUIUR8qX3VvmdCiIlq+0NCiImaBWiZcFIXFpTw8iNANGABdgLt3KSlEdBVXfcDDgLtgLeBp9T9TwFvqevXAUtQZvD2AjbWks4pwBxgkbr9E3Cruv4Z8KC6/hDwmbp+K/iysiYAAAUwSURBVPC/WtL3X2CSum4BAuvKPURJxXAU8Cp17+5y9z0EBgBdgfhS+6p1z4BgIEF9DVLXgzRdvzY+GE66UdWe/VqL2hagZBY7ADRS9zUCDqjrnwPjS7UvbudCTZHAn8AQYJH6oUkFTBfeT5T5Pr3VdZPaTrhYn7/6hRQX7K8T95CSfC7B6j1ZhFJb2O33EGh2gcGo1j0DxgOfl9pfpl1lS30aktTJhDxq17MLsBFoKJVCT6ivjtqB7tD+PvAE4KjsHAKcl1I6KtuU1lCsT30/Q23vSqKBs8Asddg0UwjhQx25h1LKUyiFx0+gFO7KALZSt+6hg+resxrfy/pkMDQl5KlNhBC+wC/Av6SUlRWxrFXtQojrgRQp5VaNGtxxb00oXetPpZRdgByU7vTFqO17GISSUrI5EAH4ANdWoqHOfT65uKYaa61PBiMRaFJqOxJIukhblyOEMKMYi++llI60zWfU2rOorynq/trW3hcYLYQ4BvyIMix5HwgUQjhmKJfWUKxPfT8ASHehPsc1E6WUG9XtuSgGpK7cw6HAUSnlWSllEUq50D7UrXvooLr3rMb3sj4ZjDqTkEcIIYCvgH1Syhml3loIODzOE1F8G479E1SvdS8gw9GFdAVSymlSykgpZTOU+7RSSnk7sAoYexF9Dt1j1fYu/XWUUiYDJ4UQjkIfV6MkiK4T9xBlKNJLCOGt/r8d+urMPSxFde/ZMuAaIUSQ2pO6Rt1XNa50bLnA2XMdyhOJI8AzbtTRD6ULtwvYoS7XoYxZ/wQOqa/BanuBUnLhCLAb6FaLWgdR8pQkGtgEHAZ+BjzU/Z7q9mH1/eha0nYVsEW9j/NRPPZ15h6iFFfZD8QD3wEe7r6HwA8oPpUilJ7CvTW5Z8A9qtbDwN1ar6+Hhuvo6GimPg1JdHR03IxuMHR0dDSjGwwdHR3N6AZDR0dHM7rB0NHR0YxuMK4ghBDr1NdmQojbnHzupyu6ls7lhf5Y9QpECDEIeFxKeX01jjFKKW2VvJ8tpfR1hj6d/2/v/n1risM4jr8/DBikohGx6VCxsTCRdPAHCEM3YzEQFpM/oEktVjFYyoitWLBoKqqtxGoxEIOgbYjWY3ieqydNq8eP/kj6eSXf9Nyeb7/3njs8Pefc53nuxuUzjE1E0nRtDgLHJU1Uz4etkoYkPa++CWdrfp+y78dtMvEHSfckvag+EQP1u0FgR6033HyuyjIcqp4SryT1N9Z+rIV+GMOVUYmkQUmv67VcW8v3yFawVhmHHus/gOn62Udlf9bjAeBqbW8jsy97at4M0NOY28ki3EFmQHY3117iuU4Dj8h+JnvJlOt9tfYnso5hC/CMzKDdTZZhd85+d633++axMHyGYZC1BGckTZBl+t1Ab+0bi4g3jbkXJU0Co2QBUy+/dwy4ExHzEfEeeAIcaaz9NiJ+kOn1+4HPwFfgpqRTwOw/H539Nw4YBllzcCEiDtfoiYiHtW/m16S893GCbBRzCHhJ1lCstPZyvjW258nGNHPAUbIS+CQw8kdHYqvKAWNz+kK2Fux4AJyvkn0kHahmNot1AR8jYlbSQbLtW8f3zt8v8hTor/ske8gWc2PLvbDqMdIV+UXel8gCNdsgVvXb223DmgLm6tLiFnCdvBwYrxuPH8j/7ouNAOckTZH3GUYb+24AU5LGI0vpO+6SrewmyQrfKxHxrgLOUnYC9yVtJ89OLv/dIdpq8MeqZtaaL0nMrDUHDDNrzQHDzFpzwDCz1hwwzKw1Bwwza80Bw8xa+wkIlAZs3iqOWAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Alpha Monte-Carlo\n",
    "for alpha in [0.1,0.05,0.01]:\n",
    "    lerror=[]\n",
    "    ntrial = 1000\n",
    "    V = np.zeros((3,4))\n",
    "    for i in range(ntrial):\n",
    "        lS,lr,la = gen_trial()\n",
    "        visited = []\n",
    "        for i,s in enumerate(lS):\n",
    "            if s not in visited:\n",
    "                visited.append(s)\n",
    "                V[s]=V[s]+alpha*(sum_returns(lr,i)-V[s])\n",
    "        lerror.append(np.sum(np.abs(TrueV - V)))\n",
    "\n",
    "    plt.plot(lerror)\n",
    "    print(V,alpha)\n",
    "plt.grid()\n",
    "plt.ylabel('sum of errors for all states')\n",
    "plt.xlabel('iterations')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.86262647 0.93885098 0.99414437 0.        ]\n",
      " [0.8109944  0.         0.92847537 0.        ]\n",
      " [0.77620545 0.72279576 0.7296817  0.13456901]] 0.2\n",
      "[[0.85969565 0.91700798 0.96883275 0.        ]\n",
      " [0.79002454 0.         0.82873825 0.        ]\n",
      " [0.72090565 0.68222598 0.63657478 0.50871852]] 0.1\n",
      "[[0.8636855  0.92365425 0.97220719 0.        ]\n",
      " [0.80101746 0.         0.74523917 0.        ]\n",
      " [0.7388866  0.68653012 0.64284457 0.35195923]] 0.05\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQwAAADUCAYAAABplh1lAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXd4VNXWh9816b0nlECAUKRJCb0JgqjYEbErdlGvetWrgnrVq/daP3u7Vrw2VEQFVKSGJjX03ksSQmjpdTL7++OcNEgyJyGTxn6f5zynzD7n/ObMzJpd1l5LlFJoNBqNFWz1LUCj0TQetMHQaDSW0QZDo9FYRhsMjUZjGW0wNBqNZbTB0Gg0lnGZwRCRTiKyvsySISIPu+p+Go3G9Uhd+GGIiBuQBPRXSh1w+Q01Go1LqKsmyUhgjzYWGk3jpq4MxnXAd3V0L41G4yJc3iQREU8gGeiqlDpSwet3A3cD+Pj4xLVq1crpNR0OBzZbw+2vbej6QGusDRq6PrCucefOnceUUhFOCyqlXLoAVwBzrJSNi4tTVli4cKGlcvVFQ9enlNZYGzR0fUpZ1wisURZ+o3VhHq9HN0c0miaBSw2GiPgCFwDTXXkfjUZTN7i78uJKqRwgzJX30Gg0dUfD7rHRaDQNikZlMDILMnlh+Qtsy91W31I0mrOSRmUwfBwOZu78id0n4+tbikZzVuLSPozaxi7edM3NYVeRdhjVaOqDRlXDcHP3IDZPOGDLJqcwp77laDRnHY3KYHi42YjJ88MhsC51XX3L0WjOOhqVwQAILQjDphQJRxLqW4pGc9bR6AxGji2K9gV2Nh3bVN9SNJqzjkZnMNI9I+mRn8fmY5twKEd9y9FozioancHI9Izi3PwCsgqz2Zu2t77laDRnFY3OYOT5RNEzLx+AdUd1x6dGU5c0PoPh25wYu51QNx/Wp66vbzkazVmFU4MhIg+JSKAYfCYia0VkdF2IqwiHXyQOJZzrEcLGoxvrS4ZGc1ZipYZxu1IqAxgNRAC3AS+7VFUV+Pp4k0oIPZQn+zP2k5aXVl9SNJqzDisGQ8z1GOALpdSGMseqPlEkWESmich2EdkmIgNrKrQYf093klQ45+blAbDxmK5laDR1hRWDkSAiczAMxp8iEgBYHc98G5itlDoH6AGc8TRTf293DqtQuqSl4iZuuh9Do6lDrEw+uwPoCexVSuWISBhGs6RKRCQQGAZMAFBKFQAFNZdq4Odl1DD8MtfRMXY4G45uONNLajQai1gxGAroAlwK/AvwA7wtnNcOOAp8ISI9gATgIaVUdtlCZaOGR0VFER8fX+VFD6TYyVJhSFE+kXmBLD+5jvkL5+MmbhYk1Q1ZWVlO30d9ozWeOQ1dH7hAo7MowcCHwPvANnM/BFht4bw+gB0j2xkYzZMXqjrHStTwxTtT1Z2TnlPq2UA1c/W7qtuUbmrb8W2WIiPXFU0pmnR90tA1NnR9StVP1PD+Sqn7gTzTwJwEPC2clwgkKqVWmvvTgN7WzFjl+Hm5k6zCAejpHgDAhlTdLNFo6gIrBqPQzI2qAEQkAgudnkqpFOCQiHQyD40EttZUaDEBXu4kq1AAWuZmE+YdpvsxNJo6wkofxjvAz0CkiPwbGAc8Y/H6fwO+MbOf7cVCZ6kz/L3dOUkAdps37hlJ9IjoQcKRBBzKgU0aneOqRtOocGowlFLfiEgCRg1BgCuVUpaGR5VS6zH6MmoNPy93QMjyjiI4/RAjeo9lwaEFrE9dT++oM27xaDSaKrDiGv6VUmq7Uup9pdR7SqltIvJVXYirCD9Pw8alezaHtIOMaj0Kd5s7ixIX1ZckjeaswUodvmvZHbM/I841cpzjZhO83OCYRzM4eQB/T3+6hHXRDlwaTR1QqcEQkUkikgmcKyIZIpJp7qcCv9aZwgrwcRdS3ZpB7gnIy6BnRE82H9tMQdEZ+4VpNJoqqNRgKKVeUkoFAK8ppQKVUgHmEqaUmlSHGk/D2x12u7U3dg6uoH/z/hQ4Cvhx54/1KUujafI4bZIopSaJSIiI9BORYcVLXYirDB83Ybe0MnbSDjC05VA6hXRi7oG59SlLo2nyWOn0vBNYDPwJPG+un3OtrKrxdofDhf5gc4eMZESEQS0GseHoBnLtufUpTaNp0ljp9HwI6AscUEqNAHphzBGpN7zdhYx8BwS0gIxkAPo064PdYWfTUR1NXKNxFVYMRp5SKg9ARLyUUtuBTk7OcSk+7kJmnh3CYiHJyE/SK7IXbuLGypSVTs7WaDQ1xYrBSBSRYOAXYK6I/Aoku1ZW1TTzE5LScslr0Q+O74LCXAI8A+gW3o0VySvqU5pG06Sx0ul5lVIqTSn1HIZL+GfAFa4WVhVh3kbAr/RAs6KzfxkA/Zr1Y9OxTRzKPFRf0jSaJo0lT8/ibaXUIqXUDOBzl6pygq+HYTCOhQ8wDhz8C4DLYi9DoVievLy+pGk0TZpG5+kJRh8GQLrDC1rGwV7DLbxNYBsifSJZk7KmPuVpNE2W6nh6ZjQcT09jnZFnh8gucGwX2AsQEeKaxbH6yOriID4ajaYWqY6nZ2B1PT1FZL+IbBKR9SJSa3/7vmYNIzOvENoOg/x0OGIMp/Zt1pdjucc4kHGgtm6n0WhMrDRJZomIH4CI3CQib4hITDXuMUIp1VMpVWvT3H1KDIYdmp1rHDy+B4C+UX0BWH1kdW3dTqPRmFgxGB8COWYg38eBA8D/XKrKCd5mkyQzzw6hbUFscGwnADGBMYT7hLM6RRsMjaa2sWIw7GaQ0CuAt5VSbwMBFq+vgDkikmBGB68V3G2Cj4cbWfmF4O4FkV1LHLhEhL5Rffkr+S8Kiwpr65YajQZrIfoyRWQScBMwzBwl8bB4/cFKqWQRicRw+tqulFpctkB10wyAETrdy2Zjx75DxMen0lmFEpS4mRXmuVHZUaTnp/PRnx/R3be7Ram1x1kZft4FNHSNDV0f1E+agWbAI8BQc781cIuVkOSnXOc54LGqylhJM6CUETr9/NcXqvu+TjAOLPi3Us8GKVWYp5RSKs+ep3r9r5d6bdVrlq5X2zSl8PP1SUPX2ND1KVUPaQaUUilKqTeUUkvM/YNKKad9GCLiZ6ZVxOw0HQ1srolRq4gAbw8y8swmR2g7QEHaQQC83LzoE9WHeQfn4VBWszpqNBpnuDLMdhSwVEQ2AKuA35RSs2vr4gHe7kanJ0BIW2N9Ym/J65fFXkZSVhLrUtfV1i01mrMeK30YNUIptRcjAbNLCPB2JznNjH0R2s5Yn9hX8vrI1iPxcffhk42fEHdBvTqmajRNhkabyCPAy4OsfLOG4RcOnv5wstRg+Hr4Mqr1KJYlLyMxM7GeVGo0TYuqXMM3icjGCpZNIrKxLkVWRLkmiYjRLCnTJAG4s/udAPyV/Bfp+el1LVGjaXJU1SS5tM5U1IAAbw9yCoqwFzlwd7MZDlyp5fMrtQ1qS6RPJC+seIEXVrzAgmsWEOEbUU+KNZrGT1VzSQ5UtdSlyIrwN909S5oloW3h5H5wFJWUERFigkq92Dcfq7VBGo3mrKSqJklmmVmqZZdMEcmoS5EVEWAajHIjJY5CyEgqV+7Gc24s2dYjJhrNmVFVDSNAlc5SLbsEKKUC61JkRQSeajBKRkrK92OMjBnJ1Eun0sKvBUuSltSlRI2myWF5lEREIkWkdfHiSlFWCPA2vNMzS5y3TF+MJW9AXvkOzq5hXbmh8w3sTttNcla9hiPVaBo1VkL0XS4iu4B9wCJgP/CHi3U5JdA0GCdzzPSIgS2N9b5FMOWS08oPizZyL7219q060afRNEWs1DBeAAYAO5VSbYGRwDKXqrJAhyh/3GzCpiSzNmFzM3wxAFI2gb18ntU2gW0Y2Hwgcw/MJacwp47VajRNAysGo1ApdRywiYhNKbUQ6OliXU7x9nAjxNeDE9llprCHtCndPqXzU0SY0HUCdoedtalr60akRtPEsGIw0kTEHyNd4jci8jZgd60sawT7epKeW6Ym0a9MyI30Q3BwZUkkLoCekT1xt7nr4DoaTQ2xYjCuAHKAvwOzgT3AZa4UZZVgHw9Olq1hxN0K9y41ttMOwuej4d3eJS/7evjSPby7jiqu0dQQK9Pbs5VSDqWUXSn1pVLqHbOJUu8E+3qSlntKVK3wToAYTlwV0CeqD1uObyGrIMvl+jSapkajnXwGEOzrQVpO+c5N3D0hvAMsfq30mD2/ZLN/8/4UqSIu+fkSZu6ZWUdKNZqmgcsNhoi4icg6EZlV29cO9vEgLaeCuJ1xE8rvp5fOVu3brC/tgtpxIu8Ek5dOZtvx8vNPNBpN5VRpMMwf+9dneI+HAJf8KoN8PMgtLKLAfkpUrdDY8vt7F5Zs2sTG430fL9nXIyYajXWqNBhKqSIgQkQ8a3JxEYkGLgE+rcn5zgjyNZy30k/txwg+xRG1zEgJwOCWg9l06yaa+TVjfep6V0jTaJokViJu7QeWicgMILv4oFLqDQvnvoWRy8RqWoJqEeRjGIyMvEIiArxKXyieVwJGJ2h6xdnce0b0ZG3qWpRSiIgrJGo0TQorBiPZXGxU44cvIpcCqUqpBBEZXkW5GqUZiI+P58BRwx1k4bKVHAp2K1em+IYninxxT9zG2gquG5QZRGpOKt/P+55mHs0svCtrnJXh511AQ9fY0PVBPaQZUKVpAgIA/2qUfwlIxKihpGD4cnxd1TnVSTOglFIJB06omCdmqQXbj5xeaNsspVZ9qtSsR5T6d0uliopOK5Kcmay6TemmpmyeYum+VmlK4efrk4ausaHrU6oe0gyISDcRWYeRImCLmcWsqwVDNEkpFa2UagNcByxQSt1UI6tWCcUT0DJO7cMAOOcS6HsHRHaGgkzIPHxakeb+zWkf3J4liXrau0ZjBSvDqh8DjyilYpRSMcCjwCeulWWN4iA62flFlRcKamWsT5lbUsyw6GEkHEnQjlwajQWsGAw/ZUw4A0ApFQ/4VecmSql4pVStxwj18yoO01dFDtWgaGOdXnHk8KEth2JXdpYmLy1uSmk0mkqwYjD2isgzItLGXJ7GiI1R7/h6uCFSJupWRRTHyajEYPSM7EmIVwj/WPQP+n7TVwfY0WiqwIrBuB2IAKabSzhwmytFWcVmE8L9vTiSkVd5Ie8gI05GJU0Sd5s7t3S9BYD8onwu/OlCjuUec4VcjabRU1UQ4K/MzVuUUg8qpXqby8NKqZN1pM8pMaG+HDheRUAcEcOR6xTnrbJcEXtFuf1vtn1TW/I0miZFVTWMOBGJAW4XkRARCS271JVAZ7QO8+XgCScRtKL7wqGV5VIQlCXCN4JnBz7LY30eA9DNEo2mEqpy3PoII/5FOyABKOsKqczj9U6LIB9SM/NxOBQ2WyXemm2GwNovYf230PvmCouM6zgOgA1HN+h0BBpNJVSVZuAdpVRn4HOlVDulVNsyS4MwFgDh/p4UOVRpMOCK6HiRsU7Z5PR6cVFxHM4+rGsZGk0FWAmgM7EuhNSUiABvAI5lVWEwvAMhsmulc0rK0ieqDwDTdk4jsyBTD7VqNGVo1AF0wKhhABzLyq+6YEAUZB1xer0OIR0Y3GIwn2z6hEHfDeLpZU/XhkyNpknQ6A1G8SzVlPQqhlYB/KMgK9Xp9Wxi4/XzXi/Zn7Fnxhnp02iaElbmkviJiM3c7mgmNvJwvTRrRIf4YhM4cDy76oL+kUaTpKgKr9Diop7+jO84vmQ/JTvlTGVqNE0CKzWMxYC3iLQE5mM4bU1xpajq4OluI9jXk+PZVfRhgFHDAHiljaXrPjXgKe459x4AlicvPwOFGk3TwYrBEKVUDjAWeFcpdRXQxbWyqkeAtztZ+U5SpRS7iBdkQW6a02vaxMZ9Pe8j1DuU+EPxZy5So2kCWDIYIjIQuBH4zTxmJfBOneHv5c6OlMyqC4V3LN0+bC0sn01sXN3hahYcWsCWY1vOQKFG0zSwYjAeAiYBPyultohIO2Chk3PqFIeCHUecGIzIzjD4IWP7iPUf/+3dbsfd5s7UHVPPQKFG0zRwGjUcuEwpdblS6hUApdRepdSDzi4sIt4iskpENojIFhF5vpY0n8aozpEoxenRw8sLggv+BX4RcGSr5Wv7e/ozPHo4v+z+hQ1HN9SCWo2m8WIlanhcDa+dD5yvlOqBkbz5IhEZUMNrVUnx0GpmnvMREMI7wom91br+swOfJcAjgP9t+V9N5Gk0TQYrfRHrzIjhP1I+avj0qk4y4wQWh7HyMBeXuE0Wh+o7mVNImL9X1YX9oyz3YRQT7B3MZbGXMW3nNE7mnSTEO6SmUjWaRo0VgxEKHAfOL3NMYcTGqBKzSZMAtAfeV0qtrKBMjaOGF3MszZiFOm3+CgY0r/otxaYX0iItmSXVjKQcUxBDgaOAd+e+y7CAYdXS1xDRGs+chq4P6jFq+JksQDBGR2m3qspVN2p4MfmFRarbP2erHs//6fzkJW8q9WygUrnplu5VljE/jVH3zr232voaIlrjmdPQ9SlVP1HDo0XkZxFJFZEjIvKTmdGsOkYpDYgHLqq+SXOOp7uNIF8jz6rD4aTVE97BWO+t/kDPsOhhrE5ZTa49twYqNZrGj5Vh1S+AGUALoCUw0zxWJSISISLB5rYPMArYXnOpVTNhUBsAMp05cLUZYqxP7q/2PYZGDyW/KJ85++dU+1yNpilgxWBEKKW+UErZzWUKRoxPZzQHForIRmA1MFcpVesZ3IspTpuYXlE297J4B4FXYKVBgauiX7N+tPBrwcw9M2siUaNp9FgxGMdE5CYzk7ubiNyE0QlaJUqpjUqpXkqpc5VS3ZRS/zpzuZUTE2ZkPticnO68cFA0HN1R7Xu429wZ0nIIK1NWcijDeWwNjaapYTVq+HiMdIeHgXHmsQZFXEwI/l7urNjr1JYZzZIDy8DuJIZGBVzV4SoAZu+fXe1zNZrGjhVPz6uV4ekZoZSKVEpdqZQ6UEf6LONmE8L9Pfnf8gO8OXdn1YVbDwCHHY5Wv0ulW3g3ekT04OttX2N3OOkv0WiaGFY8Pa+oqkxDorgf4+35u6ouGNXdWKdsrtF9xncaz4m8E6xPrZ4DmEbT2LHSJFkmIu+JyFAR6V28uFxZDfByd7NWMCwWvIJgxQdQg5id57c6H3ebO3MOGKMlSsf91JwlWPH0HGSuy3ZaKsp7fjYIBrQLZdX+E4DxIxapJO2AzQ3O+wfMedqYVxIWW637+Hv6c0HrC5ixZwbebt58seULVt24Ch93nzN9CxpNg8ZZH4YN+FApNeKUpcEZC4C2EaU5ojNynfQvNO9hFqw4haIzbut2GzmFOXyxxXBJ2Xrc+gxYjaax4qwPwwE8UEdazphgH8+S7RNV5SmB0ghchzfW6F6dwzpz17l3lezrqFyaswErfRhzReQxEWnVEFMllqVLi8CS7RPZToZMQ9tBzBBY9KqlaOIV8UDPB3iy35MA/LTzJ3IKnaRs1GgaOVb9MO7HCAacYC5rXCmqpkQFejP9PqPL5US2E49PERjzGuSnw9Zfa3Q/EeHGzjfy6ehPySzM5I2EN2p0HY2msWAl81nbCpYGkyrxVKICjUxoRzMtOGVFdjbcxE/1x1DKGHJ1VBHBqwz9m/fn8tjL+XnXz+QXVd8ZTKNpLFiZreorIk+LyMfmfgcRudT10mpG80Bvgnw8rLmIi0BEJ9g6o3xm9+R18NFgmH6X5ehco2NGU+AoYO6BuWQWOYkvqtE0UqzOVi2gdHg1EXjRZYrOEJtNaBnswxFnmdCK6XwZZKfCtDLe7tlHjfXmafBOL8jPqvjcMsRFxeHt5s2kJZOYnDhZe4FqmiRWDEasUupVoBBAKZULVOLg0DAID/Bynmu1mL53Guutv8BJ0+M9/5Qawsl9Ti/j7+nPFe1LnWJn7Z2lO0E1TQ4rBqPAjGehAEQkFiPAb5WYoyoLRWSbGTX8oTPUaplwf082JKbzjx8tRPn29IMLXjC2E6YY61MNxqpPLN33knaXlGw/s+wZ/rbgb5bO02gaC1YMxrPAbKCViHyDkS7xcQvn2YFHlVKdgQHA/SJSJxnTIsxAwD8mJGIvstBxOfABCO8ES9+AnBNGdjSAJw9C3ARY+yWkbHJ6mV6Rvfh97O/08u0FwKqUVew66WRei0bTiLAySjIXI03iBOA7oI9SKt7CeYeVUmvN7UxgG0bELpcTXiZy+IZE52kRsdlgsJlqJf5lw2UcwNMfhj5mbG92GvMYgFYBrbgg8IKS/Xvn3YtDWRtt0WgaOlZqGCiljiulflNKzVJKHavuTUSkDdALOC1quCsIDyj1+Fy+x0J8DIC2ZiTwVf8tPWZzg+BWRi6TY06mzJehlVcrNt6ykRcGv0BqTioJRxIsn6vRNGRcniNVRPyBn4CHlVIZFbx+xmkGTuXg0dIRiu+X76abzcJ8EeVgoGcIXgUnSw4V36ObCsL70CbWWAzXnpWVxaJFi7AV2fASL15e9DIPRD5Q+WS4euCsDJFfyzR0fVCHaQaAtlbCjle1YCQv+hN4xEr5mqYZOJVdRzJUzBOzVMwTs1S7Sb9ZuqZSSql5zxspCIqXYmZPNvaLiqqt762Et1S3Kd3UTb/dpFKzU5VSSqXnp6tXVr2iHot/TOXb863rq0WaUoj8+qKh61OqbtMMTAMQkfk1MURi/J1+BmxTStWpz3T7yAA+uqk3V/eOxtv99Lf4n9+30ebJ304/MaJz6faTB0u3w9ob6+XvVVvLfT3u4/G+j7Pp2CZu+P0GcgpzeGD+A3y19Stm75/N1O06ybOm8VCVwbCJyLNARxF55NTFwrUHAzcD54vIenMZUyuqLXBRt+a0i/Aju6CItFNmrn682PDezD41JUHbYdAyDib8bkQXL6bH9ca6Gv0YxXi4eXBzl5t5a8RbpGSn8OiiR1mXuq7k9em7rHWmajQNgaoMxnVAHkY/R0AFS5UopZYqpUQZUcN7msvvtSHaKiM7RwIwdXX5CN+hfkanaHGwnRICouCuBdBmcPnjHt5G/IzUbTXWcl70ecQGxbI0aSkALw5+kcn9J7MnfQ970vbU+LoaTV1SqcFQSu1QSr0C3K6Uev7UpQ411phzmgUSE+bLpsTy80raR/oDcDSjGhPFWg8y5pgU1czlW0R4+/y3S/avaH8FI1uPRBDm7J/DvvR97D65u0bX1mjqCiujJH+JyBtAcQbiRcC/lFIWZnfVP2F+nqTllm+SeJn9GtkF1fjxR54Dqgj+ryPcuxQCW1RbS0xgDG8NfwubGPeP9I1kUItBfLDhAz7Y8AEAc8fNpZlfs2pfW6OpC6z4YXwOZGLkJhkPZGAhVWJDIcjHg/Tc8rExPNyMt51TUFTRKRXT+XLociXkHDeCB9eQkTEjGdF6RMn+vwb/C28375L9X3b/UuNrazSuxurks2eVUnvN5XmgwcbDOJVmQd7sOpJFbgXGIctZHtay+IbC+C8hui/smldr+iJ9I3mi3xO424zK3rfbvuX11a+X9HVoNA0JKwYjV0SGFO+IyGCg0aQvH9w+nHy7g9lbDgNGYJ0F242QfHuPOp+2fhqdLoaj22qUm7UyxnUcx7qb1/HMgGc4mX+SL7d+ycR5E7XR0DQ4rBiMe4H3RWS/iOwH3gPucamqWqR/2zA83Wz8sSkFgKS0UltXdtsysSON9fbaH/AZ32k8t3a5FU+bMYozcd5EDmXqHK6ahoOVyWcblFI9gHOBc5WRYLlmobbrgYgAL/q0CeF4ttHxmV9Y2jQ56SzuZ0U07wE+IXCkZlnTnPFY38dIuDmBKRdNAeCOP+/QiZ81DQZLk88AlFIZqoK5II0BX083Eg6cZNbGZPLsxszRnq2CSc3Mq14/Bhhh/aK6GVPeMw67QK1B78je3Nb1Ng5nH+aBBQ9QUOQkbYJGUwdYNhiNmZsHtgHgldnbS4IDX9mzBYVFiqW7jlb/gr1vMdbHdtSSwtMRER7p8wj9m/dnb/pevtjcaAamNE2Ys8JgnNcxgkGxYRw6kctjZhSubi0N1+/DVmN/lqVlHACFB1fT79/zWLD9SK1pPZX3R75P59DOTNkyhR0nXGegNBorWIka7iYil4vIg9WcS9KgeOD89uX2owK9EYGTOTXoxwhtBwEtyD+YQGpmPi/MqrnLuDO83LyY3H8yCsWE2RPKpTFIzEzkxRUvkp7fKHzoNE0AKzWMmRjRtsKoxlyShsag2HCeGlM6G9XX041Ab4/TJqZZQgSi4/A9GE8UJ3C4OHt7z8iePNH3CbIKs0hIMYLx2B12Lp5+Md/v+J4759ypjYamTrBiMKKVUmNN561GNZfkVKJDSrOr+3i6EernyYnsig1G/I5UPoyvYlLYiKex2XNZ6f0AgY6TlZerJUa3GY2fhx+Tl05mceLicgGGd57cyRW/XMH+9P18svET3ln7zmlhAY/mHCUlO8XlOhsrq/efYMaG5PqW0eCxMpfkDxEZrZSa43I1LmZox4iSbW93N4J9PZi18TDv3XB62Yemric9t5ALukSVTFYrR+Q5HI8eSVjifG7N+oLEkxcRHeLrMu1+Hn68OuxV7p9/P/fPv7/k+KejP8Xb3Zubfr+Jy365rOT4oBaDyCzIZECLAaw7so575hmuM++PfJ9h0cNOu/7ZzlM/b2LnkSx6tQqmVajrPsfGjpUaxgrgZxHJFZEMEckUEafDqyLyuYikiohrHBZqgL9XqX202aRkv8B+epDeEF8PAHanVp7F7N7CR/mzqA+jbAk8NHV9Las9nWHRw/jp8p+I8o0C4Ml+T9K/eX96RPTghcEv4ONeWoO67c/beHDhgwydOrTEWADcP/9+UnNqlny6KbPziOH1m1wTZ76zCCsG4/+AgYCvUipQKRWglAp0dhIwBbjoTMS5mjHdmwOUuIqXxdvDDYCM3Mr9NDo2C+QvR1eCJRv/pCUVzlepbTqGdGTeNfOYN24e4zuNLzl+Zfsr+eKiL7i6w9Vc2q40k2XZTtKn+j8FwD+X/ZMcRw4fb/yY9amuN3SNgeJwq2m5NegEP4uw0iTZBWw24/5ZRim12IwW3qCY/fDQkn+Tnq2CAbj36wT2vTSmXJBer2KDkVf5F8jHw41f3M7nGfdpXFO0gLnbbubyHtWf9l4TovyiTjvWNayI657WAAAgAElEQVQrXQd1RSnF/IPzybXnsvbmtbiJW8mU+l0nd/HDzh9YxjIwHUjX3rwWD5tHnehuqPh4uJFTUER6TiGTf95Ez+hgxvdtVd+yGhzizA6IyBSM2al/UCbjmZU4nabBmKWU6lZFmbJRw+OmTnUe4zIrKwt//wr6FWrAk0tySMlWfDjKFx/3UoPxn5W57Dzp4IpYD67q4Fnhuf/bms+qw3YWhb8KJ/Zyn99bTOwXVqv6aopDOchx5ODvdrqObbnbmHtyLrsKjSRLnuJJC48W3Bp+K+Ee4XUttVLq6jkWORR3zc3BoeDaTp58v8PoCJ9ykV+D0HcmWNU4YsSIBKVUH2flrNQw9pmLp7nUKkqpj4GPAfr06aOGDx/u9Jz4+HislLPCpOAkHpq6nsyg9lxc5h/lrS3L4GQaIVEtGT68a4Xn/nFsI/5pRwkb9TD8OIGxGf+jZ9y3rE9YXWv6XMFwhtM5vjNDhg1h+q7pvLDiBfYX7OeVI6+w+qbV9S2vhJp8zkopXp+zg8HtwxkUa834vTt/Fw5lxGsNa94KdhijY87uXZvfQ1dR2xqtTD47LTxfYx1WrYjhHY24n/E7y/dj5JsdoTM3JLPajP352I8b+PdvW4tTKJBvL8LLwwZdr8Ih7lzrHo/7H4/Wofozw93mzvhO43k0ztCcV5THRxs+snx+SnZKg5tN+9/Fe3l/4R7+87t1Z7pfywynVjbMXsxfu49ZT47VBLHi6blQRBacutSFuLogyNeD4Z0iOHiifKb1ArvRgXk8u4BrPloOwLSERD5Zso/4Hcb8k3y7oyTc39Ze/wTAf+t39Fs5ETIbj8/DhG4TWHvzWnpG9OT99e+TmFlxrI8P1n9QrpN07IyxjJk+hmp2bwHw/sLd3PDJihqdWxXFHdibkzKYlmAtZknb8NKmR1lfjIry8t7w6Uqud4HuxoKVUZLHgH+YyzPAemCNs5NE5DtgOdBJRBJF5I4zEepKWof6siMlk8IiB0lpuaTnFFJQ5KBFUGnovLJfkH3HsgHDYHiaBiOt843cUWD8U/vmJhs5WhsRHjYPXh72Mh42DybOm0iRo/yIT0FRAR9u+JCb/7iZozlH2XFiB5kFxpDzjb/fWLJtldf+3MFfe46TeLJ2hzHzywyRF88bqor/LtrD3K1HaBPmS7NA73JhGw+n51VqGP7Y7PwPoT6Nir3Iweak9FrXYKVJklBmWaaUegTob+G865VSzZVSHkqpaKXUZ7Wi2AV0axlEYZFiY2Ia13z4Fz3+NYfs/CKGdojgmrhoAF76Y3tJ+eJqa769CC93YzSlY5Q/8fThyy6fkhHQAfbMh0OrINdCMugGQkv/loztMJb9GfuJT4wv99qJvNKUDOf/eD7jZo4r2d90bBPXzLymWpPjimtmxca3tsgvLOLG9gUsDPgnt/j8BelJkLKp0vL/W34AgP3Hc7ioWzPC/T25bXAbAIa+upCZGysOYfDzuqrTbx46kcPFby/hvm/qJ6/uvG1HuPTdpfy5v2ZR7ivDSpMktMwSLiIXAk0qrHWxJ2dGrp1kc/bqiewCPN1tXNnLSDhfnPwIKBOMp7RJEhnoTfeWQczNaM2x8H6QdhA+uwBeiYGcE8TvSCWvsPb9NIoctfsP8njfxwF4edXLJQmXCh2FPLPsGQAuanMRN5xT6hq74JoFjO84nqSsJMbNHMeXW7609K9W7DTnrM+guhTYHXS076Rt4W7+pd5DvRsHHw2pNHZJ/7ahANw6MIZnL+vC6qdG8cgFHbm+X2sAdh8prTk5HKpE97bDVfsuzt92hO0pmfxuRnrjwHLYv+xM355lsvKN71p0QO3m87XSJEnAaIIkYDQxHgUabPOiJgR6Gz4IGXmF+Jj+FwCe7jYGtw9n4WPDy5X/btVBnvp5EwVFpQYDICrQi51HMtnVcixcX2Z4+NW2tPpmGF98+QnYa+8HMmdLCrGTf6/Vf2lPN09eGfoKDuXglj9uYdrOaQz+bjArDq8AYEDQbUw452HeGP4Gj/V5jAjfCJ4Z+Ay/XvkrsUGxvL7mdS75+RL2pu+t8PpKKfIKiwjyMZ75/uO1XMOwO/CjNGSB2M0mz8oPKyxfUOSgXbgfz1/RDRFBRAjw9uClsd0BeGfBbg4ez8Fe5KDd5N/JyrdjE0g8mUtmQeWGMbewtGmklIIvL4MpYyDLefwVh0Nx8HiO03JVUdz/0sK/diNYWGmStFVKtTPXHZRSo5VSTSo6bYS/FzaBzUnlZ3ymZBhfvLbhftzYv3W5175ZeZDjWQUlTRKAC7s2IzUzn7kHHEaw4Jt+AtNdO9Z2mImJT8CLEfBKG5h6I3xzDUy/u8a6Ew4Yk94+Xly7mdPGtBvDd5d8h7vNneeXP0+u+aPLOTiBR6fuY+BLCxjVehS3dr215Jx2Qe345pJvuL/n/RzKPMS7a9+t8NpvzN3JOc/MLtmfs6V2YolsSkwnY/cK7sn/ghaFB08vsOJDIxFVGQqLHMzaeJjKfvbFnaFvztvJg1NLz+1uxlKZe6Byp76ytcncwiJwmGWXOk8zPGdrCsNeW8iXf+0/7bWdRzK5/9u1JYGgnN3fw1bHNQwRuUZEAsztp0Vkuoj0rlUV9UyQrwf92oayct8JCoocjDJTLO5IKa2Ohvl7ATAuLprXxp0LGEGEyyZDGts7mlA/T5Ym2yksckD7UfB0Ci/2Wc5H9lJ3bXJPwvZZsGsObPwetpdPDG0vcliq1kcEGJq+W3WIR36oHRfvjLxCcguKCPeO4Nrof+ONMWHPkR+Jr6NTSbnkCgIP+Xn4cW+Pe5nQdQLzDs7jmcUvkW/Px+6wsyRxCQsOLuDXDUafwV6zVrT1cAb7z6CGlFtQxN+/X89l7y1l5pRXuUXNYNDxaQAcVkZzo6j3BCgqgK2/AsY//rSERPr/x8gzXlkNbebfhtAuwo+f1yWVNi2Azs0D6RDpT8KRivsHcguKyLOXGow168t8Nrucz+E8YmbkW7r7GJ8v3cejn/5B7JMzuOXzVYx+czG/bTxM33/Pq7JmWdz561HLIbKsOG49o5T60Uw1cCHwOvAhFjo+GxNxMSG8v9D4p46N9GfetlQGx4aVvH59v1bMWJ/EHUPack6zAN6at4uktFz2Hi3/oV0TF81/F+/l3fm7eGS08QPLtzt42X4Dr9vH40ce+Xgw78EBREdFwGvtjX+/cy4BDGPR/qk/uKxHC969vtdpOpVS3P/tWtqG+5FaJtXj9LVJ3DW0HZ2bO5/ms+doFg8uyOGiYxsY0SmSfcezWbAtle0pmRXEOH2UuDa+NA8M4skHzyElPY9xHy3ngW/X8s51vSqc2XnPufewMTmZX/Z9y0/b53J1pwv4Zf+3APhHxsDJW8Hhy8guAcTvX8+C7V24fUhbp7pPpcDu4J6vE1hihln0E6MmlBXVB/9ulzA7ow+Zyz9j2Dl/p+e+RahVn/KeYxzfrDlSUnsEiJUkWPgf6HoVRJbGTPH3cue6vq1IXvULf/NbwJqOD3PP3AIKixTDO0UwZVkWDofCVuZffOeRTEa/ubhkP9jXA9v8Mm5Lx3cbEefPqTwvefFkyLlbj7Bu607WeE8kzv18Ju+8s1y5pbuOUuRwkG93EOrnSfOg0smH9Wkwik3lJcCHSqlfReS52pVR/1zUtXmJwQj09mDl5JElSZsBmgf5EP+P0oxlH90Ux/R1idxixgst5qFRHfjv4r3M2nS4xGAUVw8v7dmatQfTOHgih80nhMhIN5KaX0zbfd/Bc0HsHP0lo2cYbfuZG5Lx93LjsnNbMKh9qcfiQ1PXl/u3A1g+6XwGvrSAi99ewqSLz+Ge82IrfI+H03N55Y/t/LLe8DX4YU0iP6yp3FdhYLsw3ry2J80KE2HOU/C/7TRvO4Lpnkvonbobx9sCouDGadDhgpLz/D39OXnwKoLSvMhrObPEWDzY8xHeXvcmfrFvcEXrO/H224Cv+o4Pti9lbJ+PCfY25va8kfAGW49vZZSMIrswGz+P8i7a09cm8uT0TQjGD+OVq7tzbd/W8JzRGes/0ag5DDuaxcglmSyPT+HmsFsZc/I5/or/nRRHV/rEhNA7ysbVAzsTs+p5WPQZLHoFhj4GI58hqyALDzcP7h4WCykbYOtShka0wY0LeSjkL9Z4xmFzFHDwRA5tzKbLyewCfl1ffvTklk4OhmwzDUjPm2DDtzD1Bvj7ZgiKrvC555epnfxzRBgshxvcF3D9C9PZdjiTUD9PBrw0n2d+3VLuvL5tQsjItbOjTEetWy03SawYjCQR+S8wCnhFRLxogrFAu0cHMaJTBAt3HKVL80CiAr2dlu8eHXTacV9Pdy5q40F8Uql/Qb7dQZswX966rhfHs/KJe3Ee87elMvnnzYTm9GKe13cA7PrjfeDhkvO+W3WI+B1HuWlADJ2bB9CrVUiFQV6aBXrz+jU9eOzHDbz0x3bG9o4uaa4Y9y9i6CsLSTXbvT4ebtzZzR2viBg2JKZzVa+WXNS1GSJG8GGlFI7CPNzWfw2z/q9cNdrt5Bf0Nj/9VIJpxknYVMZgLHkD1n3NjBOG8d2b7M7VLZsTpdz47LfW5BTeS3ibmcxMfqtUn8c2xky/hMkDJpGclVwS8HglK3njhzfxyutJcmJ3Xr30So5m5fOyOcTdIdKfu4a1Y3yfVlBBE65NmB+ebjY2J6XzdH4zxnjDwz6z+eCh+wgJ9If/tISNpcmskv3CsC1/m8AOozh/yUPkF+UzLHoYr+QexxfwTVzMnvMCYNmHtAbGemNk6bn2a+h8Gef/X3y5kI8dIv0Ze/L9UkGXvgHdx8FXV8LSt+CS10/TXPx9cbMJW/91IV7Ja4yhBkAOb6BLi54opTivYwSLdpbvQF29vzSQk5tNuGVgDFCDINdVYMVgjMeYpv66UipNRJpjOHE1OT68KY69R7Pp0sLK7P3K8feEvEIHXy3fz7V9W5NXWOqvUVxr+dH0QmzRsgttk77mdY+PuNptKZ3D3uV48/N4fH0E3WUvzbJOsHVuBB84epCNUeWcevcA8u0Obv18FWD8yMfFRdM61Jfx/13OO/N38cKVpfP9vl5xkNTMfHq3DmZUlyhu6Nea9av+YvjwDhXqlxN7cXuvDygHBMdA+wtgxCQjc/26r6DP7fxxvBkTv13HXM9/0N6ey+fz1nNd6lv47fy55Dq5bgGMz3yVp/d+QZx7Evfk7uTj/nl0uPA7/r7sSZYlLSPUoyWHE8+FqD+YtGRSybmdAgax92AgWT4byPX+C982f/Hk3KMUpvXFI3Qp949oR67jOEszfiJw/xgubD4UARj5z5JruNmEyWPO4bmZW4FAfvAax/j8aXBwDpxzKRSUGovjvsFcEhWIXfnBQqMjOtQ7lPhD8VyohLe9vOiddrB0tCX2fOx7FgFFqMVvkRk9iha5O8miFYW489Ud/RgUG87RH6ZDChR6BuHh7sWhkP6EtBuD/+pP4NxroVXf056/8X2xGd+ZMhr5+DxoMxS5dSZf3t6v5PCOlEz+b84O7hjSlh6tgvFws5XULOLj4yv8jGuKU4OhlMoBppfZPwy4LiFHPeLt4XbGxgIgytf4C37m1y2E+XuRb3fgbTYmRYSLuzXD7lBMHtO51C05rSd8PJx2actpl7achV6nX9ehhCXBl9OvzZgKhyP7tQ3lwq5RfLXiAA+O7EBEgBe3T1nNgu2peHvY+GnioHJT+E+jqBA2TIUZDxj74Z3gvhVgK1OhbG10XY2ILCImzJecTC9Sty7lGjUXP8khu/vNvOt5J9OXbebO4Z24tMCX/FV+tHEk8YfXJGzrFYTBR0NeYccbsXiqZP7MDaeTPZAHWxq+DV3dH2BlQjCeRZ6MdvflpqDlvB5+nO3Np+PbbDoOgc+2lkpaeGih8Q/WtjWfF2VS9ic4Li6a95euxC/gMP2vnwTvzeDI/kVMzdrJbTYhsMvVsHkaIyIDUaqIXsqLY/YsDnl48K53J/b3uZT/WzyZW1tEMTg3n5dSjzKj0zAGj36Nx/6YzJ7Cbfz76DYu+79m/GZ+ZoekOdHhs5H0HIJz9gGwvMd/KNh6hDv/t4aBtl585/k7asoYvhq5kst7RhPs68mv65P4x7SNFNgd+Hmao28F5uc87B+w+DXYvwT2LoTY80veY6dmAXx8i9OJprWClRqGppr0iXLjp4mDuPrDv9ieksmGxDQGtC3tQP3wprjTTwpuBfcth7x0WPJ/kHUEYgaTd2ANnid3YvPwwXZkM+fZ/wKbEF6myUHCl8Y5D6zm3Ohg/txyhHEf/cW3dw0omVsx629DKjcWx3bDtNsgxUxoJ27Q8SIY+3F5Y1EGbw83vr1rAKlv2IjiOPl4cEvBEyxe3QNIQiSEQd07Ehvhz1db20LeQpTN3RhePL4bso/RqcCovt/rPhMKYPEBG/9jEG/mtORy23Le8X7PuFk2fJhr49ugAFI9fclShdyWnkGSTyAF9lw+CQ7koIfR93P7gZ/gy5/4z5D/0K9ZP5YkLSGv2b/JAx5atIhf2g1n9s6f+PR4CEdDQ3ixzRDyhzyM+tPo/3hp8Iu0+OZa9nq4E7vve849uo92qUf4LLY380hmZJs2FObt5fUZV5U8i6ciwnghPIznj6QxO8CT7V7C6JWvcNOhbTRLWsecojieXt+M4wk/4B29lLWFwcwNuo4LDk7F8fsT3LXhIS7oFsV/fttpPv4sQjp+wQPzZzFUeXOFCN49bzTy4bzTC2ZPhvMeh9gRRha+ishIhtSteNZycGhtMFyAiBAXE4K/lzvfrz5IWk4h1/azEIzFP9JYriqdMVquJ2XWIyVDgwFe7nT3z6DAJxLmPAP56fDLRO4cNonc7Oa8t/Qwg1825gj+/uBQ2kdWEOjd4YD5z8Gyt0uP9b8Xhk8Cn2CnclsG+3DSTUDB44V34Wg3Eo99x7lveHvuGNq2xCGu+zVP8+ri/tw3djT+31wC+RnG0DLA6BfJDWzHO9v8eWLL5TzEUiZ6LcdTikpep989hCev5cHk9UYfQNoByDpKjw4XwBuduTLxMPlBrVnQ7SJ+VWksO7yCyUsnl9MaGxTLnvQ9pF89l99+vhIc2fwa4E/Ovml09jRGhv4e93dadrgIYoYQe8B0NUpcRTfgzc538plk8NZao+/Fx92HXHsul8fczA8rM6DZrzzRrPSZfXlkKYvdFHe36s6TbiewZ32Cb0BptegRgLatuSQrgeOed/H+bi+ad+hBh2Y2UlPXkFQEixKTWAT4+/pwiac/+EeQOHAir+z6jtGz72dkh6vwGPMmHh6lfx6pOakkpu1jz6/3MCZ1P6EdHwCudvpZWkUbDBfSPMibXalZtAr1IS6mkn+C6uDpa1RRC3KQWX9npn0q9vARsM/8F9n8E16bf+IxoLv3ABYXdmFQjB9d0oqgxeXlLuWTkwTfXWt0aMYMhhGTjXVVTZYKiLnlI5JT9vJkl8vKDeuVZWBsGANjrzP3BLbNNBaxQceL8QlvzxPd4M/C57lw57OGsWg9kCWt/8bQQcZwM60HGAuAX5k4Fz4hkHUEL59gLr7gNS4G0vPTWZS4iNdWv0aePY/zWp1H36i+vLjyRYb8VDqaMyEjm29th5m77h0AxnU058cMexS+Wmrk0fWPgp43QNeruAMI8Q5hxp4ZfDDyAz6e+zF3D76bBcuXk7KzG16RfxAYtoOXU5KY4+vNb37eTCIdATxMY9Enqg8++f1YkvYBAL/5F48AFYFtPRuOFuGwCQFFDr45nMLl0S14IzSY35c/Q1J2CnvS94CfL/F+vpC2BL7tQ//m/TmUcYjD2YdRxW5ovuDe93oi3Wu3qaINhgu5bXBbDp7I4YHz25cLQFxjPP3BngsfDoST+wFw37fQeK39KOh8Gaz4CI5u40JWcKHHCkgGfgBaD4Jzr4EtP8O+xaVONC16w62zKm16OCOgbR8C2lbjSxl5DqRuATdPuGk6hJcmmLrwur/BSg8jUVS7ERQtW+H8em2HwdHtUFjqSh3kFcTlsZdzeWypkSwoKmB/xn6+3vY1ABN7TOS+nvdxa+4xftz5I3aHnUBPs/+qzTCImwD97oao8sGTxnYYy9gOYwHo6dsTXw9ffn1gMP3+PZ+8w+P5ZEw/hiZPoeei/3DILYLmoR0YGfc3VhxeTr4jn0n9JhHkFQRMNJJsCzgObyBm5j8gMxkH4HD3xnbdt9i+HkvLQjtJHu74ZiXR0r8lvaN6MyoijnVpO/h10xcctsG+4zuIcfOhm3cLYg9vpWNBAX+PiuCfR5fyzxbnU5s4DdFXl/Tp00etWeN05nyDj3TkMn1rvyrtkOx3j9ERtvRNaDMYWg8E39DSsoc3wuJXjcTRu+dBYvlIWpn+sQSMexda9QO3OoznmZcO9nyj6eUES88xN83oDOw+Dlqc7uh2KgVFBaw4vIKekT1LDUQNKatv+tpEHvlhA4v+MZyYMD9IOwQJX0Cf2yv1tyhHYa7RNPQONs5x9wSHg5z3epOJIurBCqbqz/1n+eYkGH8AN07jotk3kpSVxITwCTx6ifOgTiJiKUQfSimXLRjDsTuA3cCTzsrHxcUpKyxcuNBSufrCZfocDqUOrVZq83RjuzqkJSq18GXj3LREFT9/jms01iKN7XMutBfV/k2KipQqyKn4NYdDqRX/Veq5EKU+HKJU2qGSl5Iyk9Tmo5stP0NgjbLwm3ZZk0RE3ID3gQuARGC1iMxQSm2t+kxNpYhAdB9jqS5BLWH4EyW7yrarFoVpANzdXODPaLOBreK+IUSg/91GH4unX7n+pxb+LWjh34J44mtXTq1erTz9gN1Kqb1KqQJgKnCFC++n0ZydePlXu7O6prisD0NExgEXKaXuNPdvBvorpR44pVy9phlwBQ1dH2iNtUFD1we1n2bAlf0X1wCfltm/GXi3qnN0H0bdoTWeOQ1dn1LWNWKxD8OVTZJEoKy3UjTGIJ9Go2mkuNJgrAY6iEhbEfEErgNmuPB+Go3GxbhslEQpZReRB4A/ATfgc6XUFienaTSaBkyDctwSkaPAAQtFw4FjLpZzJjR0faA11gYNXR9Y1xijlIpwVqhBGQyriMgaZaVHt55o6PpAa6wNGro+qH2NTS5ylkajcR3aYGg0Gss0VoPxcX0LcEJD1wdaY23Q0PVBLWtslH0YGo2mfmisNQyNRlMPNCqDISIXicgOEdktIk/Wo45WIrJQRLaJyBYRecg8Hioic0Vkl7kOMY+LiLxj6t5YV5njRMRNRNaJyCxzv62IrDT1fW861CEiXub+bvP1NnWkL1hEponIdvNZDmxIz1BE/m5+vptF5DsR8a7vZygin4tIqohsLnOs2s9MRG41y+8SkVsruleFWPEfbwgLhvPXHqAd4AlsALrUk5bmQG9zOwDYCXQBXsWM+wE8Cbxibo8B/gAEGACsrCOdjwDfArPM/R+A68ztj4CJ5vZ9wEfm9nXA93Wk70vgTnPbEwhuKM8QaAnsA3zKPLsJ9f0MgWFAb2BzmWPVemZAKLDXXIeY2yGW7l8XX4xaelADgT/L7E8CJtW3LlPLrxhxP3YAzc1jzYEd5vZ/gevLlC8p50JN0cB84HxglvmlOQa4n/o8MbxxB5rb7mY5cbG+QPMHKaccbxDP0DQYh8wflbv5DC9sCM8QaHOKwajWMwOuB/5b5ni5clUtjalJUvwBFpNoHqtXzKpnL2AlEKWMvC2Y6+I4dPWh/S3gccBh7ocBaUqp4uSpZTWU6DNfTzfLu5J2GGm5vjCbTZ+KiB8N5BkqpZIw8ggfxMjDkw4k0LCeYTHVfWY1fpaNyWBUFCGkXod4RMQf+Al4WCmVUVXRCo65TLuIXAqkKqUSLGqoj2frjlG1/lAp1QvIxqhOV0ZdP8MQjIBPbYEWgB9wcRUaGtz3k8o11VhrYzIYDWq6vIh4YBiLb5RSxZnhjpipJDHXqebxutY+GLhcRPZjRDo7H6PGESwixRMOy2oo0We+HgSccKG+4nsmKqVWmvvTMAxIQ3mGo4B9SqmjSqlCjOx/g2hYz7CY6j6zGj/LxmQwGsx0eRER4DNgm1LqjTIvzQCKe5xvxejbKD5+i9lrPQBIL65CugKl1CSlVLRSqg3Gc1qglLoRWAiMq0Rfse5xZnmX/jsqpVKAQyLSyTw0EthKA3mGGE2RASLia37exfoazDMsQ3Wf2Z/AaBEJMWtSo81jznFlx5YLOnvGYIxI7AGeqkcdQzCqcBuB9eYyBqPNOh/YZa5DzfKCERB5D7AJ6FOHWodTOkrSDliFEcX9R8DLPO5t7u82X29XR9p6AmvM5/gLRo99g3mGwPPAdmAz8BXgVd/PEPgOo0+lEKOmcEdNnhlwu6l1N3Cb1ftrT0+NRmOZxtQk0Wg09Yw2GBqNxjLaYGg0Gstog6HRaCyjDYZGo7GMNhhnESLyl7luIyI31PK1J1d0L03TQg+rnoWIyHDgMaXUpdU4x00pVVTF61lKqYadN1BzxugaxlmEiGSZmy8DQ0VkvRnzwU1EXhOR1WbchHvM8sPFiPvxLYbjDyLyi4gkmHEi7jaPvQz4mNf7puy9TC/D18yYEptE5Noy146X0ngY35gelYjIyyKy1dTyel0+I40T6srjUC/1vwBZ5no4pvenuX838LS57YXhfdnWLJcNtC1TttiL0AfDAzKs7LUruNfVwFyMeCZRGC7Xzc1rp2PMY7AByzE8aEMxpmEX136D6/u56aV00TUMDRhzCW4RkfUY0/TDgA7ma6uUUvvKlH1QRDYAKzAmMHWgaoYA3ymlipRSR4BFQN8y105USjkw3OvbABlAHvCpiIwFcs743WlqDW0wNGDMOfibUqqnubRVSs0xX8suKWT0fYzCCBTTA1iHMYfC2bUrI7/MdhFGYBo70A9jJvCVwOxqvRONS9EG4+wkEyO0YDF/AhPNKfuISEczmM2pBAEnlVI5InIORti3YgqLzz+FxcC1Zj9JBEaIudRFpqsAAACjSURBVFWVCTNjjAQppX4HHsaYoKZpILgsGbOmQbMRsJtNiynA2xjNgbVmx+NRjH/3U5kN3CsiGzH6GVaUee1jYKOIrFXGVPpifsYIZbcBY4bv40qpFNPgVEQA8KuIeGPUTv5es7eocQV6WFWj0VhGN0k0Go1ltMHQaDSW0QZDo9FYRhsMjUZjGW0wNBqNZbTB0Gg0ltEGQ6PRWEYbDI1GY5n/B4WdYm+RB4v0AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Alpha Temporal Differences\n",
    "for alpha in [0.2,0.1,0.05]:\n",
    "    lerror=[]\n",
    "    ntrial = 1000\n",
    "    V = np.zeros((3,4))\n",
    "    for i in range(ntrial):\n",
    "        lS,lr,la = gen_trial()\n",
    "        for i,s in enumerate(lS):\n",
    "            visited.append(s)\n",
    "            if i+1<len(lS):\n",
    "                V[s]=V[s]+alpha*(lr[i]+gamma*V[lS[i+1]]-V[s])\n",
    "            else:\n",
    "                V[s]=V[s]+alpha*(lr[i]-V[s])\n",
    "        lerror.append(np.sum(np.abs(TrueV - V)))\n",
    "\n",
    "    plt.plot(lerror)\n",
    "    print(V,alpha)\n",
    "plt.grid()\n",
    "plt.ylabel('sum of errors for all states')\n",
    "plt.xlabel('iterations')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}