{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Policy Evaluation\n",
"\n",
"Let's try to evaluate a policy using Monte Carlo and TD(0).\n",
"\n",
"The world and the policy are the following:\n",
"\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"State Matrix:\n",
"[[ 0. 0. 0. 1.]\n",
" [ 0. -1. 0. 1.]\n",
" [ 0. 0. 0. 0.]]\n"
]
}
],
"source": [
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"\n",
"#Define the state matrix\n",
"state_matrix = np.zeros((3,4))\n",
"state_matrix[0, 3] = 1\n",
"state_matrix[1, 3] = 1\n",
"state_matrix[1, 1] = -1\n",
"print(\"State Matrix:\")\n",
"print(state_matrix)\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Reward Matrix:\n",
"[[-0.04 -0.04 -0.04 1. ]\n",
" [-0.04 -0.04 -0.04 -1. ]\n",
" [-0.04 -0.04 -0.04 -0.04]]\n"
]
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAUgAAAEECAYAAABKjq0kAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAACQFJREFUeJzt3VGIpXd5x/Hfs9kdFSzkYksqs1kii4SGQBVLEAJeBKGrluamFwbqlbAUVGJYG9Ir8b4YBb1ZTFCoRAQVRCwSaEoo2NaYRkm6FYKUZFcxlLSNChriPr2YuYjJPp53zZmc48znAwNzZv7n5eG/w3fec868Z6u7A8CrHdv0AADbSiABBgIJMBBIgIFAAgwEEmCw1kBW1dmq+mFVPV1V963z2IdJVT1YVc9V1ZObnmWbVdWNVfVIVV2sqqeq6u5Nz7SNquqNVfVvVfX9/X365KZn2mZVdV1V/XtVfXPV2rUFsqquS/K5JO9NckuSu6rqlnUd/5D5QpKzmx7i98BLSc539x8neVeSD/uZuqpfJbmju/8kyduTnK2qd214pm12d5KLSxau8wzytiRPd/ePuvvFJF9Ocucaj39odPejSZ7f9Bzbrrt/0t2P73/+s+z9UO9udqrt03t+vn/zxP6HK0CuoqpOJXl/ks8vWb/OQO4mefZlty/FDzNrUlU3JXlHkn/d7CTbaf9h4xNJnkvycHfbp6v7dJJ7k1xZsnidgayrfM1vMV6zqnpzkq8m+Vh3v7DpebZRd/+6u9+e5FSS26rq1k3PtG2q6s+TPNfd31t6n3UG8lKSG192+1SSH6/x+BxBVXUie3H8Und/bdPzbLvu/t8k/xTPcV/N7Un+oqr+K3tPAd5RVX//2+6wzkB+N8nbquqtVbWT5ANJvrHG43PEVFUleSDJxe7+1Kbn2VZV9YdVdf3+529K8p4k/7nZqbZPd/9td5/q7puy16d/7O6/+m33WVsgu/ulJB9J8u3sPZn+le5+al3HP0yq6qEk30lyc1VdqqoPbXqmLXV7kg9m7zf9E/sf79v0UFvoLUkeqaofZO9E5eHuXvknLKxW3u4M4OpcSQMwEEiAgUACDAQSYLAykC6EB46qJWeQ13whfFWdW8dwh519WsY+LWevllm6TysD+TteCO8faRn7tIx9Ws5eLbOeQCYuhAeOpmv6Q/H9y5m+nuSj3f3kK753LvtV3nnDG955w6438lnl2JUruXLM62Sr2Kfljvevc+L4ojeqOdIuX76Un//iytXeYOc3XPOVNFX1iSS/6O6/m9acPnOmT/z1h6/puEfRPad3c/8zlzc9xtazT8s98I6dvPvWz2x6jK132589m8e+/8uVgVzyKrYL4YEj6fiCNW9J8sX9/1LhWPbehMKF8MChtzKQ3f2D7L2TM8CR4plvgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgxWBrKqbqyqR6rqYlU9VVV3vx6DAWza8QVrXkpyvrsfr6o/SPK9qnq4u//jgGcD2KiVZ5Dd/ZPufnz/858luZhk96AHA9i06u7li6tuSvJoklu7+4VXfO9cknNJcvLkyXd+4rOfXd+Uh9QNOzv56YsvbnqMrWeflvujHM//XPq/TY+x9c5//ON5oZ+vVeuWPMROklTVm5N8NcnHXhnHJOnuC0kuJMnpM2f6/mcuX8O4R9M9p3djn1azT8vdd+z6fOVv/mHTYxwai17FrqoT2Yvjl7r7awc7EsB2WPIqdiV5IMnF7v7UwY8EsB2WnEHenuSDSe6oqif2P953wHMBbNzK5yC7+5+TrHwyE+CwcSUNwEAgAQYCCTAQSICBQAIMBBJgIJAAA4EEGAgkwEAgAQYCCTAQSICBQAIMBBJgIJAAA4EEGAgkwEAgAQYCCTAQSICBQAIMBBJgIJAAA4EEGAgkwEAgAQYCCTAQSICBQAIMBBJgIJAAA4EEGAgkwEAgAQYCCTAQSICBQAIMBBJgIJAAA4EEGAgkwEAgAQYCCTAQSICBQAIMBBJgIJAAA4EEGAgkwEAgAQYCCTAQSICBQAIMBBJgIJAAA4EEGAgkwEAgAQYCCTAQSICBQAIMBBJgIJAAA4EEGAgkwGBlIKvqwap6rqqefD0GAtgWS84gv5Dk7AHPAbB1Vgayux9N8vzrMAvAVjm+rgNV1bkk55Lk5MmTuff07roOfWjdsLOTe+zTSvZpuet3dnLnQ3+56TG23qPn/2XRurUFsrsvJLmQJKfPnOn7n7m8rkMfWvec3o19Ws0+LWev1sur2AADgQQYLPkzn4eSfCfJzVV1qao+dPBjAWzeyucgu/uu12MQgG3jITbAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAQCABBgIJMBBIgIFAAgwEEmAgkAADgQQYCCTAYFEgq+psVf2wqp6uqvsOeiiAbbAykFV1XZLPJXlvkluS3FVVtxz0YACbtuQM8rYkT3f3j7r7xSRfTnLnwY4FsHlLArmb5NmX3b60/zWAQ+34gjV1la/1qxZVnUtybv/mr3Lv+Sdfy2BHwUeTk0n+e9NzbDv7tJy9WuzmJYuWBPJSkhtfdvtUkh+/clF3X0hyIUmq6rHu/tMlAxxl9mkZ+7ScvVqmqh5bsm7JQ+zvJnlbVb21qnaSfCDJN17LcAC/D1aeQXb3S1X1kSTfTnJdkge7+6kDnwxgw5Y8xE53fyvJt67huBd+t3GOHPu0jH1azl4ts2ifqvtVr7cAEJcaAowEEmAgkAADgQQYCCTAQCABBgIJMPh/9EW6o+bXfk0AAAAASUVORK5CYII=\n",
"text/plain": [
"