{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Support Vector Machines on the ionosphere dataset\n",
    "#### Mario Martin\n",
    "\n",
    "We used this dataset in the Meta-methods notebook, so open it in another window to compare results with other methods."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np                     # Llibreria matemà tica\n",
    "import matplotlib.pyplot as plt        # Per mostrar plots\n",
    "import sklearn                         # Llibreia de DM\n",
    "import sklearn.datasets as ds            # Per carregar més facilment el dataset digits\n",
    "import sklearn.model_selection as cv    # Pel Cross-validation\n",
    "import sklearn.neighbors as nb           # Per fer servir el knn\n",
    "from sklearn.model_selection import cross_val_score  \n",
    "%matplotlib inline              "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np    # Numeric and matrix computation\n",
    "import pandas as pd   # Optional: good package for manipulating data \n",
    "import sklearn as sk  # Package with learning algorithms implemented\n",
    "\n",
    "# Loading the dataset.\n",
    "url = \"http://archive.ics.uci.edu/ml/machine-learning-databases/ionosphere/ionosphere.data\"\n",
    "df = pd.read_csv(url,header =None)\n",
    "\n",
    "y=df[34].values\n",
    "X=df.values[:,0:34].astype('float32')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's separate the data into training (for adjusting parameters), and validation. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "(X_train, X_test,  y_train, y_test) = cv.train_test_split(X, y, test_size=.3, stratify = y,random_state=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice the stratification according to labels *y* that we demand in the spliting of data. The ionosphere dataset is small and with strarification we ensure we obtain the same proportion of examples of each class in training and test sets.\n",
    "\n",
    "**Remember**. Data should be numerical and normalized or standarized before using an SVM. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Normalization is not strictily necessary in our dataset because almost all columns are in range -1..1 (except columns 0 and 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>24</th>\n",
       "      <th>25</th>\n",
       "      <th>26</th>\n",
       "      <th>27</th>\n",
       "      <th>28</th>\n",
       "      <th>29</th>\n",
       "      <th>30</th>\n",
       "      <th>31</th>\n",
       "      <th>32</th>\n",
       "      <th>33</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>351.000000</td>\n",
       "      <td>351.0</td>\n",
       "      <td>351.000000</td>\n",
       "      <td>351.000000</td>\n",
       "      <td>351.000000</td>\n",
       "      <td>351.000000</td>\n",
       "      <td>351.000000</td>\n",
       "      <td>351.000000</td>\n",
       "      <td>351.000000</td>\n",
       "      <td>351.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>351.000000</td>\n",
       "      <td>351.000000</td>\n",
       "      <td>351.000000</td>\n",
       "      <td>351.000000</td>\n",
       "      <td>351.000000</td>\n",
       "      <td>351.000000</td>\n",
       "      <td>351.000000</td>\n",
       "      <td>351.000000</td>\n",
       "      <td>351.000000</td>\n",
       "      <td>351.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.891738</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.641342</td>\n",
       "      <td>0.044372</td>\n",
       "      <td>0.601068</td>\n",
       "      <td>0.115889</td>\n",
       "      <td>0.550095</td>\n",
       "      <td>0.119360</td>\n",
       "      <td>0.511848</td>\n",
       "      <td>0.181345</td>\n",
       "      <td>...</td>\n",
       "      <td>0.396135</td>\n",
       "      <td>-0.071187</td>\n",
       "      <td>0.541641</td>\n",
       "      <td>-0.069538</td>\n",
       "      <td>0.378445</td>\n",
       "      <td>-0.027907</td>\n",
       "      <td>0.352514</td>\n",
       "      <td>-0.003794</td>\n",
       "      <td>0.349364</td>\n",
       "      <td>0.014480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.311155</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.497708</td>\n",
       "      <td>0.441435</td>\n",
       "      <td>0.519862</td>\n",
       "      <td>0.460810</td>\n",
       "      <td>0.492654</td>\n",
       "      <td>0.520750</td>\n",
       "      <td>0.507066</td>\n",
       "      <td>0.483851</td>\n",
       "      <td>...</td>\n",
       "      <td>0.578451</td>\n",
       "      <td>0.508495</td>\n",
       "      <td>0.516205</td>\n",
       "      <td>0.550025</td>\n",
       "      <td>0.575886</td>\n",
       "      <td>0.507974</td>\n",
       "      <td>0.571483</td>\n",
       "      <td>0.513574</td>\n",
       "      <td>0.522663</td>\n",
       "      <td>0.468337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.472135</td>\n",
       "      <td>-0.064735</td>\n",
       "      <td>0.412660</td>\n",
       "      <td>-0.024795</td>\n",
       "      <td>0.211310</td>\n",
       "      <td>-0.054840</td>\n",
       "      <td>0.087110</td>\n",
       "      <td>-0.048075</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.332390</td>\n",
       "      <td>0.286435</td>\n",
       "      <td>-0.443165</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.236885</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.242595</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.165350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.871110</td>\n",
       "      <td>0.016310</td>\n",
       "      <td>0.809200</td>\n",
       "      <td>0.022800</td>\n",
       "      <td>0.728730</td>\n",
       "      <td>0.014710</td>\n",
       "      <td>0.684210</td>\n",
       "      <td>0.018290</td>\n",
       "      <td>...</td>\n",
       "      <td>0.553890</td>\n",
       "      <td>-0.015050</td>\n",
       "      <td>0.708240</td>\n",
       "      <td>-0.017690</td>\n",
       "      <td>0.496640</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.442770</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.409560</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.194185</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.334655</td>\n",
       "      <td>0.969240</td>\n",
       "      <td>0.445675</td>\n",
       "      <td>0.953240</td>\n",
       "      <td>0.534195</td>\n",
       "      <td>...</td>\n",
       "      <td>0.905240</td>\n",
       "      <td>0.156765</td>\n",
       "      <td>0.999945</td>\n",
       "      <td>0.153535</td>\n",
       "      <td>0.883465</td>\n",
       "      <td>0.154075</td>\n",
       "      <td>0.857620</td>\n",
       "      <td>0.200120</td>\n",
       "      <td>0.813765</td>\n",
       "      <td>0.171660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               0      1           2           3           4           5   \\\n",
       "count  351.000000  351.0  351.000000  351.000000  351.000000  351.000000   \n",
       "mean     0.891738    0.0    0.641342    0.044372    0.601068    0.115889   \n",
       "std      0.311155    0.0    0.497708    0.441435    0.519862    0.460810   \n",
       "min      0.000000    0.0   -1.000000   -1.000000   -1.000000   -1.000000   \n",
       "25%      1.000000    0.0    0.472135   -0.064735    0.412660   -0.024795   \n",
       "50%      1.000000    0.0    0.871110    0.016310    0.809200    0.022800   \n",
       "75%      1.000000    0.0    1.000000    0.194185    1.000000    0.334655   \n",
       "max      1.000000    0.0    1.000000    1.000000    1.000000    1.000000   \n",
       "\n",
       "               6           7           8           9      ...              24  \\\n",
       "count  351.000000  351.000000  351.000000  351.000000     ...      351.000000   \n",
       "mean     0.550095    0.119360    0.511848    0.181345     ...        0.396135   \n",
       "std      0.492654    0.520750    0.507066    0.483851     ...        0.578451   \n",
       "min     -1.000000   -1.000000   -1.000000   -1.000000     ...       -1.000000   \n",
       "25%      0.211310   -0.054840    0.087110   -0.048075     ...        0.000000   \n",
       "50%      0.728730    0.014710    0.684210    0.018290     ...        0.553890   \n",
       "75%      0.969240    0.445675    0.953240    0.534195     ...        0.905240   \n",
       "max      1.000000    1.000000    1.000000    1.000000     ...        1.000000   \n",
       "\n",
       "               25          26          27          28          29          30  \\\n",
       "count  351.000000  351.000000  351.000000  351.000000  351.000000  351.000000   \n",
       "mean    -0.071187    0.541641   -0.069538    0.378445   -0.027907    0.352514   \n",
       "std      0.508495    0.516205    0.550025    0.575886    0.507974    0.571483   \n",
       "min     -1.000000   -1.000000   -1.000000   -1.000000   -1.000000   -1.000000   \n",
       "25%     -0.332390    0.286435   -0.443165    0.000000   -0.236885    0.000000   \n",
       "50%     -0.015050    0.708240   -0.017690    0.496640    0.000000    0.442770   \n",
       "75%      0.156765    0.999945    0.153535    0.883465    0.154075    0.857620   \n",
       "max      1.000000    1.000000    1.000000    1.000000    1.000000    1.000000   \n",
       "\n",
       "               31          32          33  \n",
       "count  351.000000  351.000000  351.000000  \n",
       "mean    -0.003794    0.349364    0.014480  \n",
       "std      0.513574    0.522663    0.468337  \n",
       "min     -1.000000   -1.000000   -1.000000  \n",
       "25%     -0.242595    0.000000   -0.165350  \n",
       "50%      0.000000    0.409560    0.000000  \n",
       "75%      0.200120    0.813765    0.171660  \n",
       "max      1.000000    1.000000    1.000000  \n",
       "\n",
       "[8 rows x 34 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "However, let's see how to do that properly:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "#scaler = StandardScaler().fit(X_train)\n",
    "scaler = MinMaxScaler(feature_range=(-1, 1)).fit(X_train)\n",
    "\n",
    "# Apply the normalization trained in training data in both training and test sets\n",
    "X_train = scaler.transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linear SVM\n",
    "\n",
    "Let's try an SVM with default parameters. Linear means that we are not using any kernel to move the data to a higher dimensional space."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix on test set:\n",
      " [[27 11]\n",
      " [ 2 66]]\n",
      "\n",
      "Accuracy on test set:  0.8773584905660378\n"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "#knc = LinearSVC() \n",
    "knc = SVC(kernel='linear')\n",
    "knc.fit(X_train, y_train)\n",
    "pred=knc.predict(X_test)\n",
    "print(\"Confusion matrix on test set:\\n\",sklearn.metrics.confusion_matrix(y_test, pred))\n",
    "print(\"\\nAccuracy on test set: \",sklearn.metrics.accuracy_score(y_test, pred))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not bad results. However, the linear SVM has parameter C that has to be adjusted. We will use *GridSearch* method to find the optimal value of C like we did in a previous notebook with the k value of the KNN algorithm.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEACAYAAABfxaZOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAH4ZJREFUeJzt3Xt0nPV95/H3V3frYlk3g+ObJF/ABBIMtsF2ceg2JCztljRNU2hKwxYwaUP2bJp2S07TbEpP2+ylp3vS0CSmsFDSQAnJUjelh6RLWIhkimUggG1MRjLGwsbWjC3rYus63/1jRvZYlqyRNdIzM8/ndY6O5/KM5qOx9Pk983ueeR5zd0REJBwKgg4gIiJzR6UvIhIiKn0RkRBR6YuIhIhKX0QkRFT6IiIhotIXEQkRlb6ISIio9EVEQkSlLyISIkVBBxivvr7eGxsbg44hIpJTdu3aFXX3hqmWy7rSb2xspK2tLegYIiI5xcwOpLOcpndEREJEpS8iEiIqfRGREFHpi4iEiEpfRCREVPoiIiGSdbtsSrjF+gbZ914vZkZDVQkNlWXMn1eEmQUdTSQvqPQlEMOjcTq6+tl7uIe97/Ww93Avew/30NU7eM6yJYUF1FWW0FBVSn1lKfVnXS49fbmhslQDhMgUVPoy6471DyXK/fCZco8c7WNoNA4kSn3lwkq2rGpgzaIqLr14PgUGXX2DdPUOEu0bSv47yJGeAd549wSx/iFG437Oc5UUFlBfWUJ9VWIQqK8spb6qJHG56uxBYn6ZBggJH5W+ZMzIaJyOaP9Z5f7mez0c6Tmz9t5QVcqaRfO5bnU9ay6ez5pF82luqKC4cHqbl+Jxp/vU8OnBIJocIFIHisMnBnjt3RMcm2yAKCpIDgwlZw0GiXcSZWcGj6pSqko1QEh+UOnLBTk+tvb+3plyf+tIH0MjibX34kJj5cIqNq+oZ82iRLlfuqiK+srSjDx/QYFRW1FCbUUJl1B13mXjcef4ySG6+gaJ9g6dHiCifWcGiUPJASLWN8gE48PpAWLzyjruuq6ZVRed/zlFspVKX85rZDTO/mj/6XLfe7iHNw/38l7PwOll6itLWbOoits3NZ6enlnRUElJUXbsHFZQYNRVllJXWQoXn3/Z0eQAkTowRHsTA8ah7lNs/+khnmjr5OcvaWDrlhVc21yrdwCSU1T6clr3yaHT0zKJtfde3jrSy2By7b2owFi5sJKNK+pOl/uaRfNpqMrM2ns2KCyw0xuIL51ggDjeP8SjLx7gkda3ufWBF/nAkmruuq6Zf3/5xRRNc4pKJAjmPsF72QCtW7fOdZTNudEaifKTSJQ3k2vxh0+cWXuvqyhJTsucKfeVC7Nn7T1oA8OjfO/lTv72hf3sj/aztHYed2xu4pPrl1JeonUpmXtmtsvd1025nEo/nP75tcN89jsvU1RgrGioTJR7cu59zaIqGipLNW2Rhnjc+dHeI2x7voNdB45TPa+Y265dzqc3NebVOyDJfip9mdRPD3bzyW/t4PLF1Tx6xwatmWbIrgPH2PZ8Bz/cc4TiwgI+vnYxd17XzMqFlUFHkxBQ6cuEDnWf4ub7WygtKuCpz27O2N40csb+aD9/+0IHT+7qZHAkzofXLGTrlhWsb6zRuyeZNSp9OUf/4Aif+OYODh47yfd/dxOrtdvhrIr2DfLojgP83Y63OX5ymCuXLuDuLc185P0XU1ig8pfMUunLWUbjzt2P7uLZN4/w0O3ruf6ShUFHCo1TQ6M8uesgf/uT/RyInWR5XTl3/lwTn7h6KfNKCoOOJ3lCpS9n+fOn97Lt+Q7uu/n9/NbGxqDjhNJo3Pnh7vf41vMdvHqwm5ryYm7b2MinNy5PfIZAZAZU+nLa4y+9w73ff51Pb1zOn9x8edBxQs/daTtwnG/9vw7+de8RSosK+MTVS7jzumaa6iuCjic5Kt3S124bea41EuVLT73BltUN/PEvXRZ0HAHMjPWNtaxvrCVytI8Hf9LBd3d18p2X3uEjl13E1i0ruHp5TdAxJU9pTT+PdXT18bH7W7hofhnf+91NzC8rDjqSTKKrd5BHWt/m0RcPcOLUMFcvr2HrlmZuWHMRBdroK2nQ9E7IHe8f4lf+poWegRH+8bObWVpbHnQkSUP/4AjfbUts9O08foqm+gruvK6JX71qCWXF2ugrk0u39NP6TL2Z3Whm+8wsYmb3TnD/MjP7sZm9YmavmdlNydsbzeyUmb2a/Prm9H8Uma6hkTi/8/e7ONQ9wLbbrlbh55CK0iJu39zEc79/PV//jbVUlRXxR//nDTZ/9Vm+9n9/xvH+oaAjSo6bck3fzAqBt4AbgE5gJ3Cru+9JWWYb8Iq7f8PMLgOedvdGM2sEfuDuaW891Jr+zLg7f/i913iirZO/+vUP8itrlwQdSWbA3Xmx4xgPvNDBs28epay4gE+uW8qdP9fMsjoN5nJGJjfkbgAi7t6R/MaPAzcDe1KWcWB+8nI1cGh6cSVTHnihgyfaOvncv1upws8DZsbGFXVsXFHHW0d6eeD5Dh576R2+/eIBbrz8YrZuWcGVSxcEHVNySDpr+p8AbnT3O5PXbwOucfd7UpZZBPwQqAEqgA+7+67kmv5uEu8UeoAvufsLEzzHVmArwLJly64+cODAzH+yEPrh7ve4+9u7uOnyRfz1rWu1ATBPHekZ4OHWt/n2iwfoHRhhQ1MtN6y5iPqqs88AVlNeok/+hkjGNuSa2a8BHx1X+hvc/XMpy/xe8nv9pZltBB4ELgeKgUp3j5nZ1cBTwPvdvWey59P0zoV5490T/No3d7D6okoe37pRn/QMgb7BEf5h50Ee+sl+3u0+dc79BQZ1lWefTL7hrNNCjl0uoaa8RCsJOS6T0zudwNKU60s4d/rmDuBGAHffYWZlQL27HwUGk7fvMrN2YDWgVs+gIz0D3PlIGwvKi3ngt9ap8EOisrSIO36uid/e3EjPwEjyLF9jJ5IfINp39hnAOrr66eodPH1C+lSFBUZdRUnyRPLJk8onTyg/fpBYMK9YA0QOS6f0dwKrzKwJeBe4BfiNccu8A/wC8LCZrQHKgC4zawCOufuomTUDq4COjKUXTg2NctfftdEzMMx3P7ORhfPLgo4kc8zMqJ5XTPW8YlY0nP8wzu5+eoA4czrIwbPPH9w3SORIL9G+ofMOEBO9Y2g4PWAkbtcAkX2mLH13HzGze4BngELgIXffbWb3AW3uvh34AvCAmX2exEbd293dzWwLcJ+ZjQCjwGfc/dis/TQhE487X/juq7z+7gm23baO97+vOuhIkuWmPUCcGkkMCMmv8ecNjvYN8rMjvXT1DTI8eu5UcVGBUVd59raGswaJlNsXlBfr0NNzQB/OymH/85l9fP3HEf7opjXctaU56DgSYqkDxNjAcHqA6BubcjpzfbIBoj45rVRfefY7hvGDRPU8DRDj6dg7ee77L3fy9R9HuGX9Uu68rinoOBJyZkZ1eTHV5cVTninM3Tlxajg5MCTfMZwzUAzx5uFeYv0TDxDFhUZdxZnB4Kx3Eqc3WCdu1wBxNpV+Dtr59jHu/d7rbFpRx59+7HL9QktOMTMWlJewoLyElVOc1mFsgOga2+6Q+o4heVtX3yB7D/cS7RtkJH7uAFFSWDBuiqmE8pIisvHPZvGCedx53ey+a1fp55h3Yie5+9FdLKmZxzc+dTXFhWkdSUMkJ6UOEKumONNbPJ76DmKCQaJvkCM9A+w+dIKTQ6Nz9BNMzweWVKv05YyegWF++5GdjMadB29fT3W5jpopMqagwKipKKGmYuoBIsy0mpgjRkbjfPbvX+btaD/f/M2rdbINEbkgWtPPEff9YA8v/CzKf/vVK9i4oi7oOCKSo7SmnwMeaX2bv9txgLu3NPPr65cFHUdEcphKP8s9t+8of/JPu7nhsov4LzdeGnQcEclxKv0stu+9Xu75zitcevF8/tevX6kjJorIjKn0s1S0b5A7HtlJeUkhD96+jopSbX4RkZlTk2ShgeFR7n50F9G+QZ64eyOLqucFHUlE8oRKP8u4O/d+7zV2HTjO33zqKj6wRGdFEpHM0fROlvn6sxGeevUQf/DRS7jpikVBxxGRPKPSzyI/eO0Qf/mjt/j42sX87vUrgo4jInlIpZ8lXj3YzRee+CnrltfwF796hQ6iJiKzQqWfBd7tPsWdj7SxcH4p37rtakqLdLpDEZkd2pAbsL7BEe54eCeDw6M8dtc11FWWBh1JRPKYSj9Ao3HnPz/+Cj872sf/vn29jgwoIrNO0zsB+uq/7OVf9x7lK7/8frasbgg6joiEgEo/II+99A4PvLCf2zc1ctu1y4OOIyIhodIPQEskyh8/9QbXX9LAl35xTdBxRCREVPpzrL2rj9/59i6aGyr461vXUqTTHYrIHFLjzKHj/UPc8fBOigsLePDT66kq0+kORWRuae+dOTI0Eucz397FoRMDPHbXNSytLQ86koiEkNb054C786WnXuff9h/jf3ziA1y9vDboSCISUir9ObDt+Q6eaOvkP/3CKm6+cnHQcUQkxFT6syzWN8h/f2YfN11xMZ//8Kqg44hIyKn0Z9mOjhijceeu65p1EDURCZxKf5a1RGJUlRZxxeLqoKOIiKj0Z1tre5Rrmuu0P76IZAU10SzqPH6SA7GTbFpRF3QUERFApT+rWttjAGxeWR9wEhGRBJX+LGqNRKmvLGX1RZVBRxERAdIsfTO70cz2mVnEzO6d4P5lZvZjM3vFzF4zs5tS7vti8nH7zOyjmQyfzdydlvYYm1bUaa8dEckaUx6GwcwKgfuBG4BOYKeZbXf3PSmLfQl4wt2/YWaXAU8DjcnLtwDvB94H/KuZrXb30Uz/INkmcrSPrt5BNq/UfL6IZI901vQ3ABF373D3IeBx4OZxyzgwP3m5GjiUvHwz8Li7D7r7fiCS/H55ryUSBWDTCs3ni0j2SKf0FwMHU653Jm9L9RXgN82sk8Ra/uem8di81NIeY1ltuQ6sJiJZJZ3Sn2hC2sddvxV42N2XADcBj5pZQZqPxcy2mlmbmbV1dXWlESm7jYzGebEjpl01RSTrpFP6ncDSlOtLODN9M+YO4AkAd98BlAH1aT4Wd9/m7uvcfV1DQ+6fK3b3oR56B0bYpF01RSTLpFP6O4FVZtZkZiUkNsxuH7fMO8AvAJjZGhKl35Vc7hYzKzWzJmAV8FKmwmerlvax+Xyt6YtIdply7x13HzGze4BngELgIXffbWb3AW3uvh34AvCAmX2exPTN7e7uwG4zewLYA4wAnw3DnjutkRiXXlxFfWVp0FFERM6S1pmz3P1pEhtoU2/7csrlPcDmSR77Z8CfzSBjThkYHmXn28f41DXLg44iInIOfSI3w15+5ziDI3Htny8iWUmln2GtkRiFBcaGJp0SUUSyj0o/w1rao3xwSTVVZcVBRxEROYdKP4N6B4Z5rfOEPoUrIllLpZ9BL+0/xmjc2aT5fBHJUir9DGqJxCgtKuCqZTVBRxERmZBKP4Na26Osb6ylrLgw6CgiIhNS6WdItG+QN9/r1dSOiGQ1lX6GnD41ojbiikgWU+lnSGskSlVZEZcvrg46iojIpFT6GdLSHuXa5joKC3RqRBHJXir9DDh47CQHj51is46qKSJZTqWfAa3JQylv1vHzRSTLqfQzoCUSY2FVKSsXVgYdRUTkvFT6M+TutLYnTo1opvl8EcluKv0ZeutIH9G+QZ0aUURygkp/hloims8Xkdyh0p+h1vYojXXlLF4wL+goIiJTUunPwMhonH/rOMZGfQpXRHKESn8GXn/3BL2DIzo1oojkDJX+DIwdb2djs0pfRHKDSn8GWiJR1iyaT11ladBRRETSotK/QAPDo7QdOK5DL4hITlHpX6BdB44zNBLXrpoiklNU+heoJRKlqMBY31QbdBQRkbSp9C9QS3uMDy5dQGVpUdBRRETSptK/ACdODfN6Z7fm80Uk56j0L8BL+48Rd3S8HRHJOSr9C9ASiVJWXMDaZQuCjiIiMi0q/QvQ2h5lfWMtpUWFQUcREZkWlf40He0d4K0jfdpVU0Rykkp/mnYkD72wWQdZE5EcpNKfppZIlOp5xVz2vvlBRxERmba0St/MbjSzfWYWMbN7J7j/r8zs1eTXW2bWnXLfaMp92zMZfq65Oy2RGNc211JYoFMjikjumfKTRWZWCNwP3AB0AjvNbLu77xlbxt0/n7L854C1Kd/ilLtfmbnIwTl47BTvdp/i7g81Bx1FROSCpLOmvwGIuHuHuw8BjwM3n2f5W4HHMhEu27S0J06NuEnz+SKSo9Ip/cXAwZTrncnbzmFmy4Em4NmUm8vMrM3MXjSzj11w0izQEoly0fxSVjRUBB1FROSCpHPgmIkmr32SZW8BnnT30ZTblrn7ITNrBp41s9fdvf2sJzDbCmwFWLZsWRqR5l487uxoj/Gh1Q2YaT5fRHJTOmv6ncDSlOtLgEOTLHsL46Z23P1Q8t8O4DnOnu8fW2abu69z93UNDQ1pRJp7+470Eusf0qEXRCSnpVP6O4FVZtZkZiUkiv2cvXDM7BKgBtiRcluNmZUmL9cDm4E94x+bC1oiY/P5OsiaiOSuKad33H3EzO4BngEKgYfcfbeZ3Qe0ufvYAHAr8Li7p079rAG+ZWZxEgPMV1P3+sklre0xmuoreN+CeUFHERG5YGkdDN7dnwaeHnfbl8dd/8oEj2sFrphBvqwwPBrn3zpifGzthNuvRURyhj6Rm4bXOk/QPzSq4+2ISM5T6aehNRLFDDY2az5fRHKbSj8NLe1RLls0n5qKkqCjiIjMiEp/CqeGRnn5QLemdkQkL6j0p9B24BhDo3HtqikieUGlP4WWSIyiAmNDU23QUUREZkylP4XW9ihrly2gvCStvVtFRLKaSv88Tpwc5o13T+iomiKSN1T65/Hi/hhxRxtxRSRvqPTPozUSZV5xIVcuXRB0FBGRjFDpn0dLe4wNTbWUFOllEpH8oDabxJGeASJH+9i8Urtqikj+UOlPolWnRhSRPKTSn0RLJMaC8mIuWzQ/6CgiIhmj0p+Au9MaibKxuY6CAp0aUUTyh0p/AgdiJzl0YkCnRhSRvKPSn0BLcj5/s463IyJ5RqU/gdZIjEXVZTTVVwQdRUQko1T648TjTmt7lE0r6jHTfL6I5BeV/jh73+vh+Mlh7Z8vInlJpT9OayQGaP98EclPKv1xWtqjNDdUcHF1WdBRREQyTqWfYng0zkv7j7FZa/kikqdU+il+erCbk0Ojms8Xkbyl0k/REolhBtc2q/RFJD+p9FO0tEe5/H3VLCgvCTqKiMisUOknnRwa4ZV3jrNJUzsiksdU+kk73z7O8KhrI66I5DWVflJrJEpxobG+sTboKCIis0aln9TSHmXtshrmlRQGHUVEZNao9IHuk0PsPtSjqR0RyXsqfeDFjhjuaP98Ecl7Kn0S++dXlBTywaULgo4iIjKrVPok5vM3NNVSXKiXQ0TyW1otZ2Y3mtk+M4uY2b0T3P9XZvZq8ustM+tOue/TZvaz5NenMxk+E947MUBHVz+bdWpEEQmBoqkWMLNC4H7gBqAT2Glm2919z9gy7v75lOU/B6xNXq4F/iuwDnBgV/KxxzP6U8xASyRxakQdSllEwiCdNf0NQMTdO9x9CHgcuPk8y98KPJa8/FHgR+5+LFn0PwJunEngTGtpj1JbUcKlF1cFHUVEZNalU/qLgYMp1zuTt53DzJYDTcCz031sENyd1kiMjc11FBTo1Igikv/SKf2J2tAnWfYW4El3H53OY81sq5m1mVlbV1dXGpEyY3+0n/d6BnS8HREJjXRKvxNYmnJ9CXBokmVv4czUTtqPdfdt7r7O3dc1NDSkESkzWtoTp0bUh7JEJCzSKf2dwCozazKzEhLFvn38QmZ2CVAD7Ei5+RngI2ZWY2Y1wEeSt2WF1kiUxQvmsbyuPOgoIiJzYsrSd/cR4B4SZb0XeMLdd5vZfWb2yymL3go87u6e8thjwJ+SGDh2AvclbwtcPO7s6IixaUUdZprPF5FwmHKXTQB3fxp4etxtXx53/SuTPPYh4KELzDdr9hzuofvksPbPF5FQCe1HUM/sn6+NuCISHuEt/fYYKxdWsnB+WdBRRETmTChLf2gkzs79x9istXwRCZlQlv6rB7s5NTzKJs3ni0jIhLL0WyJRCgyubdaavoiESyhLv7U9yhWLq6meVxx0FBGRORW60u8fHOGVd7o1tSMioRS60n/p7WOMxF2HXhCRUApd6bdGopQUFnD18pqgo4iIzLnQlX5LJMZVyxcwr6Qw6CgiInMuVKV/vH+IPYd7NLUjIqEVqtLf0ZE4lLI24opIWIWq9FsiUSpLi/jgkuqgo4iIBCJUpd/aHuOaplqKCkP1Y4uInBaa9jvUfYr90X5N7YhIqIWm9McOpbxZ58MVkRALTem3tseoqyhh9cKqoKOIiAQmFKXv7rS2R9m4oo6CAp0aUUTCKxSl397Vz5GeQZ0aUURCLxSl39qenM/Xh7JEJORCUfotkShLauaxrK486CgiIoHK+9IfjTs72mNayxcRIQSlv/vQCXoGRtikXTVFRPK/9FsiiePtbNRJ0EVE8r/0W9ujrL6okoVVZUFHEREJXF6X/uDIKDvfPsYmzeeLiAB5XvqvvNPNwHBc++eLiCTldem3RqIUGFzTXBt0FBGRrJDXpd/SHuMDSxYwv6w46CgiIlkhb0u/b3CEnx7s1lE1RURS5G3pv7Q/xkjc9aEsEZEUeVv6LZEYJUUFXLW8JugoIiJZI29Lv7U9xrrlNZQVFwYdRUQka6RV+mZ2o5ntM7OImd07yTKfNLM9ZrbbzL6Tcvuomb2a/NqeqeDnE+sbZO/hHu2qKSIyTtFUC5hZIXA/cAPQCew0s+3uvidlmVXAF4HN7n7czBamfItT7n5lhnOf146OxKEXNunQCyIiZ0lnTX8DEHH3DncfAh4Hbh63zF3A/e5+HMDdj2Y25vS0RGJUlRZxxeLqIGOIiGSddEp/MXAw5Xpn8rZUq4HVZtZiZi+a2Y0p95WZWVvy9o/NMG9aWtujXNNcR1Fh3m6yEBG5IFNO7wATnVTWJ/g+q4DrgSXAC2Z2ubt3A8vc/ZCZNQPPmtnr7t5+1hOYbQW2AixbtmyaP8LZOo+f5EDsJLdvapzR9xERyUfprAp3AktTri8BDk2wzD+6+7C77wf2kRgEcPdDyX87gOeAteOfwN23ufs6d1/X0NAw7R8iVWtkbD5fG3FFRMZLp/R3AqvMrMnMSoBbgPF74TwF/DyAmdWTmO7pMLMaMytNuX0zsIdZ1NIepb6ylNUXVc7m04iI5KQpp3fcfcTM7gGeAQqBh9x9t5ndB7S5+/bkfR8xsz3AKPAH7h4zs03At8wsTmKA+WrqXj+Z5u60tsfYtKIOs4lmpUREwi2dOX3c/Wng6XG3fTnlsgO/l/xKXaYVuGLmMdMTOdpHV++gjrcjIjKJvNq9pSUSBTSfLyIymfwq/fYYy2rLWVpbHnQUEZGslDelPzIa58WOmKZ2RETOI29K/0jvIA2VpZraERE5j7Q25OaCxQvm8ezvX09im7KIiEwkb9b0x2hXTRGRyeVd6YuIyORU+iIiIaLSFxEJEZW+iEiIqPRFREJEpS8iEiIqfRGRELFs+zCTmXUB3cCJ5E3VU1we/289EE3z6VK/Xzr3TZVFuZQrTLnOl2euc02WM0y5lrv71Gehcves+wK2pXt5gn/bLuR50rlPuZRLuc6+/zx55jTXNF6nUOQ631e2Tu/80zQuj//3Qp8nnfuUS7mUa+L7gs41/nrYc00q66Z3ZsrM2tx9XdA5xlOu6VGu6VGu6Qlzrmxd05+JbUEHmIRyTY9yTY9yTU9oc+Xdmr6IiEwuH9f0RURkEip9EZEQUemLiIRIaErfzNaY2TfN7Ekz+52g86Qys4+Z2QNm9o9m9pGg84wxs2Yze9DMngw4R4WZPZJ8jT4VZJbxsuU1Gi+Lf6ey+e+wwsx2mdkvBZ1ljJldb2YvJF+z6zPyTS9k5/65/gIeAo4Cb4y7/UZgHxAB7k3zexUAD2ZptppMZctwrieD/D8FbgP+Q/LyP2Tj79tsvEYZypWx36kM58ro32EmcgH3AX8I/FK25AI+BPwL8DCwMiPPP5s/XAZfpC3AVakvElAItAPNQAnwU+Ay4ArgB+O+FiYf88tAK/Ab2ZYt+bi/BK7KwlyzUfrTyfdF4MrkMt/Jpt+32XyNMpQrY79Tmco1G3+HGfgd+zBwC3A7s1/608lVkLz/IuDvM/H8OXFidHd/3swax928AYi4eweAmT0O3OzufwFM+PbM3bcD283sn4HvZEs2S5zY96vAv7j7y9mSazZNJx/QCSwBXmUOpiSnmW3PbOe5kFxmtpcM/05lIhewZzb+DjOQqxKoIFG0p8zsaXePB53L3cd+v44DpZl4/pwo/UksBg6mXO8Erpls4eR82MdJvHBPz2qyaWYDPkdiTaPazFa6+zezIZeZ1QF/Bqw1sy8mB4fZNFm+rwFfN7Nf5AI/ep4BE2YL4DVKKxdz9zs1rVxz/HeYdi53vwfAzG4HorNV+NPNZWYfBz4KLAC+noknyuXStwlum/STZu7+HPDcbIUZZ7rZvkai2GbbdHPFgM/MXpxzTJjP3fuB/ziHOSYyWba5fo3GmyzXXP1OTWayXM8xd3+HEznv34C7Pzx3Uc4y2ev1feD7mXyiXN57pxNYmnJ9CXAooCzjZWu2bM01JpvzZWs25Zqe0OfK5dLfCawysyYzKyGxEWZ7wJnGZGu2bM01JpvzZWs25Zoe5ZrNrdQZ3Nr9GHAYGCYxIt6RvP0m4C0SW73/SNmyP1cu5MvWbMqlXJn40gHXRERCJJend0REZJpU+iIiIaLSFxEJEZW+iEiIqPRFREJEpS8iEiIqfRGREFHpi4iEiEpfRCRE/j9StTMjVQ/lgwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc. 10-fold cross on train data=  0.8744420289855073\n",
      "\n",
      "Confusion matrix on test set:\n",
      " [[27 11]\n",
      " [ 2 66]]\n",
      "\n",
      "Accuracy on test set:  0.8773584905660378\n",
      "\n",
      "Best value of parameter C found:  {'C': 1.0}\n",
      "\n",
      "Number of supports:  72 ( 49 of them have slacks)\n",
      "Prop. of supports:  0.2938775510204082\n"
     ]
    }
   ],
   "source": [
    "# List of C values to test. We usualy test diverse orders of magnitude\n",
    "#Cs = np.logspace(-3, 11, num=15, base=10.0)\n",
    "Cs = np.logspace(-3, 5, num=9, base=10.0)\n",
    "\n",
    "param_grid = {'C': Cs}\n",
    "#grid_search = GridSearchCV(LinearSVC(), param_grid, cv=10)\n",
    "grid_search = GridSearchCV(SVC(kernel='linear'), param_grid, cv=10)\n",
    "grid_search.fit(X_train,y_train)\n",
    "\n",
    "# Let's plot the 10-fold cross.validation accuracy deppending on C\n",
    "scores = grid_search.cv_results_['mean_test_score']\n",
    "plt.semilogx(Cs,scores)\n",
    "plt.show()\n",
    "\n",
    "parval=grid_search.best_params_\n",
    "cvacc = cross_val_score(SVC(C=parval['C'],kernel='linear'), X=X_train,  y=y_train, cv=10, scoring='accuracy')\n",
    "print('Acc. 10-fold cross on train data= ', cvacc.mean())\n",
    "\n",
    "\n",
    "# Let's apply the best C parameter found to the test set\n",
    "\n",
    "#knc = LinearSVC(C=parval['C']) \n",
    "knc = SVC(C=parval['C'],kernel='linear')\n",
    "knc.fit(X_train, y_train)\n",
    "pred=knc.predict(X_test)\n",
    "print(\"\\nConfusion matrix on test set:\\n\",sklearn.metrics.confusion_matrix(y_test, pred))\n",
    "print(\"\\nAccuracy on test set: \",sklearn.metrics.accuracy_score(y_test, pred))\n",
    "print(\"\\nBest value of parameter C found: \",parval)\n",
    "print(\"\\nNumber of supports: \",np.sum(knc.n_support_), \"(\",np.sum(np.abs(knc.dual_coef_)==parval['C']) ,\"of them have slacks)\")\n",
    "print(\"Prop. of supports: \",np.sum(knc.n_support_)/X_train.shape[0])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "On this dataset, the best C for a linear SVM is 1 (that casually is also the default value for parameter C) so we don't obtain any improvement tuning the C parameter. However, in other datasets we could obtain a dramatic increase of accuracy. \n",
    "\n",
    "Let's see (just for fun) how the C parameter affects performance on training and test sets. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEhCAYAAACTNXDdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzsnXmcVMXVv5/qZbp79mERmAEEFY0sAoqouOMSFBWXJIrEmMQ3RF81v5iowcQ3yavREBONRuNC3KLhdQmaiBFBomhcMAgRXHABERGGnVl6pvfb9fuj+vY2PTM9w8wwy3k+3k/3vbfq3rrt8K2655w6pbTWCIIgCH0Dx75ugCAIgtB1iOgLgiD0IUT0BUEQ+hAi+oIgCH0IEX1BEIQ+hIi+IAhCH6JV0VdKPayU2qGU+qCZ80op9Qel1Hql1HtKqcPTzl2qlFqX2C7tyIYLgiAIbSefkf6jwLQWzp8BjEpss4H7AJRS/YBfAEcBk4FfKKUq9qaxgiAIwt7Rquhrrf8F7GmhyAzgMW14GyhXSg0Bvgos1Vrv0VrXAEtpufMQBEEQOhlXB1yjCvgybX9z4lhzx5uglJqNeUugqKjoiK985Ssd0CxBEPYJWjfd0s+19r29KNXy+dpa2LIl815KwdChUF6+9/dvLzU1TdvlcMD++0O/fnlfZtWqVbu01gNbK9cRop/rl9YtHG96UOt5wDyASZMm6ZUrV3ZAswRB6HC0BsuCWMx8RiIQDpstEkkJ1/PPwx13wNatMGQIXHcdnHtuSpiVarq1h2gUgsH8tt/8pmnnojXs2QPHH596rni86ffmtng89Vu09j3XdWOx3M8Vj5utDVqolPoin3IdIfqbgWFp+0OB6sTxk7KOv9oB9xMEoTNJFyTLSgl6KGSO2aN3pcyI1OEAlwsKC82xZ5+F//kfI7QA1dUwZ475PPro/EU6GDT3bOlcc6LZFhob4eWXU8/hdOb+7nCYfZfLbB5P0+P297Zsd96Zu12bNu39s+WgI0R/IXCVUupJjNO2Tmu9VSm1BLg1zXl7OnBDB9xPEIS9xR592sIeCqVG7baQpgu7LWoeT+Z1wmEjTp9/braNG+Gpp8zx7HK//W3LbXI4TMfh8zXdKiqgshK83tzn89nOPNN0PNlUVcGKFe3+Kfeav/7VmHeyGT68U27XqugrpZ7AjNgHKKU2YyJy3ABa6/uBRcCZwHogAHwncW6PUupm4J3EpW7SWrfkEBYEoSNJH61HoykzTDhsRD8de9RZUGCENZ1QKFPYbXH//POmtuiysqaCn85TTzUvym53+808+XDDDXD99ak3EDD3nTOn5Xr286X7J7L9Fc35L9LP57quUnDNNXDjjeZ3tikshFtuadvz5YnqbqmVxaYvCHmSbW9Ot69Ho+a8LSz2iD3dZJFOtrDbor5xY1NhLy+HkSNhxAiz2d9HjjQj8qOOyj1y7egRdbaz2H7edLG1j9lk+xp+9CM4++zU75SL9N/PLmObtezf0f6eXjb7XPq1sn0bTz0FP/85fPmlGeHfcgvMmtWmn0MptUprPanVciL6gtBNyXYa2qP1SMRslgX/+EemiF17LVxwgRH2bBELheCLL1KCni7w1dW5hT1d0NOFvSWefTb3iHruXOPMzRZpyDyW3e5cgqx1qvPK9i2kC7J9LJfjuDkBzt7vIeQr+h1h0xcEoa3YUTDpW7qgR6NNTQK2iDmdxgTz9783dZj+9KewaxcceKAR8w0bUiK/dWtuYT/qqKbi3pqwZ5MelXL66eZZsjsje0TdkkBnj5izR9kdEfHTx5GRviB0Btmml1gsU9RzRZ3YI1fbvt6SqO3aBaecYj5boqIiU9DTzTJtFfbsMMR07bCdvF5vKqrFdv6KOHcJMtIXhM6kNdNLtqM0PQrG7W7qLG2OPXvgk0/M9umnZvvkE3O8JV54wQh7WycdaZ0Z1ZNuWnG5jKO3qMi03w5dzOUjELotIvqCkA+RiInnrqvLz/TSVhGsrU0J+qefwscfm8/0kXxJCRx8MEybZj7vuSf3SL+qCiZMaP5e2ROssuPuPR5zL4/HdFC2uIuw9wpE9AWhOaJRCATMNPlwOBXSaE9Cag/19SlxTx+9b9+eKlNUZET9lFPM5yGHmM/Kysz79u/fcgiiLeq2OQZSNvVcI3Z71C70akT0BSGdWCwl9KFQ5si3LTQ0ZJpj7M+tW1NlfD4YNQpOOCEl7IccYsQ9n1H1+eebz7lzjRPXDkE8/XTzVuJyGVH3eo3Ip88mFfos8n9fEGIxM1quqTGfSjUV+mefTYlrZaUZTZ9/vukg1q3LFPZPP4XNm1N1vV446CA45phMcR82rO0mEzse37LM/umnwznnmFG77UC1R+ziQBVyINE7Qt/EsozA19WZUblSZjRcUNC0bK64c4fDOElralL2/YICEyp5yCGp7eCDzWSb9phNtDYmJnuilVJG0IuKzOZ2m3uKuAtI9I4gNCUezxR6MMLZnOkmHofVq830/XTBT7/Wj3+cEvcRI/bOdBKLGYGPxVJx6IWFpnPxeFImGkHYC0T0hd5NPG5s8/X14PebfbfbjJRzjZCjUXj7bXjxRViyBLZta/7aoZDJm9LedtlmGvtNweMxuWvsPDSdnYtG6JOI6Au9D62NIPv9ZlQfj2em/s0mGIR//QsWLYJ//tOET3q9cPLJJjxy7txMB6xNZWX+7bFH8LYt3u024l5UlDIrSUik0AWI6Au9A61NWGVDgxH6WMwIvc+XW0zr6ozAL14My5YZ4S8rg1NPNSl4TzzR1AVTvy3ZGW1na7qZxus117fNNBJBI+wj5C9P6NmEwyY8sbbWjKbtyVG2YKezfbsx2SxeDG++aUR50CD4xjfMiP6YY8wIPJvs0Mj06B2tUwJvT3IqKDB+gsJC813MNEI3QqJ3hJ6HPTu2psYIvcNhhD6Xk3PjRiPyL74Iq1YZUR4xwozmp02DiRPbZlZJd7aCuafPB8XFKTONOFuFfYBE7wi9i+zZsfakqewcNlrD2rUpof/oI3N8zBgTaXPGGSbaJt+Rdzxu7mc7XG0zjc+XMtPIKF7oQYjoC90Xe3ZsbW3zk6bACPOqVUbkFy82OeOVgiOPhF/8wozo27L0nJ0R085HU1KSykUjtnihhyN/wUL3IxAwWSQDAbNv28jTiUTgrbeM0L/0EuzYYWznxx8PV15pZqoOHJjf/WwnsG2X93hgwAAzmvd4ZCQv9CryEn2l1DTgLsAJPKi1npt1fn/gYWAgsAf4ptZ6c+KcBbyfKLpJa31OB7Vd6E1obUbzO3eacMuCAmMnTycQMJE2ixebyJv6euMsnTrVmG2mToXS0vzuF4ul4uQdDnOvkpJUAjJB6KXkszC6E/gjcBqwGXhHKbVQa702rdjvgMe01n9WSk0Ffg1ckjgX1Fq3kOdV6NPYYr9jhxltv/gi/O53qSiZq682o+3Fi+G110yHUFFhRH7aNDOyzxWpk+s+dq57MG8FFRWpOHmJkRf6CPkMaSYD67XWGwCUUk8CM4B00R8N2FMTlwF/78hGCr0Qrc3I3R7Ze72wdKlZ7s+Oh9+yJRULP2QIXHyxEfqjjspvNG4vQWhZxkRTVGTSEdszXgWhD5KP6FcBX6btbwaOyiqzBrgAYwI6DyhRSvXXWu8GvEqplUAMmKu1btIhKKVmA7MBhrfF4Sb0PNLFPhw2o3jbJPOrXzXNcQPGNv/OO63b1u2Zr/Zo3uUyeWvsDJQymheEvEQ/17+07OD+a4F7lFLfBv4FbMGIPMBwrXW1UuoA4BWl1Pta688yLqb1PGAemDj9NrRf6CnYYr9jhxFlrzflnF2zxqwClb6QSDq7djUv+OkhlWBs/P36pXLIC4KQQT6ivxkYlrY/FKhOL6C1rgbOB1BKFQMXaK3r0s6htd6glHoVmAhkiL7Qi9HaTKTascOMwm2x19rMir3nHpP3pqzMHPf7m14jO8dNJJJKN2xnySwuNqN5mRglCC2Sz/vuO8AopdRIpVQBcBGwML2AUmqAUsq+1g2YSB6UUhVKKY9dBjiWTF+A0FvR2gj4558b27zTacTZ6TSpEM4+26Q/+PhjuPFG+Pe/4dZbmzplfT74yU+M2cfvN5vDAfvtZ2bWjhxpvhcWiuALQh60OtLXWseUUlcBSzAhmw9rrT9USt0ErNRaLwROAn6tlNIY886VieqHAg8opeKYDmZuVtSP0NvQ2iQ927nTjMZ9PjO6j0ZhwQK4916zutTw4fDrXxvht2fV5lr+75prTKROcbHZmku3IAhCXkjuHaFjiMeN2O/alRJ7l8uM0J96Cu6/H778Eg491EyeOvvs3BE4wWDKDFRWlkpaJhOkBKFFJPeO0DWki30sllqIu74e/vxnePBBc27SJLj5ZpO6OFvA7Vh9yzKRPBUVTXPqCILQIYjoC+3DFvudO43Y22acXbvgT38ygu/3w0knmQlWRx3VVOztdWrBRNyUlkrEjSB0MiL6QtuIx42Y2yP7wkIj+F9+aUw4Tz5pQiinT4erroJx45peIxYzE7KcThODX1oqdnpB6CJE9IX8SBd7yzJC7/PBp5+asMu//91E1VxwAVxxBRx0UNNrhMNmKygwTtqioiYTpkKxELXBWurD9TgcDhw4cCgHTocTh3LgcrhwKEfGd6WU+UQl97O/K/EJCAIgoi+0hmWlxD4eN0LvdMJ//mPEfskSc+w734Hvf79pTL29Xq39VjBokCmfJsJxHScQDbCrcRdhK4zL4aKooMhURxPXceI6jqUtwlYYO/ggruNoNFprFCo1jVCbekqp5Henw9khHYhDyaxeoWcjoi/kxrKMM3b37pTYOxzw+utG7N9806Q4uOYa+O53jU0+HXsGbjxuonDKy5s4Z2PxGP6wnz3BPcTiMTwuDyWezBTKtuDuLVrrZjsQu+PQ9kRzTbMdiMPhoL+vP8UFxbidkr9H6HmI6AuZZIt9YaE5vnixEfs1a8xo/X/+B775zabpj23nrFImCqesrElyM9uEUxeuw6Ec+Nw+fCqPTJl7QfqIfW+w4ha7ArvY0biDEk8J5d5yfC6fmI+EHoOIvmCwxX7XLjNKLyw0x/76VzOhav16MwP2N7+Br3/dpDxIJxo1ZhyXy8yQtWffJshlwikuKO5xYul0OJOmp1AsxKa6TRQ4Cuhf2J+igiJcDvknJXRv5C+0r2NZUFdnRva22IfD8MgjJhqnutpMqLr3XhORkz2hKhw2uXA8HmPPz3LO5mPC6al4XV68Li9W3GJ7w3Y0mjJPGWXeMrwub4/r0IS+gYh+XyYQMHlxwNjs/X64+24zoWrPHpg82aREmDo1M8beds5Go8a8k8M5uy9MOPsKp8NJsacYrTWN0UZqQ7V4XB76+8zo3+mQcFSh+yCi31d55BGT6GzrVhg8GMaMgbffNhOupk41MfZHZS2bEI8bsY/HUzNn08w8vcWE016UUqZzc/uIxWNsa9wGDVDuK6fMU4bH5Wn9IoLQyYjo90UeeQT++7+NgIMR/q1b4fDDTRK0sWMzy6c7Z+2Zs2nO2d5swmkvdoentcYf9lMTrMHj9CRt/xL6KewrRPT7GqGQGeHbgp/O9u2Zgm87Z91uY8IpLs5wziYnUkXqUahebcJpL/boHyBqRan2V+NQDiq8FZR6SylwStoJoWsR0e9LhEKwaZMZ1eeiujpVzl7dqqrKOGcTJpqcE6ncRX3GhLM3uJ1u3E43cR2nNlzL7uBufG4f/X398bl9MvoXugQR/b6CLfjxuInAiUablhkyxDhzi4rMd683KfZiwuk4HMpBodvMf4hYETbXb8apnPQr7EdJQYlM+hI6FRH9vkA4bBKiuVzwgx8YwS8oSC0gDkbgb7zRxOKnOWfFhNO5FDgLKHAWENdx9gT2sLNxJ0XuIvoV9pNJX0KnIKLf27EF3+2G3/8eXngBfv5zk93y1782pp6qKrNU4SWXAGLC2Rc4lIPCAjP6D8VCfFn3JW6HmwpfBSWeEpn0JXQY8pfUm4lEjOA7nbBwIfzhDzBrFsyebcw8p51m1phN5LC3TTi7A7uxtCUmnH1E+qSv9JQPFd4KmfQl7DUi+r2VdMFfswauuw6OPRZuucXY6RsbzQzagoKcJhxxKu577JQPWmuC0SD+sB+3051M+CaTvoT2kNe/bKXUNKXUJ0qp9UqpOTnO76+Uelkp9Z5S6lWl1NC0c5cqpdYltks7svFCM0SjRvAdDti2DS67DIYOhXnzjJmnsRGrrJT6AthYs5Evar+gMdpIkbtIYsi7IXbYp23m2d6wnc/2fMb2hu2EYjlCbwWhBVod6SulnMAfgdOAzcA7SqmFWuu1acV+Bzymtf6zUmoq8GvgEqVUP+AXwCRMwtpVibo1Hf0gQgJb8JUy9vxLLzWmnD//GV1WRrCxltpwPQ1lMWhswOMUE05PwuVwJVM+NEQaTMoHp4cKX0Vy5K8SeaFtM1A++/mWFXo++Zh3JgPrtdYbAJRSTwIzgHTRHw1ck/i+DPh74vtXgaVa6z2JukuBacATe990oQm24IOJ1Pnud+Hzz4nMf4z6IWXU1m7ACjTiHjaCIm+J/EPuwWRP+treuB17OYBcawHYC82krw2QXTaffUcimZ4jYSSw/4bst8PmPs0ldHIBnFzEdTzncXudg+bqZqyFkKtu87fsdridboaVDevUe+Qj+lXAl2n7m4GspCysAS4A7gLOA0qUUv2bqVuVfQOl1GxgNsDw4cPzbbuQTiwGmzebZGg+H9ZPb8D52mvsvPkGasYOxRGpxxu2cFSOgKLSfd1aoQOxJ311BbbwZguxvW+vZmZpK6dItzbQUDR/vrm69loJLV63hwxwAtGA6aA7sb35iH6uu2f/37wWuEcp9W3gX8AWIJZnXbTW84B5AJMmTepB/XI3ISH4cStGwA3R+++k4s+Psee7Mwl+/VyKHS6TO6e0zOTNEYR2km32aUVrhW5IPqK/GUh/3xgKVKcX0FpXA+cDKKWKgQu01nVKqc3ASVl1X92L9gpZ6GiU0Bef4Q/VUeeM4nv5bapuuZ3gKScSuu4aXA6nMfs4HDBgwL5uriAI+5h8wjTeAUYppUYqpQqAi4CF6QWUUgOUShrvbgAeTnxfApyulKpQSlUApyeOCXtJxIqwp2Enn3+8nE31X+J3WpR+toXKH/+S2CGjqP3dr0y4ZjwOkbBJmOaUED9B6Ou0OtLXWseUUldhxNoJPKy1/lApdROwUmu9EDOa/7VSSmPMO1cm6u5RSt2M6TgAbrKdukLbicVjBCIB9gT3EI4GcGzbgRcn3pL+OHbvof8VP0IX+dj9wO/RRYm1bQMBM/s2a1FyQRD6Jqolb/q+YNKkSXrlypX7uhndhriOm8lToVr8YT9KKTy4cO/aDeGIWbEqHGbApVfgWvsxu+c/SHTcaFM5GDTLH+63X+bKV4IgdEsaIg2M6jeqXY5cpdQqrfWk1srJjNxuiNaasBWmIdxATaiGuI7jdrrNKlRaw7atKcHXmvKf3UzBf9aw5w+/SQl+uh1fBF8QhAQi+t2IqBWlIWKEPmpFcTqcmSkR4nEj+KGwGcEDxfc9ROHCF6m/5kpC005NlQuHzSxcseMLgpCGiP4+xopbBGNB9gT3EIwGcSgHHpcHryvLBh+Pm5Wt0gTfu+glSu+8j8CM6TRc/p1U2WDAjPDFji8IQhYi+vsArTWhWIi6UB3+iJ+4jrec0dIW/EDALHACuNd8QMVPfkn48PHU3nJjyoQTDEJhEZSVddHTCILQkxDR70LCsXAyX0osHsPlcFHoLmzZaaM17NyZIfjO6q30u+JHWAMHUHPv7cnUyMRiRvwHDuyxdvxnP3qWuW/MpdpfTWVJJXOOm8P5h56/r5slCL0GEf0uIBaPsaV+C6FYCIdy4HV5k3lTWkRr2LEDGhqSgq8aGun3/WtQoRC7H7ufeL+KVNlQyCyI4uqZ/1uf/ehZrl96PcFYEIAt/i1cv/R6ABF+QeggJIduF9AYaSQcC1PiKaGooCi/POj2CL/BnxR8LIuKa2/EtX4DNXfNJXbQAanygUbo399E9PRQ5r4xNyn4NsFYkLlvzN1HLRKE3kfPHBL2MOrCdXhcntYL2mgNu3aBvx6KipOHS3/7B7yv/IvaX/yE8PFTUuVDISP25eUd2OqORWtNXbiOLf4tbKnfQrW/mi31W8y+3+xX+6tz1t3i38Kou0dR5imj3FtOmaeMMq/Z7P2M44nPCm8FpZ7SDklGJmYnobcgot/JxOIxQrEQxQXFrReGlODX12UIfuFTz1L88F9ouORCArO+kSpvWabOfoP2qR0/YkXY6t+aFPF0Ya9uMJ+N0caMOgXOAiqLKxlSMoQpw6awZP0S/BF/k2uXekq5cMyF1IXrqAuZ7YvaL6gN11IXqmvydpBNkbso2Rmkdw7l3vKcx9M7DJfDJWYnoVchot/JhGKhFnOIZ6A17NnTRPALlq+g7H/nEjrhWOpv+FFm+WCwXXb8toxctdbsCe5Jinm2sFf7q9nRuKNJTvMBhQOoKqniwIoDOX748VSVVlFVkthKqxhQOCAj33q2uAL4XD5umXpLi+IasSKmMwjXURsyHUFtqDbZSdidg72/sXYjtaFaasO1ra48VVxQTDAaxNJWxnHb7CSiL/Q0RPQ7mfpwfX7mBVvwa2pSNnzA+fkX9Lv6emIjR1Bz562Z4h5ohH792mzHzzVyvfala/l418ccUHFAU7NLfTUhK1McvS5vUrynDphKZUklVSVVVJaazyHFQ/JzVqdhC2hbzSgFzgIGFg1kYNHANt0PTERVk84hrdOoDdXy0LsP5ay7xb+FR959hJNHnsyI8hFtvrcg7Ask904nEtdx1u9ZT5G7qPVcGnv2mK2oKGmmUTW1DPzGt1H+BnYteAxraGWqfCgEBW4YUtlms87kP01mi39Ls+cVikFFg5ICbou7LexVpVVUeCt6zMIUe0tzv5dTOZNvAAdUHMDJI05m6sipHD306KaT6wQhDyT3Tg8nHAvntwpODsEnEqXfD67HuXU7ux67P1PwLctM2BrYvkRqzQm+QvHWZW8xuHgwBc6CNl+3tzLnuDk5zU63nXYbEwZPYNnny1i2cRnz35vPQ+8+hNflZcqwKUwdMVXeAoRuh4h+J9IQacDlaOUnrqlpKvhaU/bLX+P59ypqbv8V0cPHp8rbdvzKSnC3LSolakW59Y1bmz1fWVLJ8DJZrjKb1sxOB1QcwGWHX0YwGmT55uW88vkrLPt8Ga98/gosk7cAoXsh5p1OQmvNhpoNeFyeDGdlBjU1JlKnuDhjxF700GOU/eYu/Fd+D///uzyzTmMjVFQYW34b2OLfwhX/uIJVW1dx/LDjWbl1Zc6RqzgmO44NNRuSbwHLv1xOyArJW4DQImLe6cFErAhW3Gpe8GtrYffuJoLv/eerlN72B4Jnno7/B9/PrBMKgcfT5nj8Vz5/hR+8+AMiVoR7p9/LjENmSNx5F3BAxQHyFiB0O2Sk30nUBGvYFdhFUUFR05O1tWaEX1hoct4ncK39mAEzLyM26kB2/WVeZpbMuGUybA4blrdZJxaP8bu3fsfdK+7m0AGH8sDZD3BgxYF7+2hCByBvAUIuumKkL6LfSWys2YjD4Whq06+vN/l0sgTfsX0nA7/2LbRDsWvBY8QHpi1irrXJv1NZmRHO2RLbG7Zz5aIrWb55OTPHzuTmk29ucwil0DVkvwVsrNsIyFtAX6TbmHeUUtOAuzBr5D6otZ6bdX448GegPFFmjtZ6kVJqBPAR8Emi6Nta6ywjde8jakUJW2FKXFmpkv3+nIKvgkH6XfEjVEMDu594OFPwwThu+/XLW/Df2PQGVy26ioZIA3dOu5Ovj/763j6S0In43D6mjpzK1JFTgcy3gNYigsRMJ7SVVkVfKeUE/gicBmwG3lFKLdRar00rdiPwtNb6PqXUaGARMCJx7jOt9YSObXb3JhgNNu2p/X7Yts0Id5rgE49Tfv0vcH/4EXvuv4PYV0Zl1guHjTmnoqLV+8Z1nLv+fRd3LL+DAyoO4KmvPcUhAw7pgCcSupJ8fQHDSoexfPNyIlYEkPQQQn7kM9KfDKzXWm8AUEo9CcwA0kVfA6WJ72VA7sxZfYT6cD1uR5rdPRyGHdubCj5Qcud9+Ja8TN0NPyJ88gmZF4pbJkf+sGFN6mWzO7Cbq1+8mte+eI3zvnIevzn1N7n9CUKPoqW3gGUblzUpH4wF+ck/f8KHOz7MSEpX7inPSEZX5inLL9trO5E3kO5LPqJfBXyZtr8ZOCqrzC+Bl5RSVwNFwKlp50Yqpd4F6oEbtdavZ99AKTUbmA0wfHjPjhO34haBaCBTcBsbweFsIty+v/2DkvsfpvHC82n89sWZF9IaAkEYPDi1SEozrNiygiteuIKaYA2/OfU3zBo3q8/Mlu1rpL8FDL1jaJN8RwCBaIBHVz/aJHVGNqWe0iaZScs95alEdGnJ6NIT0pV6SpuPSkPWReju5CP6udQj+y9tJvCo1vp2pdQxwONKqbHAVmC41nq3UuoI4O9KqTFa6/qMi2k9D5gHxpHb5qfoRoStMJq0WbhaG+etJzO1csE771J+482Ejz6Sup9f33RmbSBgljwsbj47Z1zHeWDlA/z6jV8zrHQYC2cuZOx+Yzv6kYRuSmVJZc7Z1VUlVaz43orkkpzpeYTSM5XaSefsZHTrdq9LlrFNRrlQKNNheJumtC73lvP4e4/nXBfh1tdv5YyDzsDr8sqgZB+Sj+hvBoal7Q+lqfnmMmAagNZ6uVLKCwzQWu8Awonjq5RSnwEHAz0/PKcZ/GF/ZsROOGzSJqSN8p2bvqTiyh9jDa1iz923NQ3BjETM6L5//2bvUxOs4Zol17B0w1LOHHUmt59+O6We0mbLC72P5tJDzDluDmCS4nmLvQwqHtSm69prOGd0FlkJ6bKPV/urk8di8VjO625t2MpBdx+Ey+GipKCEEk8JJQUllHpKKS4opsRTQmlBKcWe4oxPu1yJJ61sQUm7zFPd1eyU3q5hZcO49ZRbmTVuVqfcKx/RfwcYpZQaCWwBLgKybBFsAk4BHlVKHQp4gZ1KqYHAHq21pZQ6ABgFbOiw1ncztNbUh+szQyMDgcxInXoC5JUIAAAgAElEQVS/We4Q2PXAneiyLKGOWxCNwLDhzdrx3936Lpe/cDnbG7Zz88k3850J35GRUx+kvVlJW0Mphc/tw+f2MaRkSJvqaq2Z/ODknAvilHvKueLIK/CH/fgjfvxhP/WRehrCDWzxb8G/K3U8O5V1LorcRanOI71jaKbDWL1tNQ/+50HCVhgwZqfrXrqOmmAN0w6a1qbn7EgWr1/Mra/fmjTHbarbxOznZwN0ivDnFaevlDoTuBMTjvmw1voWpdRNwEqt9cJExM6fgGKM6ed6rfVLSqkLgJuAGGABv9BaP9/SvXpynH4oFmJT3abUgilawxdfmFG7wwGxGP2+9wM8K1ax+5F7iUw+oulFGhpg0CAoKWlySmvNI6sf4abXbmJQ8SDun34/E4dM7OSnEoS20dy6CPmm+dBaE4wFMzoHf8RPfbiehkgD9eH6JucyvodN2dYW1+nu7F+2Pxt/uDHv8h0ap6+1XoQJw0w/9vO072uBY3PUewZ4Jp979AYC0UCmgyvdtKM1Zb/6Ld43/03Nr3+RW/BtO34Owa8P13PtS9fywroXOPWAU7nzq3dS4Ws9jFMQupq9fQNRSlHoLqTQXcgg2maaSicWj2V0Cl99/Ks5Hd8At59+e7vvs7f8+KUf5zy+qW5Tp9xPcu90ILWh2syUxIEAvhf/Sck983BWb0MBwZOPJ3jBOU0rRyJmgZQcidQ+2PEB3//H9/my7ktuPP5Gvj/p+y1GTwjCvub8Q8/f57Zyl8NFha8iOThqyfF90diLurp5Se5YfkfOdnVWxltRjg4iYkWIWtGUE1drfE8soOym23AlBB/As/wdfAtfzKwcjxs7/qBB4Ew5p7TWzH9vPuc8cQ6haIgF31jAFUdeIYIvCO1gznFz8LkyU5GkO773FbnaVegu5JZTbumU+4l6dBDBaBCVHt0aDlPyh/txhDJjpR2hECV33JNZORAwC6KkhXU2Rhr5weIfcP0/r+eooUex5JIlTK6a3JmPIAi9mvMPPZ/bTruNqpIqFIqqkqpukU48u13Dy4Yz7+x5nRa9IwnXOohNdZuI63jKvLNnD0NGHobK8ftqpdj6SeIZg0GTi2dQynb56e5Pmf38bNbvWc+Pp/yYH0z+QafOnhQEoXvQbRKuCS1jxS2C0SAlnoQDNjEhyxoyCFf1tqblhyQEPho1Tt4BqQRrC9YuYM4/51BUUMQTX3uC44cf3xWPIAhCH0HMOx1AKJY13T0RtdP4naavZ3GvF/+PrjJ2/Eg4accPRoNc99J1/L/F/48Jgyfw0jdf6nTBj+s4wWiQqBWlu73xCYLQOchIvwOoD9fjdqbNqk1MyFJB0xlY+w3EsXMX1pBB+H90FcFzzjDx+AMHgtfLZzWf8f3nv89Huz7i6slXc+2Ua1tfW3cv0VrTEGmgzFNGxIrQGG1MLuLuVE5cDhcuh0smfQlCL0NEfy+J6zgNkQYK3YXmgJ1rp6AA33OLCB8xgd1PPJRZKRQysfilpSz8ZCHXLb0Ol8PF4+c9nsym2Nn4I376+/ozsGhgotmaaDxK1IoSioUIRAMEogET16xJLgjjdrilIxCEHoyI/l4SjoWTI2RzwJh23Os/x/3Z59Te9LPMCjGTlyRcXsLNy/6HR1Y/whFDjuC+s+6jqqSqS9rcGGmkzFPGgMKUL0EpRYGzgAJnAUUFRfSnP1prYvEY0XiUcCxMY6SRYCxIPB5Ho3GoREfgdEsYqSD0EET095LGSGNmZE3CtON77gW0203wjLQs0/E4hEJsKtVcvuBrrNm+htlHzOanx/000zzUiQSjQTwuD4OKB7U6YldK4Xa6cTvdFLoLqfBVNOkIAtEAwVgQK26hlEKhpCMQhG6MiP5eYCdYS4Zp2qYdpxPfP5YQmnpCZkK1YICXAu/xwyU3otE8ePaDnDHqjC5rbzgWRqGoLKlstyDn6gjATHmPWlEiVsR0BNEgMR0zmZgUSdOQhJ4Kwr5FRH8viFgRovEoXndiweqEacfz9kqcu/cQnHEmz37+InPX3EN1YDtFrkIaYo2M228cD5z1APuX799lbY1aUWLxGPuX798pTmLb8etz+yjzlgEmlNV+IwhGg0k/AZic7E6HkwJngXQEgtCFiOjvBcFoMHPEnDTtLMIqL+P/Rvi5fsVcgomUqQ2xRpzKyXcmfqdLBd+KW4RiIYaXDc/MDdTJOB1OnA4nXpe3SUcQtaLJTiAYDZqlerSp43a6Oz16SRD6KvIvay/ISLCWMO2oaBTfP18lcME5zP3g/qTg21ja4va3bufCMRd2SRu11jRGG6kqqcrM87+PSO8I7MlscR0nakWJxqPJN4KGSEOz13AoBwqFQzlwOpw4lEP8B4KQJyL67SQWjxG2wqlZuAnTjnfpq6hwmMC506leuyBn3VwLTHQGWmv8ET+Digal2tkNcSgHHpcHD57kWgRaa+I6nnOzHcm2ySpiRZpdrQmMH8LuGNI3CT0V+iIi+u0kFAtlikbCtFP43AvE9h9GdPxYKjcOYkugaRqGypLKLmljQ6SB/r7+PTLvvj1JzEn+9v7szsGKW8lOInuztDlno7VOJsyzO4lcnYUg9HRE9NtJXagOtyMRZpkw7Tj21FDw71X4r5oNSnHRATO4/YMHMup1VSrXxkgjxQXFGbH4vZ22CrP9NmF3AOmdRXYnEbbCTZbws+cpuBwu6RCEHoOIfjuI6ziBaCA1Czdh2il84SWU1gRnmDDMzQ1bKFBuBhQPZKt/a5ctxByMBilwFjC4eLCYMFqgrW8TdieRPk8hFAsRjoWJxWNodPLtwE5lIZFJQncjL9FXSk0D7sKskfug1npu1vnhwJ+B8kSZOYklFlFK3QBchlkj9wda6yUd1/x9QygWSv4DB4xpRyl8zy0icvh4rOHDqI3U89yXS/nGIefzmzPv6LK22bH4VaVVIjgdTLKTcDgz/A9AxttBxIoQjAUJRUMmMimtvv1mIP9vhH1Fq6KvlHICfwROAzYD7yilFibWxbW5EXhaa31fYpH0RcCIxPeLgDFAJfBPpdTBWuex1H03piHSgFMl/tEmTDuuDV/gXvcZtf97AwDPbHiBkBXmkomXdlm77AlSnRWLLzSPHZXkwUMRRVRg/CjJN4PExLVQLEQoFiIYCyb9CEqZOQt2hyAInUk+f2GTgfVa6w0ASqkngRlAuuhrwJ56WgbY4SkzgCe11mHgc6XU+sT1lndA2/cJ9ixcrytzQlbhwhcTaRdOQ2vN4+sWMHHAOMYOGd8l7bJNTsPLhuNxeVqvIHQJDuVI5TSiKHk83cEctUyoaigWoiGWGaqafDNQTjHVCR1CPqJfBXyZtr8ZOCqrzC+Bl5RSVwNFgJ1wpgp4O6tuk6xiSqnZwGyA4cM7ZzHgjiJshbHiVspxFwhAPI7vH4sJnXQcuryMf+/4D+v8G7njyN90SZvsNMmVxZUpP4PQrUnvDHCTnLxm5zayzUT2m0EgFkhGG9kOa0l/LbSHfEQ/119U9oobM4FHtda3K6WOAR5XSo3Nsy5a63nAPDDLJebRpn1GxizchGnH8581OHftJjjjTAAe//SvlLlLOGfMBV3SJn/Yz37F+1HqLW29sNCtSc9t5HP7KKNpZxCLx5ImokA0kFoAR5Exac2pnDIfQWhCPqK/GRiWtj+UlPnG5jJgGoDWerlSygsMyLNuj6I2VJsynyRMO76FLxIvKyV00nHsCu3hhS9f4VtjLu6SGbANkQYqfBVUeHteLL6QP+mdAZCcbKe1xtLGiWw7kyNWJLnZUUV24jt7bQT7bcF2TAt9h3xE/x1glFJqJLAF45i9OKvMJuAU4FGl1KGAF9gJLAT+Tyl1B8aROwpY0UFt73JsZ1yJKzG7NRBABUN4ly4jeN5ZUFDA0x8uJKpjXeLADUQCFLmLGFg0UEZzfRSlFC7VsgPYiltY2kp+2n/HESti/AkJp3Lymqhkx2B3CjIPoffQquhrrWNKqauAJZhwzIe11h8qpW4CVmqtFwI/Bv6klLoGM6b4tjZ/RR8qpZ7GOH1jwJU9OXInGA0mZ23aph3va2/iCIUJzJhOXMf5y/pnOGa/Ixg18Cud3haXw8Xg4sHyD1JoEacjMRehmQG9/baQ3jlE41EisQiReIRwLDExLe1tAUXSfGSbkmTg0TPIKz4sEXO/KOvYz9O+rwWObabuLcAte9HGbkN9uJ4CVyLBWlrUTmz4UKITD+P1bW/zRWM1P5lyfae2I2JFACQWX+gQ8nlbyNUxpJuRgpZZUS29U8j2L8jfavdAgoLzxIpbBKIBij2JCTmBAI6duyh4+x0arvweKMXjny6gv6eCaV85u9PaEYvHCMfCjCgf0WWrbQlCPh2DncLC7hSy/Qv2pEbzn87IaSRvC12HiH6ehGJpKZITpp3Cl5ahtCYw40y2BnbwUvXrXH7Yd/HYi6p0MHEdJxAJMLxcYvGF7odDOXA4HbjJPRhJT2PR2tuC7WPIflMQU+beI6KfJ/6wH5cz8XOFwxCL4lu4iMiEcVj7D+PJ9+dhaYtZE7/VKffXWtMQbmBIyRCJxRd6JOlpLJoj2SnkcDpHYpGMJTjTfQsSopo/Ivp5YOelT4ptIIBr3ee4P/2M2l/8hFg8xvz1f+OkIVPYv98BndIGf9jPgMIByUk8gtAbsSet5eN0Tp/Elk+IqsxsNojo50EoFiKu4+aPxTbtvLgU7XYRPPN0Xql+k63BHfzqxP/tlPs3Rhop95XTv7B/p1xfEHoK6b4FD01NnOnpsnP5FWwTUkbCxG5EMqdXJyKinweNkcbUK2k4DJEwvheWEDrhWHRFOY+vXsBg7wBOPeSMDr93MBrE6/KyX9F+3fKPVBC6ExnpspvRT9vh3B1RSnX6v3MR/TyoD9fjcSZGFYEAnndW49y5m+C509nUsIVlW5dzzcT/xtXB0TShWAiHclBZUikOLEHoIGyHc1+l7z55nkSsCNF41Iz0E6Yd34tLiZeWEDr5eOav/xsKxcwJl3TofaNWlHg8ztDSoRLfLAhChyGi3woZCdbCYZS/3qRdOOM0Ik7Fk5/9ndOGnkBlxbCWL9QGrLhF2AoztGyoxOILgtChiOi3Ql24zkQTAAQCeJe9iSMYIjjjTBZvXsaucA3fPKzjRvlxHacx0khlcWUqZ78gCEIHIaLfAnYKW7fTnTLtLFpKbGgVkSMm8Pi6BQwrHMKJo05t/WJ5oLXGH/YzuGRwauavIAhCByKi3wIZs3DDYRxbt+F5+x2CM85gff1G3tqxilmjL8Lp7Bh/eEOkgQGFAyj3lnfI9QRBELIR0W+B+nB9KtdIIIBvycuoeJzAjOn8Zf2zuJSTiybM6pB7NYQbKPWUMqBwQIdcTxAEIRci+s0Q13EaIg0mVNOekPXCUiLjx9IwdD/+uuF5zhh+CgNLh+z1vYLRID63j0HFgyQWXxCETkVEvxlCsRBaJ2bthcO4PvoE9yfrCMyYzj82/ZPaqJ9Lxu+9AzccC6OUYkjJEInFFwSh0xGVaYbGSGOGaadw0VK0y0nozNN4fN0CDigezpQDTtqre0StKLF4jKGlQ1tMWSsIgtBRiOjnQGttZuG6Eqadmhp8L/6T8AnH8r7ayard73PJmJkoR/t/PituEYqFGFo6NBUSKgiC0MmI6OfAztbnUA4Ih/G8vQLnjp0EZpzJX9Y9g8dRwNfHZy8TnD+2v6CypLJLFk8XBEGwyUv0lVLTlFKfKKXWK6Xm5Dj/e6XU6sT2qVKqNu2clXZuYUc2vrMIRAMp+3oggO+FpcRLitl1/BE8s/EFzt7/NCqK2xdlo7WmIdLA4OLBlHhKOrDVgiAIrdOqIVkp5QT+CJwGbAbeUUotTKyLC4DW+pq08lcDE9MuEdRaT+i4Jnc+daG6pGlHbduO95V/EZz+Vf6+7VUaY0EumXBpu6/dEGmgv68/Fb6KDmyxIAhCfuQz0p8MrNdab9BaR4AngRktlJ8JPNERjdsX2Cv1uBwuCIfxvvwqjkCQwIwzeXzdAg4tO4gj9j+mXddujDRSXFAssfiCIOwz8hH9KuDLtP3NiWNNUErtD4wEXkk77FVKrVRKva2UOreZerMTZVbu3Lkzz6Z3DsFoMLUTCOB74SViVUNYMbKAD2o+4ZIxF7fLgRuMBilwFjCkZIjE4guCsM/IR71yKZRupuxFwAKtdfoKBcO11pOAi4E7lVIHNrmY1vO01pO01pMGDhyYR5M6j/pwfTLXjuOzDXjeXknwnDN5fP0zFDp9nD/uG22+ZsSKoFBUlVZJLL4gCPuUfBRoM5CeN3goUN1M2YvIMu1orasTnxuAV8m093crrLhFIBowIZRhszqWisepnn4Cz216ifNGnkFJUdts8VprwrEwQ0qGSCy+IAj7nHxE/x1glFJqpFKqACPsTaJwlFKHABXA8rRjFUopT+L7AOBYYG123e5CKBZKrZ2ZmJAVGTeGp/X7hKww32qHAzcYDVLuLZfQTEEQugWtir7WOgZcBSwBPgKe1lp/qJS6SSl1TlrRmcCTWut008+hwEql1BpgGTA3Peqnu9EQaTCjca1xrVqN++N1BGacwePrnmFixWjGDjuiTdez4hYaLQuaC4LQbcjL3qC1XgQsyjr286z9X+ao9xYwbi/a12XYs3B9bp8x7Ty/CO1ysmxKJetWfM4dx/0K2uiAbYw0UllSKWYdQRC6DeJVTBC2wsR13Dha/X4KF/2T8PFTeGz7EkpdxZwz7mttul4oFqKwoFAmYAmC0K0Q0U/QGGlMLn5e8PKrOHfs5IuzT+CFTS/z9QPOwufLX7y11kSsCIOKJFWyIAjdCxH9BHXhOpM7Pxym8PnFxIuLeHx4DVEd45sT2+bADUQDDPANMLN6BUEQuhEi+qQSrDkdTtTu3Xhffo3Gaafwl8+f4+gBEzi4Mn+3hJ2oTdIsCILQHRHRJzELVwNa4/37P3AEgiw+bSRfNG7hkjEXt8mBG4gGGFQ0yJiKBEEQuhki+hjTToErMSHr+cXEKgfzqPM9+hWUccaYltIMZRKMBikpKKHYU9yJrRUEQWg/fV70Y/FYMi+OY+MmPG+vZMOME1my5V9cdOC5eLz5CXhcx7G0xcCifZtGQhAEoSX6vOiHY2HzRWt8f/0bKh7n4cPB0hazJn4r7+sEogEGFA6QVbAEQejW9HnRTyZYC4fx/WMxgXFf4bE9yzhxv8mMGHRIXteIWlFcDhfl3vJObq0gCMLe0adFP67j+CN+PE4PrjXvU/DRpyyc8RW2BndwydhZeTtwg7Egg4sHSwZNQRC6PX1apcKxMFprFOB7YgHa6eTBwdUM9g7g1K+cmdc1AtEAZZ4yCt2FndtYQRCEDqBPi35jpNHkxQkE8C16iU9PncAru95h5kHn4fa2LuJxHUdrLc5bQRB6DH1W9O0EawXOAgqWvY5r2w4eOLEEhWLmxEvyukZjpJH9ivaThGqCIPQY+qzoJ2fhKgeFTz9LqKSQv6j3OHXIFKoGHNBq/XAsjMfpodRT2gWtFQRB6Bj6rOgHogFQoPbU4F26jAVfG83O8B4uGfvNVh249mpYg0sGS0I1QRB6FH3WLlEXMgnWPM8/h6MxwLxD/AxzDObEg09rtW4gGqBfYT+8Lm8XtFQQBKHj6JMj/Vg8RtgK43a4KPzr31h7SD9eD33CrIPOx+lpWcituIVC0d8nq2EJgtDz6JOiH4qFUErh2LQZz1sruO+swbiUk4vymIEbiAYYVCwJ1QRB6JnkJfpKqWlKqU+UUuuVUnNynP+9Ump1YvtUKVWbdu5SpdS6xNb2lcU7gbpQHW6HG99f/0ZIWcwv38QZlScysN/QFuuFYiEK3YUUF0hCNUEQeiat2vSVUk7gj8BpwGbgHaXUwvQFzrXW16SVvxqYmPjeD/gFMAmTvHhVom5Nhz5FG4jrOI2RRorchfgW/J35pw2mxtrGJeNaduBqrYlaUYaWDhXnrSAIPZZ8RvqTgfVa6w1a6wjwJNBSvuGZwBOJ718Flmqt9ySEfikwbW8avLeEYiE0Gvf7H1Kw9hMeONLBAUXDmHLQyS3Wa4w0SkI1QRB6PPmIfhXwZdr+5sSxJiil9gdGAq+0pa5SarZSaqVSauXOnTvzaXe7aYg04HK48D35DGsGK/7tqOaSg7+GKmhezO2EarIaliAIPZ18RD+XLUM3U/YiYIHW2mpLXa31PK31JK31pIEDOy+lgT0L16Nc+J5dyH1n7YfHUcDXx1/cYr1gNMig4kGSUE0QhB5PPiq2GRiWtj8UqG6m7EWkTDttrdvphK0wcR3H+9qbhHZvZ/7QWs4eOpWKiiHN1glGg5R5yygqKOrClgqCIHQO+Yj+O8AopdRIpVQBRtgXZhdSSh0CVADL0w4vAU5XSlUopSqA0xPH9gmBaACFwvfkM8w/3E0D4RYduHEdx4pbDCgc0MUtFQRB6BxaFX2tdQy4CiPWHwFPa60/VErdpJQ6J63oTOBJrbVOq7sHuBnTcbwD3JQ4tk+oDdXijcTxvrCY+07wcWjpgRwxYkqz5RsjjQwsGmgWWREEQegF5JWGQWu9CFiUdeznWfu/bKbuw8DD7WxfhxGxIkStKOX/WMJ/SgOsKYRbR12B8niaLe9xemQ1LEEQehV9xjMZioZQKAqfepb7jvdS6PRy/vgLc5bVWhOKhhhUPEhi8gVB6FX0GdGvC9fhrakn+PbrPHlIlPOGnU5Jae5IoWA0SIWvAp/b18WtFARB6Fz6hOhbccssa/jX5/nLmDhBh8W3xn0THE0f34pbaDT9CyWhmiAIvY8+IfqhmDHteJ96hvuOLWBi+aGM3f/InGUbI40MKhokq2EJgtAr6ROi7w/78X32BStqP+SjsgiXHPw1yDEDNxQLUVhQSImnZB+0UhAEofPp9aIf13H8ET9lT/6N+4+EUmch54z7epNyWmsiVoRBReK8FQSh99LrRT8cCxO3YjQ+/ywLxii+tv+Z+Eqa5tAJRAMM8A3A48odwikIgtAb6PWG68ZII0X/fpe/DNpOxAHfHDeriQM3Fo/hUA5JqCYIQq+n14/068P1lD75Nx44UnFU+VgOGTahSZlANMDg4sGyGpYgCL2eXi36EStCLNDA8vde4LMKzSVf+UYTB24wGqSkoEQSqgmC0Cfo1aIfjAYpeelV5o0O0l8Vcea48zPOx3UcS1sMLOq8dM6CIAjdiV5t068N1RJ79mmeOwJmH3A2nqKyjPOBaICBhQNlNSyhy4hGo2zevJlQKLSvmyL0ULxeL0OHDsXtbl8iyF4r+rF4jOj2rfxf43IsB8waf0mGA9deDavMW9bCVQShY9m8eTMlJSWMGDFCQoOFNqO1Zvfu3WzevJmRI0e26xq91rwTioXw/e0f/GlinBMLRzOycnTG+WAsyODiwbIaltClhEIh+vfvL4IvtAulFP3799+rN8Veq3j14Xre/td8NpfBNydemuHADUQDlHnKKHQX7sMWCn0VEXxhb9jbv59eKfpxHSey9gMertjIYKuQ0w49K3nOiltorcV5KwhCn6RXin4oFmLPXx/jxVEwc/h03MWlyXOBaID9ivaThGpCz2D+fBgxwvijRoww+3vB7t27mTBhAhMmTGDw4MFUVVUl9yORSF7X+M53vsMnn3zSYpk//vGPzN/LtgqdQ69Uvoawn6c3PIc6DGYe872kAzccC+Nxeij1lLZyBUHoBsyfD7NnQyBg9r/4wuwDzJrVrkv279+f1atXA/DLX/6S4uJirr322owyWmu01jhypB4HeOSRR1q9z5VXXtmu9nU2rT1bXyCvJ1dKTVNKfaKUWq+UmtNMmW8opdYqpT5USv1f2nFLKbU6sTVZUL2j0VrT8OpSHj2wgdPVQVQNOih5PBwLM7hksNhUhe7BD38IJ53U/HbZZSnBtwkEzPHm6vzwh+1qyvr16xk7diyXX345hx9+OFu3bmX27NlMmjSJMWPGcNNNNyXLHnfccaxevZpYLEZ5eTlz5sxh/PjxHHPMMezYsQOAG2+8kTvvvDNZfs6cOUyePJlDDjmEt956C4DGxkYuuOACxo8fz8yZM5k0aVKyQ0rnuuuuY/To0Rx22GH85Cc/AWDbtm3MmDGDww47jPHjx/Pvf/8bgNtuu42xY8cyduxY7r777maf7cUXX+SYY47h8MMP58ILL6SxsbFdv1tPpFXRV0o5gT8CZwCjgZlKqdFZZUYBNwDHaq3HAOl/eUGt9YTElr6QeqcQsSL864V72V4MsyZ9FxJr4AaiAfoV9sPr8nZ2EwShYwiH23Z8L1m7di2XXXYZ7777LlVVVcydO5eVK1eyZs0ali5dytq1a5vUqaur48QTT2TNmjUcc8wxPPxw7uWwtdasWLGC3/72t8kO5O6772bw4MGsWbOGOXPm8O677zapt337dhYtWsSHH37Ie++9xw033ACYN4nTTjuN9957j1WrVnHooYeyYsUK5s+fz4oVK1i+fDn33nsv7733XpNnc7vdzJ07l5dffpn//Oc/HHbYYdx1110d9TN2e/Ix70wG1mutNwAopZ4EZgDpfwHfA/6ota4B0Frv6OiG5kvAv4fHIisYHvZy4sTzAOO8VSj6+2Q1LKEbkRgJN8uIEcakk83++8Orr3Z4cw488ECOPDK1uNATTzzBQw89RCwWo7q6mrVr1zJ6dGbos8/n44wzzgDgiCOO4PXXX8957fPPPz9ZZuPGjQC88cYbyZH7+PHjGTNmTJN6/fr1w+Fw8L3vfY/p06dz1lkmKOPVV1/lySefBMDlclFaWsrrr7/OBRdcQGGhico799xzeeONNzj99NMznu2tt95i7dq1TJkyBYBIJMJxxx3X9h+sh5KPeacK+DJtf3PiWDoHAwcrpd5USr2tlJqWds6rlFqZOH5urhsopWYnyqzcuXNnmx4gm7XP3M+y4RaXVJyMs6gYgMZoI4OKB0lCNaFnccstUJgVVlxYaI53AkVFqfxT69at4/QlcDIAABTeSURBVK677uKVV17hvffeY9q0aTljwwvSQqGdTiexWCzntT2JN+70MlrrVtvkdrtZuXIl5557Ls888wzTp09Pnss207Z0vfRn01ozbdo0Vq9ezerVq1m7di3z5s1rtS29hXxEP5cBPPvXdQGjgJOAmcCDSqnyxLnhWutJwMXAnUqpA5tcTOt5WutJWutJAwe2P5QyakV58p1HcVnwta/+EBwOQrEQRe4iiguK231dQdgnzJoF8+aZkb1S5nPevHY7cdtCfX09JSUllJaWsnXrVpYsWdLh9zjuuON4+umnAXj//fdzmo/8fj/19fWcddZZ/P73v0+agE4++WTuv/9+ACzLor6+nhNOOIG//e1vBINBGhoaeO655zj++OObXHPKlCm89tprbNiwATC+hXXr1nX483VX8hH9zcCwtP2hQHWOMs9praNa68+BTzCdAFrr6sTnBuBVYOJetjkn8+/7bw6c4+OeIZsoiMOKlx9Da03UijKoWFbDEnoos2bBxo0Qj5vPLhB8gMMPP5zRo0czduxYvve973Hsscd2+D2uvvpqtmzZwmGHHcbtt9/O2LFjKSvLTItSV1fH9OnTGT9+PFOnTuWOO+4A4J577mHJkiWMGzeOSZMm8fHHHzN58mRmzpzJkUceydFHH80VV1zBuHHjmtx30KBBPPTQQ1x44YWMHz+eKVOm8Omnn3b483VXVGuvWEopF/ApcAqwBXgHuFhr/WFamWnATK31pUqpAcC7wAQgDgS01uHE8eXADK110y49waRJk/TKlSvb9BDz7/tvZm+5j0Ba/qHCKNzWfyYzL7+Hfr5+bbqeIHQWH330EYceeui+bka3IBaLEYvF8Hq9rFu3jtNPP51169bhcvXKSPIOJdffkVJqVcKq0iKt/rpa65hS6ipgCeAEHtZaf6iUuglYqbVemDh3ulJqLWAB12mtdyulpgAPKKXimLeKuS0Jfnv52YZ5BLKsNwE33Lblaa7w/qWjbycIQgfQ0NDAKaecQiwWQ2vNAw88IILfBeT1C2utFwGLso79PO27Bn6U2NLLvAU0fb/qYDYVWTmPf1lsSUI1QeimlJeXs2rVqn3djD5Hr1DE4Y25o3KaOy4IgtBX6RWif8sBsymMZh4rjJrjgiAIQopeIfqzrriXeVVXMLzBidIw3O9kXtUVzLri3n3dNEEQhG5Fr/GazLriXs6P3s7uwG6qSqskRFMQBCEHvWKkb+N1eSWhmtCrmP/+fEbcOQLH/zoYcecI5r+/9+mKt23bxkUXXcSBBx7I6NGjOfPMM7ttnPqIESPYtWsXQDJtQjbf/va3WbBgQYvXefTRR6muTk0v+q//+q+ck8H6Ar1mpA9mWrZL9apHEvow89+fz+znZxOImkybX9R9weznjZ9q1rj2TdLSWnPeeedx6aWXJnPXrF69mu3bt3PwwQcny1mWhdPZvQIh7Oyc7eHRRx9l7NixVFZWAvDggw92VLM6lFgs1ulhq71qpC8IPYkfLv4hJz16UrPbZc9dlhR8m0A0wGXPXdZsnR8ubjm18rJly3C73Vx++eXJYxMmTOD444/n1Vdf5eSTT+biiy9OzmS94447kqmK7VTJjY2NyVmyY8eO5amnngJgzpw5yRTI2Tn6Ae677z6uv/765P6jjz7K1VdfDZjkaEcccQRjxoxpNg9OcbGZjKO15qqrrmL06NFMnz49mc4Z4KabbuLII49k7NixzJ49G601CxYsYOXKlcyaNYsJEyYQDAY56aSTsCeBPvHEE4wbN46xY8cmE8DZ9/vZz37G+PHjOfroo9m+fXuTNr322mvJRWgmTpyI3+8HTIrncePGMX78eObMMdnoV69ezdFHH81hhx3GeeedR01NDQAnnXQSP/3pTznxxBO566672LlzJxdccAFHHnkkRx55JG+++Wbz/0PbgQyLBaGbErZyp1Bu7ng+fPDBBxxxxBHNnl+xYgUffPABI0eOZNWqVTzy/9u7++CoyiyP499jDMa4TAwD64BRkJW4FTbsgrzEQkBQQ6BkIQEHNQrBkV0WqVp1UbC0mMEplKHYMoMwMODgMNYu4GolsrgOkpIXa4eXDCwOIySizCovDqtkEgkBR8PZP26n7TTdSRqSfm6nz6eKKri3O/d00pzcfu7vee4rr7Bnzx5UlWHDhjFq1CiOHj1Kr169eOuttwBvqYSamhrKysqoqqpCRKitrb3oa0+ZMoXbbruNJUuWALBx40aeeeYZANauXUu3bt04d+4cQ4YMYfLkyXz3u5FXxS0rK6O6upqDBw9y6tQpcnJyePjhhwGYM2cOCxZ4U4geeughNm/ezJQpU1i+fDlLly5l8ODmE1ZPnjzJvHnz2LdvH5mZmeTn51NeXs6kSZM4e/YseXl5LFq0iKeeeoo1a9bw7LPPNnv+0qVLWbFiBcOHD6e+vp60tDTefvttysvL2bNnD+np6dTU1AAwbdo0XnrpJUaNGsWCBQtYuHBh8BdpbW0tO3bsAOCBBx7g8ccf5/bbb+fTTz9l7NixHD58uIWfamys6RvjSGlBy0sr9yntwyd1Fy+t3DujN9tLtndITUOHDuWmm24CvKWPCwsLgytUFhUV8d5771FQUMDcuXOZN28e99xzDyNGjAgup/DII480WwI5VI8ePejbty+7d++mX79+VFdXB9f0WbZsGWVlZQAcO3aMI0eORG36O3fu5P777yclJYVevXoxZsyY4L5t27axZMkSGhoaqKmpoX///kyYMCHq662srOSOO+6gaaHH4uJidu7cyaRJk+jSpUvwddx6661s3br1oucPHz6cJ554guLiYoqKisjKyqKiooIZM2YEl3ju1q0bdXV11NbWMmrUKACmT5/OvffeG/w6U6dODf69oqKi2fWGL7/8kjNnztC1a9eoryMWNrxjjE8tunMR6anNl1ZOT01n0Z2XvrRy//79W5wFG74EcSTZ2dns27eP3Nxcnn76aZ577jmuvPJK9u7dy+TJkykvL6egoIDGxsbg0EfT2ffUqVN57bXXeOONNygsLERE2L59OxUVFezatYv333+fgQMHRlzGOVSksMb58+eZPXs2r7/+OgcPHmTmzJmtfp2W1h5LTU0NHifastHz58/n5Zdf5ty5c+Tl5VFVVYWqxhwmCf2+X7hwgV27dgWXfj5x4kS7NXywpm+MbxXnFrN6wmp6Z/RGEHpn9Gb1hNWXfBEXYMyYMXz11VesWbMmuK2ysjI4tBBq5MiRlJeX09DQwNmzZykrK2PEiBGcPHmS9PR0HnzwQebOncv+/fupr6+nrq6O8ePHU1payoEDB0hJSQk2rqa7ZRUVFVFeXs769euDZ7d1dXVkZmaSnp5OVVUVu3fvbvE1jBw5kg0bNtDY2Mhnn33Gtm3bAIINvnv37tTX1zdL9HTt2jU43h5q2LBh7Nixgy+++ILGxkbWr18fPBtvi48//pjc3FzmzZsXXO0zPz+ftWvX0hC41WVNTQ0ZGRlkZmYGbzLz6quvRj1Ofn4+y5cvD/470i0kL4cN7xjjY8W5xZfV5MOJCGVlZTz22GMsXryYtLQ0+vTpQ2lpKSdOnGj22EGDBlFSUsLQoUMBL+Y4cOBAtmzZwpNPPskVV1xBamoqK1eu5MyZM0ycOJHz58+jqrz44osRj5+ZmUlOTg6HDh0Kft2CggJWrVrFgAEDuOWWW8jLy2vxNRQWFvLuu++Sm5tLdnZ2sHlee+21zJw5k9zcXPr06dPsLmAlJSXMmjWLq6++ml27dgW39+zZkxdeeIHRo0ejqowfP56JEye2+ftZWlrKtm3bSElJIScnh3HjxnHVVVdx4MABBg8eTJcuXRg/fjzPP/8869atY9asWTQ0NNC3b9+oN5hftmwZjz76KAMGDOCbb75h5MiRwXsHtIdWl1aOt0tZWtmYRGFLK5v2cDlLK9vwjjHGJBFr+sYYk0Ss6RsTZ34bUjWJ5XLfP9b0jYmjtLQ0Tp8+bY3fXBJV5fTp06SlpV3y17D0jjFxlJWVxfHjx/n8889dl2ISVFpaGllZWZf8fGv6xsRRampqcMarMS60aXhHRApEpFpEPhKR+VEe830ROSQiH4jIv4dsny4iRwJ/prdX4cYYY2LX6pm+iKQAK4C7geNApYhsUtVDIY/pBzwNDFfVP4nIXwa2dwN+CAwGFNgXeO6f2v+lGGOMaU1bzvSHAh+p6lFV/TOwAQifsjYTWNHUzFW1aa3TscBWVa0J7NsKFLRP6cYYY2LVljH964FjIf8+DgwLe0w2gIj8N5AC/EhVfx3ludeHH0BE/gFouot5vYhUt6n6yLoDX1zG8zuK1RUbqys2VldsOmNdvdvyoLY0/UjLxYXnza4E+gF3AFnAeyLyN218Lqq6Goh854QYichv2zIVOd6srthYXbGxumKTzHW1ZXjnOHBDyL+zgJMRHvOmqn6tqn8AqvF+CbTlucYYY+KkLU2/EugnIjeJSBfgPmBT2GPKgdEAItIdb7jnKLAFyBeRTBHJBPID24wxxjjQ6vCOqn4jInPwmnUKsFZVPxCR54Dfquomvm3uh4BG4ElVPQ0gIj/G+8UB8Jyq1nTECwnRLsNEHcDqio3VFRurKzZJW5fvllY2xhjTcWztHWOMSSLW9I0xJolY0zfGmCRiTd8YY5JIp2z6IjLDdQ2RiMhfu67BGJPcOmV6R0Q+VdUbXdcRzsd1HVTVXB/UcR3eMh0KnFTVU45LQkQy8NaLCtYFbFHVWsd1jQUmhdX1ZmD5E1c1ZeAtvDgJ6BHY/H/Am8BiH3zP7P1FAjd9EfldtF1AtqpeFc96ggcXWRZtFzBdVb8Tz3qCBxcpirYLWKWqPaLs73Ai8nfAKiADOBHYnAXUArNVdb+juqbhrRL7TlhddwMLVfVXjuoqxZsA+Su8We9NdU0DjqjqPzuqawvwLrBOVf8Y2PY9YDpwl6re7ague3+FHjeBm/4pvFU8w5dpFuA3qtor/lWBiJwB/gX4KsLuf1XV7nEuCQAR+Rr4NyKsfQRMUdWucS4pSEQOAP+oqnvCtucBP1fVv3VUVzUwLPysKzC7fI+qZjuq68NIxxYRAT5U1X4OykJEqlX1llj3dTR7fzWXyHfO2gz8haoeCN8hItvjX05QJfB7Vf1N+A4R+VH8ywn6HbBUVX8fvkNE7nJQT6hrwv9DAqjqbhG5xkVBAULkX5IXiLyYYLycF5Ghqro3bPsQ4LyLggI+EZGn8M70T0FwSKWE5qvtxpu9v0IkbNNX1R+0sO+BeNYSZgpR/uOpqsv75D0GfBllX2E8C4ngbRF5C2+4oqk53IA3XOFsjBpYBOwXkXf4tq4b8T5+/9hZVV4TXSkiXfl2eOcGvJ9viaOaAKYC84EdTTdSAk7hrdX1fWdV2furmYQd3oHgx9mhNL8Isld98qICdw5Tu1NY60RkHN7Nea7HO8s5DmxS1f9yXFcm3jBiaF1b/PAzDYyXB+tqGkc3F7P3V8gxfdIfYyYi+cDPgCM0vwhyM97FmXcc1XUjsAS4E+9CkQDfwbvANV9V/9dFXS0RkXtUdbPrOkznJSKDXF0wNc0lck7/p3iJgHGq+kjgTwHeR6OfOqxrI1AGfE9V+6nqzUBPvOWnNzisqyVDXBcQTeCuar4jIr5cpVFE/NpY/8l1AZEk4/srYcf08Wo/HmH7CSA1zrWE6q6qG0M3qGojsCGwzLQzgclhTR9xm4bDNqnqD13W1QqXF0xb8nPXBUSiqoNc1xCJqs50XUMUSff+SuSmvxaoFJENNL84cx/wC2dVwT4R+RmwjuZ1TQf+x1VRIjIPuB/v00ZT6iMLWC8iG1R1savaWvFn1wVEoqr7XNdgk40un6r69Zd3h72/EnZMH0BEcoC/5+KLM4cc1tQF+AHNLxodA/4T+IWqRsrvx6OuD4H+qvp12PYuwAeust2tcTmL2a8zTG2y0SXVZjOYm46byE2/iaVkWiciVcBYVf0kbHtv4B1XE2cCNfh1drVfZ5jaZKPY6rIZzKHHTdSmH5KSGQPUBTZnYCmZaMcuAJbjpZ1CM8E3A3Mcn/H4dXa1X2eYHon2yUxEPgqEB+Iu8GlyiKrWhW3PwLu1qquZwjaDOUQij+lvBEqB4sCFUkQkBbgXb9w6z2Ft0QzBm0kcd6r6axHJ5tt5DU3DYZVN3z+H/Dq72q8zTG2yUWxsBnOIRD7Tb+lsJ+q+eGghJXPYVU0mdoFhifl4P8vwGaaLXQ4n2mSjmGoaBKwEIs1gnu3qonzY++u6wOY/4r2/fqKqNR1y3ARu+huAGiKnZLqrqpNp32EpmdDxw/sAP6dkTAxEZIaqvuK6Dj/yY6oIbAZzk0Ru+paSMc74JFUU+gnEb6mi43j/J52nigK1+TJK6iJVlLBN36/8nJIxsUnAVFEJcKelii6qy5dRUlepok7Z9C0lY9qDpYpi4+NUkV+jpE5SRYmc3mmJpWRMe7BUUWz8miqy+yKESOgzfUvJmGRkqaKYa5oOLMAb3rkoSqqqv3RUl5NUUcI2fUvJGHMxSxVF5scoaZN4p4oSuelbSsaYMJYqis6PUVIXqaJEHtO/APQCPgnb3jOwz5hOqZVU0XVR9sXDa3ipotERUkX/gTecEnfRoqQi4scF6kYDz4tIh6WKEvlM31IyJilZqig2Po6SOkkVJeyZvqVkTBKzVFFsrglv+ACqultErnFRUICTVFHCnukbY/zFr6kiEVkG/BWRo6R/UNU5jupykiqypm+M6XCuU0V+jJIG6op7qsiavjGmw7lMFfldvFNFCTumb4zxF7+mivwaJXWVKrKmb4xpL9fRQqoo/uUE+TJKCvyS6KmiV4AOSRVZ0zfGtBe/por6qOpPQjcEmv9iEZnhqCZwlCqyMX1jTKcWuH1jBZGjpHer6l2O6nKSKrKmb4zp1PwaJQ3UFvdUkTV9Y0zSch0ldeEK1wUYY4xDC10dWEQyRGSxiBwWkdOBP4cD267tqOPahVxjTKfm1ygpjlJFNrxjjOnUbIG65uxM3xjT2fk1SupkgTo70zfGGAdcpYqs6RtjjM90ZKrImr4xxvhMRy5QZ2P6xhjjgKtUkTV9Y4xxw8kCddb0jTHGDSepIhvTN8aYJGLLMBhjTBKxpm+MMUnEmr4xxiQRa/rGGJNE/h+n/Yr+4/LeNAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.model_selection import validation_curve\n",
    "\n",
    "def plot_validation_curve(parameter_values, train_scores, validation_scores):\n",
    "    train_scores_mean = np.mean(train_scores, axis=1)\n",
    "    train_scores_std = np.std(train_scores, axis=1)\n",
    "    validation_scores_mean = np.mean(validation_scores, axis=1)\n",
    "    validation_scores_std = np.std(validation_scores, axis=1)\n",
    "\n",
    "    plt.fill_between(parameter_values, train_scores_mean - train_scores_std,\n",
    "                     train_scores_mean + train_scores_std, alpha=0.1,\n",
    "                     color=\"r\")\n",
    "    plt.fill_between(parameter_values, validation_scores_mean - validation_scores_std,\n",
    "                     validation_scores_mean + validation_scores_std, alpha=0.1, color=\"g\")\n",
    "    plt.plot(parameter_values, train_scores_mean, 'o-', color=\"r\",\n",
    "             label=\"Training score\")\n",
    "    plt.plot(parameter_values, validation_scores_mean, 'o-', color=\"g\",\n",
    "             label=\"Cross-validation score\")\n",
    "    plt.ylim(validation_scores_mean.min() - .1, train_scores_mean.max() + .1)\n",
    "    plt.legend(loc=4)\n",
    "\n",
    "\n",
    "training_scores, test_scores = validation_curve(SVC(kernel='linear'), X_train, y_train, param_name=\"C\", param_range=Cs,cv=10)\n",
    "plot_validation_curve(range(len(Cs)), training_scores, test_scores)\n",
    "plt.xticks(range(len(Cs)), Cs,rotation='vertical');\n",
    "plt.ylim([0.6, 1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice that from value of C=1, increasing C results in better accuracy on the training set but worse performance on the test set. This is because being too demanding on the separation of data in the training dataset, we are overfitting to it and we decrease performance in the test set. A nice picture of typical overfitting."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Polynomial kernels\n",
    "\n",
    "We have seen that performance with a linear SVM is Ok but not competitive with Metamethods. However, it could happen that using kernels we could even improve accuracy. We'll try first ploynomial kernel with degree 2 with default parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix on test set:\n",
      " [[29  9]\n",
      " [ 0 68]]\n",
      "\n",
      "Accuracy on test set:  0.9150943396226415\n"
     ]
    }
   ],
   "source": [
    "knc = SVC(kernel='poly',degree =2) \n",
    "knc.fit(X_train, y_train)\n",
    "pred=knc.predict(X_test)\n",
    "print(\"Confusion matrix on test set:\\n\",sklearn.metrics.confusion_matrix(y_test, pred))\n",
    "print(\"\\nAccuracy on test set: \",sklearn.metrics.accuracy_score(y_test, pred))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Better performance than the obtained with a linear SVM... It could even be increased because we didn't tune the C parameter for the polynomial kernel. Let's do that now."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEACAYAAABfxaZOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHFVJREFUeJzt3Xt81PWd7/HXJwkhgNwTUAkBFLzg3ebQ1ru2KrVWqnVd3O1jbU+71LbqVq1V2z3q0tVebOvRHleru7Rn6yqL2gtbOaWud22txAtYUCRgZ5IGJcwAQgYSknzOH5noGBMySWby+/1m3s/HYx7O/Ob3C++M5J0f3/nO92fujoiIFIeSoAOIiMjwUemLiBQRlb6ISBFR6YuIFBGVvohIEVHpi4gUEZW+iEgRUemLiBQRlb6ISBFR6YuIFJGyoAP0VFlZ6TNnzgw6hohIpLz44otb3b2qv/1CV/ozZ86krq4u6BgiIpFiZrFs9tPwjohIEVHpi4gUEZW+iEgRUemLiBQRlb6ISBFR6YuIFBGVvgTmrR17eKVhO7pkp8jwCd08fSlcO/fs5Y+bkjxbv5Vn67dSv2UXAGccNoXvXHAUU8dVBJxQpPCp9CVv9nZ0srphO89s2Mpz9Vt5uWE7HZ1OxYgS5s2azEW11XR0wu2PvcGZP3qKGz91BBccPw0zCzq6SMFS6UvOuDv1W3Z1nclv2MrzmxK0tHVgBkdPG8+XTjmIk+ZUcnzNRCpGlL573Pwj9+eaB1dz9YOrWfHqZm7RWb9I3ljYxlNra2tdyzBEx5Z39vDcxq3vns2//U4rADMmj+ak2ZWcNLuSjx48mQmjy/f5dTo6nZ8+9ya3rlzPyLISbjrvCM4/Tmf9Itkysxfdvba//XSmLwPS0trOH99M8OyGBM/WN/PG213j8hNHj+CEdMmfNLuS6ZNGD+jrlpYYXzz5IM44bArXPLSGq5atZsWrb3HL+UcyRWf9IjmjM33pV/dZ+O/Wvs1L8W20dzrlZSXMmzmJE2dXcvKcSuYeMI6SktyclWee9VeMKOWfzjuCBcceqLN+kX3I9kxfpS/9+tdnNvHPj7zGEQeO4+Q5VZw0u5Lame8fl8+Hjc27+PqDq3k5vp0z507l5vOPZMpYnfWL9EalLzlRv2Un59zxLKfMqeLev/vQsJ9td3Q6//bsJn7wuzcYXd511n/eMTrrF+kp29LXh7OkT+0dnVy1bDVjyku55YIjAyna0hJj0SkHs+KKk5k5eQz/sPQVLr3vRZp3tg57FpFCoNKXPt315EbWNO7gnz99VODDKrOn7MfDXz6B6z5xGE+sb+as255i+eomfZpXZICyKn0zm29m682s3syu6+X5GWb2mJmtMbMnzaw647lLzGxD+nZJLsNL/qxt2sHtj23gU8ccyCePPiDoOEDXWf+lpx7MiitOombyGK544GW+fN9LbN2ls36RbPVb+mZWCtwJfAKYC1xsZnN77PYD4N/d/WhgMfCd9LGTgBuBDwPzgBvNbGLu4ks+tLZ3cPWy1UwcU87i844IOs4HzJ4ylocv/SjXzj+Mx1/fwpk/eorfrGkKOpZIJGRzpj8PqHf3Te7eBiwFFvTYZy7wWPr+ExnPnw086u5Jd98GPArMH3psyafb/3sDr7+1k+9ecBQTx+z7Q1VBKSst4cunHcwjV5xEzaTRXHb/y3zlP17UWb9IP7Ip/WlAQ8bjxvS2TKuBz6Tvnw+MNbPJWR4rIfJSfBt3P7WRi2qr+djhU4OO0685U8fy8JdP4BvzD+W/123hrNue5pE1m4OOJRJa2ZR+b1M2er579nXgVDN7GTgV+AvQnuWxmNkiM6szs7rm5uYsIkk+7G7r4OvLVnPA+FH8r3N7juCFV1lpCV85bTa/ueIkqieO4qv3v8SXfl7HpuZdQUcTCZ1sSr8RmJ7xuBp43wCquze5+wXufhzwrfS2Hdkcm973HnevdffaqqqqAX4LkivfX/k6m7a2cOuFRzO2YkTQcQbskKlj+UX6rP+ZDVs587an+eYvX+Xtd/YEHU0kNLIp/VXAHDObZWblwEJgeeYOZlZpZt1f63pgSfr+SuAsM5uYfgP3rPQ2CZnfb9zKT5/7M587YSYnzK4MOs6gdZ/1P3XN6Xz2wzU8WNfAqbc+wfd++zo7du8NOp5I4PotfXdvBy6jq6xfA5a5+1ozW2xm56V3Ow1Yb2ZvAFOBm9PHJoFv0/WLYxWwOL1NQmRXazvXPLiGWZVjuHb+YUHHyYmqsSP5pwVH8thVp3H2Eftz15MbOeX7T/CTpzayZ29H0PFEAqNlGITrf7GG/1zVwIOXfpQPzZgUdJy8WNu0g1tXrufJ9c0cML6Cr318Dp85vpqyUn0+UQqDlmGQrDzx+hYeeKGBRaccXLCFD3DEgeP52efnsXTRR5g6roJrH36V+bc/w2//9JY+1StFRaVfxLan2rj24TUcMnU/rjxzTtBxhsVHDprML79yAnd/9kO4O5fe9yIX3PV7nt+UCDqayLBQ6RexG5evJdnSxo8uOpaRZfldJjlMzIz5R+7Pyq+dwvc+cxSbt+9h4T3Pc8mSF1jbtCPoeCJ5pdIvUite3cyvX2ni8jPmcOS08UHHCURZaQl//T9qePKa0/jmOYfxSsN2PnnHs/zD0peJJ1JBxxPJC72RW4Sad7Zy9v9+mmkTRvGLr5zACL2ZCcCO3Xv5yVMbWfLcm3R0On8zr4bLzphD1diRQUcT6ZfeyJVeuTvf+uWr7Gpt54cXHaPCzzB+1Ai+Mf8wnrrmdP6qdjr3/THOqbc+wY8efYOdezTHXwqDLoxeZH758l/43bq3+eY5h3HI1LFBxwmlqeMquOX8o/jiSbP44e/e4I7HNnDf8zG+evpsTj+0SlftkrwZWVbCgRNG5fXP0PBOEdm8Yzdn3fY0h04dy39+6aOU5uhC5oVuTeN2vvfb13muXjN8JL+OnT6BX331xEEdm+3wjs70i4S7842H1tDe4fzgr45R4Q/A0dUT+I8vfoS6Pydp3LY76DhSwCaMzv+aVyr9InH/C3Ge2bCVby84gpmVY4KOE0m1MydROzPoFCJDo3fxikA8keLmR17jpNmV/O2HZwQdR0QCpNIvcJ2dztcfXE2pGd+78GhKNKwjUtRU+gVuyXNv8sKfk9zwqblMy/OsABEJP5V+AavfspPvr1zPxw+fwoUfqg46joiEgEq/QLV3dHL1stWMLi/llguO0txyEQE0e6dg3f3URlY37uD//M1xTBlbEXQcEQkJnekXoLVNO7j9sQ2ce/QBnHv0gUHHEZEQUekXmNb2Dq5etprxo8r59oIjg44jIiGj4Z0C8+PH6nn9rZ3869/VMnFMedBxRCRkdKZfYJauauCsuVP5+NypQUcRkRBS6ReQltZ2tu5q5ZjpE4KOIiIhpdIvIPFk19WeaiaNDjiJiISVSr+AxNKX+JsxWaUvIr1T6ReQeLIFgBmTtIqmiPROpV9AYokU40eNYPwwrMktItGk0i8g8WRKQzsisk9Zlb6ZzTez9WZWb2bX9fJ8jZk9YWYvm9kaMzsnvX2mme02s1fSt7tz/Q3Ie+LJFNP1Jq6I7EO/H84ys1LgTuBMoBFYZWbL3X1dxm7/CCxz97vMbC6wApiZfm6jux+b29jSU3tHJ3/ZtptPHnVA0FFEJMSyOdOfB9S7+yZ3bwOWAgt67OPAuPT98UBT7iJKNpq276G90zW8IyL7lE3pTwMaMh43prdlugn4rJk10nWWf3nGc7PSwz5PmdnJQwkrfYulZ+7UaOaOiOxDNqXf20Ls3uPxxcDP3L0aOAf4uZmVAJuBGnc/DrgKuN/MxvU4FjNbZGZ1ZlbX3Nw8sO9AAM3RF5HsZFP6jcD0jMfVfHD45gvAMgB3/wNQAVS6e6u7J9LbXwQ2Aof0/APc/R53r3X32qqqqoF/F0JDMkV5aQlTx2ntfBHpWzalvwqYY2azzKwcWAgs77FPHPgYgJkdTlfpN5tZVfqNYMzsIGAOsClX4eU9sUSK6kmjKNWFz0VkH/qdvePu7WZ2GbASKAWWuPtaM1sM1Ln7cuBq4F4zu5KuoZ/Pubub2SnAYjNrBzqAS909mbfvpojFkilmaLqmiPQjq/X03X0FXW/QZm67IeP+OuDEXo57GHh4iBmlH+5OPNHCh2dNCjqKiIScPpFbABItbbS0dWh1TRHpl0q/AHQvqayZOyLSH5V+AYgntI6+iGRHpV8Auufoa90dEemPSr8AxJIt7D+ugooRpUFHEZGQU+kXgHgiRY3G80UkCyr9AqA5+iKSLZV+xO1u66B5Z6vexBWRrKj0I657uqaGd0QkGyr9iIsl0hdDn6wllUWkfyr9iHv3g1ka3hGRLKj0Iy6WSDG2oowJo0cEHUVEIkClH3HxZIqaSaMx05LKItI/lX7ExZMprbkjIllT6UdYR6fTuC2l6+KKSNZU+hHWtH03eztcZ/oikjWVfoRp5o6IDJRKP8L0wSwRGSiVfoTFEilGlBoHjB8VdBQRiQiVfoTFky1UTxxNaYmma4pIdlT6ERZLpLTQmogMiEo/otydeEJz9EVkYFT6EbU9tZedre060xeRAVHpR1QsqYuhi8jAqfQjSksqi8hgqPQjKp7Qmb6IDJxKP6JiyRRTxo5kVHlp0FFEJEKyKn0zm29m682s3syu6+X5GjN7wsxeNrM1ZnZOxnPXp49bb2Zn5zJ8MdPMHREZjH5L38xKgTuBTwBzgYvNbG6P3f4RWObuxwELgX9JHzs3/fgIYD7wL+mvJ0MUT6aYrqEdERmgbM705wH17r7J3duApcCCHvs4MC59fzzQlL6/AFjq7q3u/iZQn/56MgR79nbw1jt7mKEllUVkgLIp/WlAQ8bjxvS2TDcBnzWzRmAFcPkAjpUBauheXVPDOyIyQNmUfm8Lu3iPxxcDP3P3auAc4OdmVpLlsZjZIjOrM7O65ubmLCIVt1hCq2uKyOBkU/qNwPSMx9W8N3zT7QvAMgB3/wNQAVRmeSzufo+717p7bVVVVfbpi1RM6+iLyCBlU/qrgDlmNsvMyul6Y3Z5j33iwMcAzOxwukq/Ob3fQjMbaWazgDnAC7kKX6wakin2G1nGpDHlQUcRkYgp628Hd283s8uAlUApsMTd15rZYqDO3ZcDVwP3mtmVdA3ffM7dHVhrZsuAdUA78FV378jXN1MsYokWpk8ajZmWVBaRgem39AHcfQVdb9Bmbrsh4/464MQ+jr0ZuHkIGaWHWDLFIVPGBh1DRCJIn8iNmI5OpzG5WzN3RGRQVPoR89Y7e2jr6NTMHREZFJV+xHQvtKYPZonIYKj0Iyae7FpSWatrishgqPQjJpZIUVZiHDihIugoIhJBKv2IiSVTTJs4irJS/a8TkYFTc0RMPJHS0I6IDJpKP2LiSa2jLyKDp9KPkB2pvezYvVczd0Rk0FT6ERJLz9zRxVNEZLBU+hHSvaSyhndEZLBU+hESTy+prDdyRWSwVPoREku0ULnfSMaMzGqdPBGRD1DpR4hm7ojIUKn0I0Rz9EVkqFT6EdHa3sHmd/ao9EVkSFT6EdGQ3I27Zu6IyNCo9COie3VNlb6IDIVKPyK619Gv0adxRWQIVPoREUumGF1eSuV+5UFHEZEIU+lHRPfMHTMLOoqIRJhKPyJiSU3XFJGhU+lHQGen64NZIpITKv0I2LKzlbb2Tmom601cERkalX4ExBLp6Zoa3hGRIVLpR0BMq2uKSI5kVfpmNt/M1ptZvZld18vzt5nZK+nbG2a2PeO5joznlucyfLGIJ1KUlhjTJo4KOoqIRFy/a/SaWSlwJ3Am0AisMrPl7r6uex93vzJj/8uB4zK+xG53PzZ3kYtPLJniwAkVjCjVP8xEZGiyaZF5QL27b3L3NmApsGAf+18MPJCLcNIlnmjRdXFFJCeyKf1pQEPG48b0tg8wsxnALODxjM0VZlZnZs+b2acHnbSIxZMpajRdU0RyIJtLMPX2EVDvY9+FwEPu3pGxrcbdm8zsIOBxM3vV3Te+7w8wWwQsAqipqckiUvF4Z89etqX26k1cEcmJbM70G4HpGY+rgaY+9l1Ij6Edd29K/3cT8CTvH+/v3uced69199qqqqosIhWP7oXWNF1TRHIhm9JfBcwxs1lmVk5XsX9gFo6ZHQpMBP6QsW2imY1M368ETgTW9TxW+hbrXl1TwzsikgP9Du+4e7uZXQasBEqBJe6+1swWA3Xu3v0L4GJgqbtnDv0cDvzEzDrp+gXz3cxZP9K/2Lvr6OuNXBEZumzG9HH3FcCKHttu6PH4pl6O+z1w1BDyFb2GZIrJY8rZb2RW/6tERPZJE79DLpZIMV3j+SKSIyr9kIsltLqmiOSOSj/E2to72bxjt2buiEjOqPRDrHFbik5HSyqLSM6o9EMsnl5dU8M7IpIrKv0Qe7f0NbwjIjmi0g+xWCJFxYgSqsaODDqKiBQIlX6IxRJdF0M36235IxGRgVPph1g82UKNllQWkRxS6YeUuxNPao6+iOSWSj+kmne2smdvp0pfRHJKpR9S3RdD1xIMIpJLKv2QimkdfRHJA5V+SMUTLZQYVE9U6YtI7qj0QyqWTHHA+FGUl+l/kYjkjholpDRzR0TyQaUfUnEtqSwieaDSD6Fdre0kWto0c0dEck6lH0KxRPq6uPo0rojkmEo/hOIJLaksIvmh0g+h7iWVa1T6IpJjKv0QiiVTTBw9gnEVI4KOIiIFRqUfQvH0ksoiIrmm0g+hWLJF18UVkbxQ6YfM3o5Omrbv0Zo7IpIXKv2Qadq+m45O15u4IpIXKv2Q0eqaIpJPWZW+mc03s/VmVm9m1/Xy/G1m9kr69oaZbc947hIz25C+XZLL8IUopumaIpJHZf3tYGalwJ3AmUAjsMrMlrv7uu593P3KjP0vB45L358E3AjUAg68mD52W06/iwIST7RQXlbC1LEVQUcRkQKUzZn+PKDe3Te5exuwFFiwj/0vBh5I3z8beNTdk+mifxSYP5TAhS6Wnq5ZUmJBRxGRApRN6U8DGjIeN6a3fYCZzQBmAY8P9FjpEk+mNJ4vInmTTen3dsrpfey7EHjI3TsGcqyZLTKzOjOra25uziJSYXJ34smUxvNFJG+yKf1GYHrG42qgqY99F/Le0E7Wx7r7Pe5e6+61VVVVWUQqTFt3tZFq69CZvojkTTalvwqYY2azzKycrmJf3nMnMzsUmAj8IWPzSuAsM5toZhOBs9LbpBfxZNeSyjrTF5F86Xf2jru3m9lldJV1KbDE3dea2WKgzt27fwFcDCx1d884Nmlm36brFwfAYndP5vZbKBzdc/RrtI6+iORJv6UP4O4rgBU9tt3Q4/FNfRy7BFgyyHxFJZZIYQbTJ40KOoqIFCh9IjdEGpIpDhhXwciy0qCjiEiBUumHSEwzd0Qkz1T6IRLTOvoikmcq/ZBoaW1n665WZmgdfRHJI5V+SLx7XVyd6YtIHqn0Q6K79GdoTF9E8kilHxLxd9fR1/COiOSPSj8kYskWxlWUMX70iKCjiEgBU+mHRCyR0pu4IpJ3Kv2Q0OqaIjIcVPoh0N7RyV+27dbqmiKSdyr9ENi8Yw/tna6ZOyKSdyr9ENDqmiIyXFT6IRDTOvoiMkxU+iEQT6QoLy1h/3EVQUcRkQKn0g+BWCJF9aRRlJb0dklhEZHcUemHQDyZ0swdERkWKv2AuXtX6euDWSIyDFT6AUu2tLGrtZ3pOtMXkWGg0g9YrHt1TZW+iAwDlX7A3l1dU9M1RWQYqPQD1r2OvoZ3RGQ4qPQDFkuk2H9cBRUjSoOOIiJFQKUfsHiyRZdIFJFho9IPWCyhJZVFZPio9AO0u62DLTtbNXNHRIaNSj9ADdvSq2vqTF9EhklWpW9m881svZnVm9l1fexzkZmtM7O1ZnZ/xvYOM3slfVueq+CFIPbudE19GldEhkdZfzuYWSlwJ3Am0AisMrPl7r4uY585wPXAie6+zcymZHyJ3e5+bI5zF4RYomtJZQ3viMhwyeZMfx5Q7+6b3L0NWAos6LHP3wN3uvs2AHffktuYhSmeTDF2ZBkTRo8IOoqIFIlsSn8a0JDxuDG9LdMhwCFm9pyZPW9m8zOeqzCzuvT2Tw8xb0HpnrljpiWVRWR49Du8A/TWSN7L15kDnAZUA8+Y2ZHuvh2ocfcmMzsIeNzMXnX3je/7A8wWAYsAampqBvgtRFdDMsVhB4wNOoaIFJFszvQbgekZj6uBpl72+bW773X3N4H1dP0SwN2b0v/dBDwJHNfzD3D3e9y91t1rq6qqBvxNRFFHp9OwLaXr4orIsMqm9FcBc8xslpmVAwuBnrNwfgWcDmBmlXQN92wys4lmNjJj+4nAOoTNO3azt8O10JqIDKt+h3fcvd3MLgNWAqXAEndfa2aLgTp3X55+7iwzWwd0ANe4e8LMTgB+YmaddP2C+W7mrJ9i1r26ppZgEJHhlM2YPu6+AljRY9sNGfcduCp9y9zn98BRQ49ZeLrX0Vfpi8hwyqr0o2Dnnr3c+Ou1QcfI2mtv7WREqXHghFFBRxGRIlIwpd/e4ayKJYOOMSDnHTON0hJN1xSR4VMwpT9xTDnPfOOMoGOIiISaFlwTESkiKn0RkSKi0hcRKSIqfRGRIqLSFxEpIip9EZEiotIXESkiKn0RkSJiXcvmhIeZNQOxoHP0UAlsDTrEAEQpb5SyQrTyRikrRCtvGLPOcPd+16YPXemHkZnVuXtt0DmyFaW8UcoK0cobpawQrbxRytqThndERIqISl9EpIio9LNzT9ABBihKeaOUFaKVN0pZIVp5o5T1fTSmLyJSRHSmLyJSRFT6IiJFRKUvIlJEVPpDZGafNrN7zezXZnZW0Hl6MrMxZvZ/0xn/Nug8/Qn769lT+vV90czODTpLf8ysxMxuNrMfm9klQefZFzOrMbPlZrbEzK4LOk9fzOwgM/s3M3soY1uof+aKuvTTf6G2mNmfemyfb2brzay+v79w7v4rd/974HPAX+cxbma+geS+AHgonfG84cjX00DyBvF6DjZr2rXAsuFN+b5cA8m7AJgG7AUaQ571EOARd/+fwNyw5nT3Te7+hR5fIvCfuX1y96K9AacAxwN/ythWCmwEDgLKgdV0/aU7CvhNj9uUjON+CBwfwtzXA8em97k/7K9zEK/nEF7bjwML6foFdW7YX1vgOuBL6X0eCnnWycATwOPA58OaM+P5hzLuB/4zt69bwVwYfTDc/Wkzm9lj8zyg3t03AZjZUmCBu38H+MA/4c3MgO8C/8/dX8pv4i4DyU3XGV018AoB/ctuIHnN7DWG+fXMNMDXdj9gDF0ltdvMVrh75zDGHWjeBqAtvU/HcGXsNsCse4Eb08c8BPw0pDnX9fIlAv+Z25fQBQqBaXT9cHRrTG/ry+V0nfFdaGaX5jNYP/rK/QvgM2Z2F/BfQQTrQ195w/J6Zuo1q7t/y92/BtwP3Dvchb8P+/q7cLaZ/Rh4Oohgvegr62+BK8zsbuDPAeTqqdecZjY5nfE4M7s+/VxYf+YAivtMvw/Wy7Y+P8Hm7ncAd+QvTtZ6ze3uLcDnhztMFvrKG5bXM9M+/064+8+GL0pW+nptU0DP8eeg9ZX1T8CFwx1mH/rKmQAu7bExrD9zgM70e9MITM94XA00BZRlIKKWO0p5o5QVopU3KlmjkrNfKv0PWgXMMbNZZlZO1xt1ywPOlI2o5Y5S3ihlhWjljUrWqOTsX9DvJAd5Ax4ANvPeFLYvpLefA7xB17v13wo6Z9RzRylvlLJGLW9UskYl52BvWnBNRKSIaHhHRKSIqPRFRIqISl9EpIio9EVEiohKX0SkiKj0RUSKiEpfRKSIqPRFRIqISl9EpIj8f49vzzZqvLM3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc. 10-fold cross on train data=  0.9189710144927539\n",
      "\n",
      "Confusion matrix on test set:\n",
      " [[31  7]\n",
      " [ 0 68]]\n",
      "\n",
      "Accuracy on test set:  0.9339622641509434\n",
      "\n",
      "Best combination of parameters found:  {'C': 100.0}\n",
      "\n",
      "Number of supports:  71 ( 19 of them have slacks)\n",
      "Prop. of supports:  0.2897959183673469\n"
     ]
    }
   ],
   "source": [
    "Cs = np.logspace(-3, 11, num=15, base=10.0)\n",
    "\n",
    "param_grid = {'C': Cs}\n",
    "grid_search = GridSearchCV(SVC(kernel='poly',degree =2) , param_grid, cv=10)\n",
    "grid_search.fit(X_train,y_train)\n",
    "\n",
    "scores = grid_search.cv_results_['mean_test_score']\n",
    "\n",
    "plt.semilogx(Cs,scores)\n",
    "plt.show()\n",
    "\n",
    "parval=grid_search.best_params_\n",
    "\n",
    "cvacc = cross_val_score(SVC(kernel='poly',degree =2,C=parval['C']) , X=X_train,  y=y_train, cv=10, scoring='accuracy')\n",
    "print('Acc. 10-fold cross on train data= ', cvacc.mean())\n",
    "\n",
    "\n",
    "knc = SVC(kernel='poly',degree =2,C=parval['C']) \n",
    "knc.fit(X_train, y_train)\n",
    "pred=knc.predict(X_test)\n",
    "print(\"\\nConfusion matrix on test set:\\n\",sklearn.metrics.confusion_matrix(y_test, pred))\n",
    "print(\"\\nAccuracy on test set: \",sklearn.metrics.accuracy_score(y_test, pred))\n",
    "print(\"\\nBest combination of parameters found: \",parval)\n",
    "print(\"\\nNumber of supports: \",np.sum(knc.n_support_), \"(\",np.sum(np.abs(knc.dual_coef_)==parval['C']) ,\"of them have slacks)\")\n",
    "print(\"Prop. of supports: \",np.sum(knc.n_support_)/X_train.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Best result so far. Competitive with meta-methods. Now the best C value found is 100 and accuracy is a lot higher than with default parameters. It's always important when working with SVMs to find best parameters. \n",
    "\n",
    "For this C value we have a nice accuracy on the test set. Let's try what happens now with a polynomial kernel of degree 3."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix on test set:\n",
      " [[ 1 37]\n",
      " [ 0 68]]\n",
      "\n",
      "Accuracy on test set:  0.6509433962264151\n"
     ]
    }
   ],
   "source": [
    "knc = SVC(kernel='poly',degree =3) \n",
    "knc.fit(X_train, y_train)\n",
    "pred=knc.predict(X_test)\n",
    "print(\"Confusion matrix on test set:\\n\",sklearn.metrics.confusion_matrix(y_test, pred))\n",
    "print(\"\\nAccuracy on test set: \",sklearn.metrics.accuracy_score(y_test, pred))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Very bad results! That's because, again, we didn't use the optimal value for the C parameter but the default one. Let's find the best parameter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEACAYAAABfxaZOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHA5JREFUeJzt3Xt81PWd7/HXJ4Ek3G8JggQEbfBWtqApraViXVdFtkds62mh2tXWivYIPWV7tqttj/XQuu05Z7tWLVWptbV7VilLTxUrLXWrFeqtBO9ggYhVZlAMMEEgA7l99o+ZhEnIZQJJfjO/3/v5eMzDmd98f8lnxsx7fnzn8/uOuTsiIhINBUEXICIi/UehLyISIQp9EZEIUeiLiESIQl9EJEIU+iIiEaLQFxGJEIW+iEiEKPRFRCJEoS8iEiEDgi6gvdLSUp88eXLQZYiI5JWNGzfudvey7sblXOhPnjyZqqqqoMsQEckrZvZmNuM0vSMiEiEKfRGRCFHoi4hEiEJfRCRCFPoiIhGi0BcRiRCFvoSKu7N113527K0LuhSRnJRzffoiPbX3YD1/rN7Nuq01rN9Ww673DjOw0Lj1E9P4dOXEoMsTySkKfck7DU3NPP9mgvXbdrNuWw2vxPfhDiMGDeSj7yvl3IpSHn3lbb626mW2vrOfm+aeTmGBBV22SE5Q6Ete+Mvug6zfVsOTW3fzzOu7OVjfRGGBMWPiSJb8zVTOrSjlr8pHtob75WeX851HX+PeP77BtncPcOdnZzC8ZGDAj0IkeAp9yUnvHWrgmdf3pKdsdvNWeo6+fNQg5s2YwOyKMs45ZQwjBnUc5AMKC7jl0jM5ddww/udDr/KJZU9x71UfZErpkP58GCI5x9y9+0Fmc4DbgULgXnf/Xrv7TwLuA8qAvcCV7h5L33cV8M300O+4+/1d/a7KykrX2jvR09TsvBLfx/qtNazbVsPzb9XS1OwMLirkI6eM4dyKMmZPLWPymMGY9Wyq5rnte/jSvz1PU7PzoyvOYtb7SvvoUYgEx8w2untlt+O6C30zKwS2AhcCMWADsMDdN2eM+Xfg1+5+v5n9NfB5d/+cmY0GqoBKwIGNwNnunujs9yn0oyNZ38QjL+/kya01PFW9m9q6BgCmTRjBuRWlzJ5axlmTRlE04PibzHbsreOL91dRXXOAb/2XM/jch0/q8ZuHSC7LNvSzmd6ZCVS7+/b0D14BzAM2Z4w5A1iSvv4E8FD6+sXAY+6+N73vY8Ac4MFsHoSE10s7almy8kW21xxk7LBiLjjtBGZPLeWj7ytlzNDiXv99E0cP5pf/7SN8ZcWL3PzwJv78zn7+16VnMrBQXcsSLdmE/gRgR8btGPChdmNeAj5FagroE8AwMxvTyb4TjrlayXsNTc3c+Xg1y56oZuywYu7/wkxmV5T2y1H30OIBLP/c2fzz77bwoz+8zvaaA9x1xdmMGlLU579bJFdkc5jT0aux/ZzQ/wDOM7MXgPOAONCY5b6Y2UIzqzKzqpqamixKknxU/e5+Pvmjp7nj99uY94ET+e1XZnPe1LJ+nWYpKDC+Nuc0fvCZ6Tz/Vi3zlj3F1l37++33iwQtm9CPAZlnuJQDOzMHuPtOd/+ku88AvpHeti+bfdNjl7t7pbtXlpV1+8Uvkmeam52f/PEN5t7xR2KJOu664iz+5TPTO+286Q+XzZjAyuvOIdnQxCd/9DS/f21XYLWI9KdsQn8DUGFmU8ysCJgPrM4cYGalZtbys24i1ckDsBa4yMxGmdko4KL0NomIeG2SK+59jm//ejPnvq+UtUtmc8m08UGXBcD0iSNZvWgWU0qH8MWfV3HPk6+TTTebSD7rNvTdvRFYRCqsXwNWuvsmM1tqZpemh30M2GJmW4ETgFvT++4Fvk3qjWMDsLTlQ10JN3dn1cYYc25bx8uxWv73p6Zx71WVjB1WEnRpbYwfMYiV153D304bz3d/82e+uvIlDjU0BV2WSJ/Jqk+/P6llM//tOXCYr//qFdZu2sXMyaP5/qc/wMTRg4Muq0vuzg8fr+b7j21lxqSR3PO5s3PuDUqkK9m2bKpfTXrVY5t3cfEP1vHEn2v4+tzTeHDhh3M+8AHMjMUXVHD3lWfz57f3M++HT/FqfF/QZYn0OoW+9Ir9hxr42qqXuPbnVYwdVsIjiz/Kwtmn5N1CZ3PeP45VXzqHAjMuv/tpHn357aBLEulVCn05bs9u38OcH6xn1cYYN5x/Cg/dMItTxw0LuqxjduaJI3h40Szef+IIbnjgeW57bCvNzbk1DSpyrBT6cswONTRx66ObWfDjZxlYaPz79R/hHy4+rVeWTQha6dBi/u3aD/Ffzy7n9t9v44YHnqeuvjHoskSOm1bZlGPyanwfS37xItvePcCVH57E1+eezuCicP05FQ8o5P9c/lecOm4Y/7TmNd68q47lf3c25aNy/zMKkc7k/yGZ9KvGpmbu/P02Llv2FPuSDdz/hZl857JpoQv8FmbGF889mZ9c/UF27K3jktvX86sXYurnl7yl0Jesba85wOV3P8P3H9vKJdPG87slqWUUouD8U8fy6JfP5bRxw1jyi5e44YHn2XuwPuiyRHpMoS9ZeeC5t5h7x3re2H2QOxbM4M4FMxg5OFoLlU0aM5gVC8/hxktO4z82v8tFt63T8g2SdxT60q1Yoo6v/+oVzj5pFL9bMptLP3Bi0CUFprDAuP68U3h40SxKhxZxzf1V3PjLlzlwWB/ySn5Q6Eu33tyT+qrCRedXcMJwnaUKcPr44Ty8aBZf+tgprKzawSW3r+NPb2iFEcl9Cn3pVixx5Ptp5YjiAYX845zTWHndORjGZ5Y/w3fXvMbhRq3dI7lLoS/diieSFBiMG6Gj/I5UTh7Nb/77uSyYOYl71m3n0jufYtNOLeEguUmhL92KJZKMHzFIXy3YhSHFA/inT0zjp5//IHvr6rls2VMse6KaJp3JKzlGr2LpVqw2yYSRmtrJxvmnjuV3X5nNRWeM4/+u3cKn73mGv+w+GHRZIq0U+tKteCKp+fweGDWkiB9+dga3z5/Otl37ueT29fy/Z9/UCV2SExT60qXGpmbeee8QExT6PWJmzJs+gbVLZlM5eRTffOhVrv7pBna9dyjo0iTiFPrSpbf3HaKp2XWkf4zGjxjEz78wk6XzzuS5N/Zw0W3reOSlo74mWqTfKPSlS/HaJAATRmqRsWNlZvzdOZNZ8+VzmVI6hMUPvsCXH3yB2jot4yD9T6EvXYolUqGvI/3jd3LZUFZdfw5fvXAqa155m4t/sI4nt9YEXZZETDiXRpReE0+H/viR6tHvDQMKC1h8QQXnnzaWJb94kavu+xOf/dAkPjh5VNClSQ4YNbiIj506tk9/h0JfuhSvreOE4cUUDygMupRQef+EETyy+KP889ot/OSpN3jgubeCLklywPSJIxX6EqxYQj36faVkYCHf/PgZXHfeKRzUgm0C/fKtcwp96VK8NskHykcGXUaolQ0rpmxYcdBlSETog1zpVHOzs7M2qR59kRBR6Eun3t1/mIYm9eiLhIlCXzrVsqSy5vRFwkOhL51qOTGrfJROzBIJC4W+dKrlxCwd6YuER1ahb2ZzzGyLmVWb2Y0d3D/JzJ4wsxfM7GUzm5vePtnMkmb2Yvpyd28/AOk7sUSS0qFFDCpSj75IWHTbsmlmhcAy4EIgBmwws9Xuvjlj2DeBle5+l5mdAawBJqfve93dp/du2dIfYok6HeWLhEw2R/ozgWp33+7u9cAKYF67MQ4MT18fAWgZwRCI1yY1ny8SMtmE/gRgR8btWHpbpluAK80sRuoof3HGfVPS0z5Pmtm5x1Os9B93J55Qj75I2GQT+tbBtvZfAbQA+Jm7lwNzgX81swLgbWCSu88A/h54wMyGt9sXM1toZlVmVlVTo1UHc8HuA/UcbmxWj75IyGQT+jFgYsbtco6evrkGWAng7s8AJUCpux929z3p7RuB14Gp7X+Buy9390p3rywrK+v5o5Bepx59kXDKJvQ3ABVmNsXMioD5wOp2Y94CLgAws9NJhX6NmZWlPwjGzE4GKoDtvVW89B316IuEU7fdO+7eaGaLgLVAIXCfu28ys6VAlbuvBr4K/NjMlpCa+rna3d3MZgNLzawRaAKud/e9ffZopNe09uhrekckVLJaZdPd15D6gDZz280Z1zcDszrY75fAL4+zRglAPJFk5OCBDC3WQqwiYaIzcqVD6tEXCSeFvnQo1aOv0BcJG4W+HMXd09+YpQ9xRcJGoS9Hqa1roK6+SUf6IiGk0JejqHNHJLwU+nKUeG3qxCwd6YuEj0JfjtJypF+uOX2R0FHoy1FiiSTDigcwfJB69EXCRqEvR4mlV9c062itPRHJZwp9OYp69EXCS6EvR9HZuCLhpdCXNvYlG9h/qFGra4qElEJf2oirR18k1BT60saRdfQV+iJhpNCXNvSNWSLhptCXNuKJJCUDCxg9pCjoUkSkDyj0pY1YIkn5qMHq0RcJKYW+tBGvTWpqRyTEFPrSRixRpw9xRUJMoS+tDh5uJFHXoHZNkRBT6EurI+2aOjFLJKwU+tKq9cQszemLhJZCX1q19OhP1PSOSGgp9KVVrDZJUWEBpUOLgy5FRPqIQl9atayjX1CgHn2RsFLoS6t4Qj36ImGn0JdWqbNxFfoiYZZV6JvZHDPbYmbVZnZjB/dPMrMnzOwFM3vZzOZm3HdTer8tZnZxbxYvvedQQxO7DxzWkb5IyHX7zddmVggsAy4EYsAGM1vt7pszhn0TWOnud5nZGcAaYHL6+nzgTOBE4D/MbKq7N/X2A5Hj09qjP1qhLxJm2RzpzwSq3X27u9cDK4B57cY4MDx9fQSwM319HrDC3Q+7+xtAdfrnSY450qOvE7NEwiyb0J8A7Mi4HUtvy3QLcKWZxUgd5S/uwb6SA2IJfXmKSBRkE/od9e95u9sLgJ+5ezkwF/hXMyvIcl/MbKGZVZlZVU1NTRYlSW+L19YxoMA4YXhJ0KWISB/KJvRjwMSM2+Ucmb5pcQ2wEsDdnwFKgNIs98Xdl7t7pbtXlpWVZV+99JpYIsn4kSUUqkdfJNSyCf0NQIWZTTGzIlIfzK5uN+Yt4AIAMzudVOjXpMfNN7NiM5sCVAB/6q3ipfeoR18kGroNfXdvBBYBa4HXSHXpbDKzpWZ2aXrYV4Frzewl4EHgak/ZROpfAJuB3wI3qHMnN7V8Y5aIhFu3LZsA7r6G1Ae0mdtuzri+GZjVyb63ArceR43Sx+obm9m1/5CO9EUiQGfkCm/vS+Kuzh2RKFDoy5EefYW+SOgp9KW1R3+i5vRFQk+hL8RqkxQYjBuhHn2RsFPoC7FEHeOGlzCwUH8OImGnV7mkevQ1ny8SCQp9UY++SIQo9COusamZd95Tj75IVCj0I+6d9w7R1Ozq0ReJCIV+xKlHXyRaFPoRd2Qdfc3pi0SBQj/iWr4mcbx69EUiQaEfcbFEHWOHFVMysDDoUkSkHyj0Iy5eqx59kShR6EecevRFokWhH2HNzc7OWn1jlkiUKPQj7N39h2loUo++SJQo9CMsXlsHqEdfJEoU+hF2ZB19hb5IVCj0I6wl9E/UnL5IZCj0IyyWSDJmSBGDiwYEXYqI9BOFfoSpR18kehT6ERZL1KlzRyRiFPoR5e6pb8zSfL5IpCj0I2r3gXoONzbrbFyRiFHoR1TL6po60heJFoV+RMUSqROzykcr9EWiRKEfUa3fmKUjfZFIySr0zWyOmW0xs2ozu7GD+28zsxfTl61mVptxX1PGfat7s3g5drFEkhGDBjKsZGDQpYhIP+r2rBwzKwSWARcCMWCDma12980tY9x9Scb4xcCMjB+RdPfpvVey9Ia4VtcUiaRsjvRnAtXuvt3d64EVwLwuxi8AHuyN4qTvqEdfJJqyCf0JwI6M27H0tqOY2UnAFODxjM0lZlZlZs+a2WXHXKn0mtYefYW+SORks+iKdbDNOxk7H1jl7k0Z2ya5+04zOxl43MxecffX2/wCs4XAQoBJkyZlUZIcj9q6Bg7WN6lHXySCsjnSjwETM26XAzs7GTufdlM77r4z/d/twB9oO9/fMma5u1e6e2VZWVkWJcnxUI++SHRlE/obgAozm2JmRaSC/aguHDM7FRgFPJOxbZSZFaevlwKzgM3t95X+1dqjr+kdkcjpdnrH3RvNbBGwFigE7nP3TWa2FKhy95Y3gAXACnfPnPo5HbjHzJpJvcF8L7PrR4LRso6+Ql8kerJaSN3d1wBr2m27ud3tWzrY72lg2nHUJ30glkgytHgAIwapR18kanRGbgS19OibdfQZvYiEmUI/gmKJpKZ2RCJKoR9B8USdevRFIkqhHzHvHWrgvUONOtIXiSiFfsQcWV1TJ2aJRJFCP2LUrikSbQr9iImnT8zSnL5INCn0IyaWSFIysIAxQ4qCLkVEAqDQjxj16ItEm0I/YlI9+voQVySqFPoRE6/VOvoiUabQj5C6+kb2HqxX545IhCn0I+RIj75CXySqFPoRcqRHX3P6IlGl0I+QWK1OzBKJOoV+hMQSdRQVFlA2tDjoUkQkIAr9CIknkpw4soSCAvXoi0SVQj9C1KMvIgr9CGk5G1dEokuhHxGHGpqo2X9YH+KKRJxCPyJ2pjt3dDauSLQp9CNCPfoiAgr9yIjrSF9EUOhHRixRx4AC44Rh6tEXiTKFfkTEE0nGjShhQKH+l4tEmRIgIlI9+praEYk6hX5EpHr09SGuSNQp9COgvrGZd947pCN9Ecku9M1sjpltMbNqM7uxg/tvM7MX05etZlabcd9VZrYtfbmqN4uX7Lyz7xDu6twRERjQ3QAzKwSWARcCMWCDma12980tY9x9Scb4xcCM9PXRwLeASsCBjel9E736KKRLsUQdoCWVRSS7I/2ZQLW7b3f3emAFMK+L8QuAB9PXLwYec/e96aB/DJhzPAVLz7Wuo685fZHIyyb0JwA7Mm7H0tuOYmYnAVOAx3u6r/SdWCJJgcG4ESVBlyIiAcsm9DtafN07GTsfWOXuTT3Z18wWmlmVmVXV1NRkUZL0RDyR5IThJRQN0Of2IlGXTQrEgIkZt8uBnZ2Mnc+RqZ2s93X35e5e6e6VZWVlWZQkPRFL1Gk+X0SA7EJ/A1BhZlPMrIhUsK9uP8jMTgVGAc9kbF4LXGRmo8xsFHBRepv0I62jLyItuu3ecfdGM1tEKqwLgfvcfZOZLQWq3L3lDWABsMLdPWPfvWb2bVJvHABL3X1v7z4E6UpjUzNv7zuk1TVFBMgi9AHcfQ2wpt22m9vdvqWTfe8D7jvG+uQ47dp/mKZmV4++iAA6Izf0YnvVoy8iRyj0Q651HX3N6YsICv3Qa/nGrBMV+iKCQj/04okkZcOKKRlYGHQpIpIDFPohF6tVj76IHKHQD7l4Qj36InKEQj/EmpudnbXq0ReRIxT6IVZz4DD1Tc3q0ReRVgr9ENM6+iLSnkI/xFraNcs1py8iaQr9EGsJfU3viEgLhX6IxWuTjB5SxOCirJZYEpEIUOiHWCyR1Hy+iLSh0A+xeKJOPfoi0oZCP6TcnXitjvRFpC2FfkjtOVjPoYZmHemLSBsK/ZBqbdfU2bgikkGhH1JxtWuKSAcU+iEVr02djavQF5FMCv2QiiWSDC8ZwPCSgUGXIiI5RKEfUvFEUvP5InIUhX5IxRJJTe2IyFEU+iGkHn0R6YxCP4T2JRs4cLhRPfoichSFfgipR19EOqPQD6Ejoa8jfRFpS6EfQvFahb6IdEyhH0KxRB1DigoZMUg9+iLSVlahb2ZzzGyLmVWb2Y2djPm0mW02s01m9kDG9iYzezF9Wd1bhUvnWnr0zSzoUkQkx3T7lUpmVggsAy4EYsAGM1vt7pszxlQANwGz3D1hZmMzfkTS3af3ct3SBfXoi0hnsjnSnwlUu/t2d68HVgDz2o25Fljm7gkAd3+3d8uUnlCPvoh0JpvQnwDsyLgdS2/LNBWYamZPmdmzZjYn474SM6tKb7+so19gZgvTY6pqamp69ACkrf2HGtiXbFCPvoh0KJtvzO5oYtg7+DkVwMeAcmC9mb3f3WuBSe6+08xOBh43s1fc/fU2P8x9ObAcoLKysv3Plh5o6dzR9I6IdCSbI/0YMDHjdjmws4MxD7t7g7u/AWwh9SaAu+9M/3c78AdgxnHWLF2I7dWJWSLSuWxCfwNQYWZTzKwImA+078J5CDgfwMxKSU33bDezUWZWnLF9FrAZ6TOtR/qa3hGRDnQ7vePujWa2CFgLFAL3ufsmM1sKVLn76vR9F5nZZqAJ+Ad332NmHwHuMbNmUm8w38vs+pHeF0vUUTyggNKhRUGXIiI5KJs5fdx9DbCm3babM6478PfpS+aYp4Fpx1+mZCtem2rXVI++iHQkq9DPB/sPNfCthzcFXUbg/vTGXs44cUTQZYhIjgpN6Dc2ORve3Bt0GYEbVFTIxWeeEHQZIpKjQhP6o4YUsf5rfx10GSIiOU0LromIRIhCX0QkQhT6IiIRotAXEYkQhb6ISIQo9EVEIkShLyISIQp9EZEIsdSyObnDzGqAN4Ouo51SYHfQRfRAPtWbT7VCftWbT7VCftWbi7We5O5l3Q3KudDPRWZW5e6VQdeRrXyqN59qhfyqN59qhfyqN59qbU/TOyIiEaLQFxGJEIV+dpYHXUAP5VO9+VQr5Fe9+VQr5Fe9+VRrG5rTFxGJEB3pi4hEiEJfRCRCFPoiIhGi0D9OZnaZmf3YzB42s4uCrqc9MxtiZvena7wi6Hq6k+vPZ3vp53ejmX086Fq6Y2YFZnarmd1pZlcFXU9XzGySma02s/vM7Mag6+mMmZ1sZj8xs1UZ23L6NRfp0E//Qb1rZq+22z7HzLaYWXV3f3Du/pC7XwtcDXymD8vNrK8ndX8SWJWu8dL+qK+9ntQbxPN5rLWm/SOwsn+rbFNXT+qdB0wAGoBYjtc6FXjU3b8AnJGrdbr7dne/pt2PCPw11yV3j+wFmA2cBbyasa0QeB04GSgCXiL1RzcN+HW7y9iM/b4PnJWDdd8ETE+PeSDXn+cgns/jeG7/BphP6g3q47n+3AI3Atelx6zK8VrHAE8AjwOfz9U6M+5flXE98NdcV5fQfDH6sXD3dWY2ud3mmUC1u28HMLMVwDx3/y5w1D/hzcyA7wG/cffn+7bilJ7UTeqIrhx4kYD+ZdeTes3sNfr5+czUw+d2KDCEVEglzWyNuzf3Y7k9rXcHUJ8e09RfNbboYa0NwLfS+6wCfpqjdW7u4EcE/prrSs4VlAMmkHpxtIilt3VmMakjvsvN7Pq+LKwbndX9/4FPmdldwCNBFNaJzurNleczU4e1uvs33P0rwAPAj/s78LvQ1d/CxWZ2J7AuiMI60FmtvwW+bGZ3A38JoK72OqzTzMaka5xhZjel78vV1xxAtI/0O2EdbOv0DDZ3vwO4o+/KyVqHdbv7QeDz/V1MFjqrN1eez0xd/k24+8/6r5SsdPbc1gHt55+D1lmtrwKX93cxXeiszj3A9e025uprDtCRfkdiwMSM2+XAzoBq6Yl8qzuf6s2nWiG/6s2XWvOlzm4p9I+2AagwsylmVkTqg7rVAdeUjXyrO5/qzadaIb/qzZda86XO7gX9SXKQF+BB4G2OtLBdk94+F9hK6tP6bwRdZ77XnU/15lOt+VZvvtSaL3Ue60ULromIRIimd0REIkShLyISIQp9EZEIUeiLiESIQl9EJEIU+iIiEaLQFxGJEIW+iEiEKPRFRCLkPwEpF9mTUNgQEQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc. 10-fold cross on train data=  0.9106231884057971\n",
      "\n",
      "Confusion matrix on test set:\n",
      " [[31  7]\n",
      " [ 1 67]]\n",
      "\n",
      "Best combination of parameters found:  {'C': 1000.0}\n",
      "\n",
      "Accuracy on test set:  0.9245283018867925\n",
      "\n",
      "Number of supports:  65 ( 13 of them have slacks)\n",
      "Prop. of supports:  0.2653061224489796\n"
     ]
    }
   ],
   "source": [
    "Cs = np.logspace(-3, 11, num=15, base=10.0)\n",
    "\n",
    "param_grid = {'C': Cs}\n",
    "grid_search = GridSearchCV(SVC(kernel='poly',degree =3) , param_grid, cv=10)\n",
    "grid_search.fit(X_train,y_train)\n",
    "\n",
    "scores = grid_search.cv_results_['mean_test_score']\n",
    "\n",
    "plt.semilogx(Cs,scores)\n",
    "plt.show()\n",
    "\n",
    "parval=grid_search.best_params_\n",
    "\n",
    "cvacc = cross_val_score(SVC(kernel='poly',degree =3,C=parval['C']) , X=X_train,  y=y_train, cv=10, scoring='accuracy')\n",
    "print('Acc. 10-fold cross on train data= ', cvacc.mean())\n",
    "\n",
    "knc = SVC(kernel='poly',degree =3,C=parval['C']) \n",
    "knc.fit(X_train, y_train)\n",
    "pred=knc.predict(X_test)\n",
    "print(\"\\nConfusion matrix on test set:\\n\",sklearn.metrics.confusion_matrix(y_test, pred))\n",
    "print(\"\\nBest combination of parameters found: \",parval)\n",
    "print(\"\\nAccuracy on test set: \",sklearn.metrics.accuracy_score(y_test, pred))\n",
    "print(\"\\nNumber of supports: \",np.sum(knc.n_support_), \"(\",np.sum(np.abs(knc.dual_coef_)==parval['C']) ,\"of them have slacks)\")\n",
    "print(\"Prop. of supports: \",np.sum(knc.n_support_)/X_train.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confusion matrix shows only 1 more error than the confusion matrix obtained with the quadratic kernel. Given the low number of cases we have in the test set, we cannot conclude that polynomial kernel with degree 2 is better than with degree 3. But remember that when there are several classifiers with a similar performance, we should always choose the simpler one! So we will choose the quadratic polynomial kernel as best polynomial kernel, not because of performance only but because is the better combination of performance and simplicity.\n",
    "\n",
    "## RBF Kernel\n",
    "\n",
    "There's another possibility for the kernel: The RBF kernel. This is the default kernel in the implementation of SVMs in sklearn, so we don't need to explicitely say the kernel used. Let's try it with default parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix on test set:\n",
      " [[30  8]\n",
      " [ 0 68]]\n",
      "\n",
      "Accuracy on test set:  0.9245283018867925\n"
     ]
    }
   ],
   "source": [
    "knc = SVC() \n",
    "knc.fit(X_train, y_train)\n",
    "pred=knc.predict(X_test)\n",
    "print(\"Confusion matrix on test set:\\n\",sklearn.metrics.confusion_matrix(y_test, pred))\n",
    "print(\"\\nAccuracy on test set: \",sklearn.metrics.accuracy_score(y_test, pred))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Promising performance for default parameters. But we have to search for the best parameters. In this case we have two parameters to adjust: the C parameter and the gamma parameter. We will find the best combination using the *GridSearch* method."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAT4AAAEECAYAAACx2Vj7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHcBJREFUeJzt3Xu0JWV95vHvQ2PTXAQaWxFBLiqoiIrSIoqJMcpFJxEvYwI6AV0alom4RuMkalYiDJoJcSVLZYIokh7URFuXOrHNdOgwijoZBbtRBGnCrVVsRRFBRiO3PueZP6o27N5n73Pq7LPrnKpTz2etWr13XX5VB7p/561633p/sk1ERJfsstQXEBGx2JL4IqJzkvgionOS+CKic5L4IqJzkvgionOS+CKic5L4IqJzkvgionOS+KIySRct9TVETMKuS30B0SyS9hu1CXjJYl5LRF2S+GLQT4HvUyS6HpffH7UkVxQxYUl8MWgb8ELbtw5ukPSDJbieiInLM74Y9H5g9Yht713MC4moizItVUR0TW51l4CkR9j+2VJfxyiSngScAhxI8XzvR8AG29fXdL7X2f4fdcSOGCa3ujWTdJ6kNeXntZK2AVdK+r6k5y8g7sl9n/eR9HeSrpH0CUn7LyDu24H1FJ0Z3wA2l58/Kekd48adw3+tKW7EULnVrZmka20/tfx8OfAntjdLOgL4hO21Y8b9pu1nlp8vBn4MfAR4BfB82y8bM+6NwFNsPzCwfiVwne3Dx4x7zahNwBG2dxsnbsQ4cqtbv4dJ2tX2DmB325sBbN8oaVL/2NfaPrr8/D5JZywg1jTwGIohLf0OKLeNa3/gJOCugfUCvraAuBHzlsRXvwuAjZLOAy6V9H7gc8ALgasXEPdRkv6IInHsLUl+qPm+kEcYbwG+KOkmoDd85WDgCcBZC4j7T8Betmf8zJK+vIC4EfOWW91FIOkFwBuBIyh+2WwH/hFYN3hLOY+YZw+s+qDtn0p6NPBe26cv4Hp3AY6l6NxQeb2bbU+NGzOiSZL4YgZJ4qHE1+vV/YYX+JelrrhznPNJtv+trvjRTkl8i0DSScDL2Pkf/OdtX9q0uJJOBD4I3AT8sFx9EMWt7h/a/pcmxa1w3lttH1xH7GivJL6alc/0jgA+RnHLCMU/+NOBm2z/54bFvR54se3vDaw/DNho+8lNilvGOH/UJuAM23uPGzuWpyS+mkm60fYRQ9YLuHEBw0PqinsT8OSyF7p//Upgq+0nNCluGeMXwNuA+4Zs/hvba8aNHctTenXrd6+kY21/Y2D9s4B7Gxh3HbBZ0noe6tV9LHAq8HcNjAvFIOvv2J4xLEbSOQuMHSOc9II9/bM7q/V3XXXNfZtsnzz3nosjLb6aSXomcCHwcB66JX0s8P8onm1d1aS4ZewjgZeyc6/uBttbx41Zc9z9gHtt/2ohcWJ+jnn6Kl+56aBK+z7sgFuuGnewfh3S4quZ7W8Czy6HmTz4D972j5sYt4y9FdhaJhTbHhx03LS4d/Y+Tzp2zMZMeSFj2pdO3tVdJLZ/bPsq21soxvQ1Mq6kgyWtl3Q7cCXwDUm3l+sObVrcgdg/LWNvnlTsWc55bR1x28TANK60NE1afEvjpcA5DY37KYo5+V7TG7AsaQXwKorJC45rWNzaYkt6xahNwKPHibncTC/oLcalk8S3NDT3LksWd43tT/WvKJPJeknvbmDcOmN/CvgHGNpkWbWAuMuCMVMt7SNI4lsaxzQ47lWSPgh8lJ17X88AvtXAuHXGvgb4a9vfGdwg6UULiLtsNPE2tookvkVSTkN1IbC/7aMkPQ14qe33NCzu6cDrKebI63Wa/AD4AgsbdlJX3Dpjv4Wil3yYly8gbm0k7QO8k+KNnkeWq28HPg+cZ/vnkzqXgamWJr4MZ1kkkr4C/DHwYdvPKNd9x/ZRTYwb7SRpE/Al4KO9Hv6y5/8M4EW2T5jUuZ7+9JXetLHa2PADDrqtUcNZ0qu7ePYYMth4x9A9mxF3Bkm/1aa4dcau85oX6FDbf9U/rKns+f8riunFJmq64tI0SXyL5w5Jj6d8UC7pPwK3NTjuMM9qWdw6Y9d5zQvxfUl/0l9+QNL+ZUmBiZYHNWaq4tI0eca3eN4EXAQ8SdIPge8Cr2liXI0uNjQ4B2Aj4tYZu85rrsnvAu8AviKpVwD+J8AGiuE9k2OYal5OqyQtvkVie5vtF1E8cH6S7ecBv9m0uKqp2FBdcdt6zXOc93XjHmv7Lttvt/0k2/uVy5Ntv52iw2NiigHM7bzVTefGEqprrriFxFV9xYZqidvWa57jvI37ezHMU5+20p+r2LlxxGOb1bmRW92aafbqYgspA1lLXOorNlRX3Dpj13bNLfx7MYOB6Za2m5L46ldXdbG64tZVbKiuuHXGrvOa2/b3YgYD97f0aVkSX/3qqi5WS1zbl5aDoidabKiuuG29Zlr292KUadf19mW98owvIsZy5NNW+u//qdpcDccc8oM844uI9jNiqqW3uu286paTdGbi1hu7bXHrjF3nNU9blZamSeJbGnX9RWxb3Dpjty1unbFriVtMUqBKS9PkVjcixiSm3M62UxLfhKzUbl7FnpX2XcUe7K39Jt6r1La4dcaeb9yHH1ltWN4+B+zOgU/Zt3LcX2ytnhia8N/iXv6d+31fpSaagQdYsaBrWypJfBOyij15tl641JcRY3r+p+6pJe5XnrZ7LXHrcqW/WHlfOy2+iOig6QY+v6siiS8ixlJ0bqTFFxGdklvdiOiYYlqqJL6I6BAj7nd6dSOiY6ZbeqvbzqteAEknS7pB0s3DZtGV9OuSvilpR1m/IiKG6HVuVFmapnlXVCNJK4ALgBcDRwKnSTpyYLdbgdcCn1jcq4toFyOmXG2pokKj5BBJX5R0jaQvSzqob9sZkm4qlzPmOlenEh/FvGo3l3Uq7qeop3BK/w62v2f7GppZKiCiUabZpdIyl4qNkr8GPmb7acC5wF+Wx+4HnA08m+Lf+NmSVs92vq4lvgPZucTe9nJdRMyTDVPepdJSwZyNEoqE2Hu15PK+7ScBl9m+0/ZdwGXAybOdrGuJb1ibe+x3IyWdKWmLpC0PcN8CLiuijcR0xQVY0/u3Ui6DM8ZUaZR8G3hl+fnlwMMlPaLisTvpWq/uduCxfd8PoqiTOhbbF1HUtK3tJf6IpjJwvyunkDvmmIG5SqPkvwB/K+m1wFeBHwI7Kh67k64lvs3A4ZIOo/iPdirw6qW9pIh2MhOdZHTORontHwGvAJC0F/BK23dL2g78xsCxX57tZJ261bW9g6I61ibgeuDTtq+TdK6klwJIelb5H/JVwIclXbd0VxzRbBMczvJgo6SsW3wqsKF/B0lrJPWCvRNYV37eBJwoaXXZqXFiuW6krrX4sL0R2Diw7l19nzdT/MaIiFkUdXUn03ayvUNSr1GyAljXa5QAW2xvoGjV/aUkU9zqvqk89k5J76ZIngDn2r5ztvN1LvFFxKRMdlr5Co2SzwCfGXHsOh5qAc4piS8ixjLJFt9iS+KLiLE1sZBQFUl8ETEWWzww3c4U0s6rjoglV8zHlxZfRK32+ddH1Bb7bY+4tJa4p32/vjd63njI82qLXU1mYI6Ijik6N9Lii4iOaeJce1Uk8UXEWCb8ytqiSuKLiLGl2FBEdIoND0wn8UVEhxS3ukl8EdExbX1zo53pegEkrZN0u6TvjNguSeeXBU+ukfTMxb7GiDboDWepsjRN5xIfcAmzz8f/YuDwcjkTuHARrimihYpb3SpL0zTvimpm+6vAbHN1nUJRycm2rwD2lXTA4lxdRLvMo+ZGo+QZ30yjCpfctjSXE9FMRZW15iW1KpL4ZqpcuKSsFHUmwCr2qPOaIhrHiB3TK5b6MsaSxDdT5UpsqbIWXdfE29gqOveMr4INwOll7+5xwN22c5sbMaDNvbqda/FJ+iRF0ZI1ZTW1s4GHAdj+EMWc/y8BbgZ+Bbxuaa40ovma2GNbRecSn+3T5thuyupNETGLhrbmquhc4ouIycgMzBHRSWnxRUSnGNiR2VkioksyEWlEdFKe8UVEtzjP+CIetPtX9q8l7gcO/mwtcQF20161xN1vxY5a4gLssmrVxGPq3uqJbNJV1iSdDHwAWAFcbPu8ge0HAx8F9i33eYftjZIOBa4Hbih3vcL2G2c7VxJfRIxtUolP0grgAuAEitdGN0vaYHtr325/Bnza9oWSjqR42eDQctstto+uer4kvogYixFTk+vVPRa42fY2AEnrKaaI6098BvYuP+/DiHfoq2hnX3RENMIE5+MbNR1cv3OA/1S+aroReHPftsMkfUvSVyT92lwnS+KLiLHY85qkYI2kLX3LmQPhqkwHdxpwie2DKN6n/7ikXSjmyjzY9jOAPwI+IWlvZpFb3YgYm6s/47vD9tpZtleZDu71lGUjbH9d0ipgje3bgfvK9VdJugU4Atgy6mRp8UXEmKq19ip2gGwGDpd0mKSVwKkUU8T1uxV4IYCkJwOrgJ9KemTZOYKkx1HUy9k228nS4ouIsc2jxTdHHO+QdBawiWKoyjrb10k6F9hiewPwNuAjkt5KcRv8WtuW9OvAuZJ2AFPAG23PVldn+SY+SeuA3wJut31UuW4/4FMUXeDfA37H9l1Djj2Dousc4D22P7oY1xzRJpMex2d7I0WnRf+6d/V93gocP+S4zwLzGuS5nG91L2FmGcl3AF+0fTjwxfL7TsrkeDbwbIou9rMlra73UiNaqCw2VGVpmmWb+EaUkTyFYuQ35Z8vG3LoScBltu8sW4OXMXsd3ohOMsWtbpWlaZbtre4I+/fqZ9i+TdKjhuxTZTwRkCpr0XWZnWU5qVxeMlXWouvc0r/1y/ZWd4SfSDoAoPzz9iH7VC4vGdF1bb3V7Vri2wCcUX4+A/j8kH02ASdKWl12apxYrouIPnYSX+OUZSS/DjxR0nZJrwfOA06QdBPFLBDnlfuulXQxQDn+590UAyo3A+fONSYooqtSV7dhZikj+cIh+24B3tD3fR2wrqZLi1g2pqebl9SqWLaJLyLqZZp5G1tFEl9EjK2lnbpJfBExJk/uXd3FlsQXEeNraZMviS8ixpYWX7ROXdXQHrnql7XErdPd0/cs9SW0Ulvf3Ejii4ix2ODJFRtaVEl8ETG2tPgionuS+CKiWzKAOSK6KC2+iOiUDGCOiE5qaYuvnX3RJUnrJN0u6Tt96/aTdJmkm8o/V5frJel8STdLukbSM0fEPEbSteV+50tq56+0iMVgVVsaptWJj/lVUnsxRaHhwynqZFw4IuaF5fbevik0FDGKKy4N0+rEN89KaqcAH3PhCmDf3jT0PeX3vW1/3baBjzG8EltEmLT4GmSnSmpAr5JaleppB5brZ9snIkrF9PNzL03Tpc6NKtXTKldYg5SXjGjibWwVy7HFN6qSWpXqadvL9bPt8yDbF9lea3vtw9htwRce0Tq51W2MUZXUNgCnl727xwF3926Je8rvv5B0XNmbezrDK7FFhEHT1ZamaXXim08lNWAjsA24GfgI8Id9ca7uC/sHwMXlfrcA/1z3zxHRThVbexVbfJJOlnRDOZTsHUO2HyzpcknfKoekvaRv2zvL426QdNJc52r1M755VlIz8KYRcY7u+7wFOGoiFxix3E3oGZ+kFcAFFI2V7cBmSRtsb+3b7c+AT9u+UNKRFI2ZQ8vPpwJPAR4D/G9JR9ieGnW+Vrf4ImKJTW4c37HAzba32b4fWE8xBG3wbHuXn/fhoefvpwDrbd9n+7sUd2vHznayVrf4ImKJVW/xrZG0pe/7RbYv6vs+bLjZswdinAP8i6Q3A3sCL+o79oqBY2cdhpbEFxHj6Q1gruYO22tn2V5lKNlpwCW2/0bSc4CPSzqq4rE7SeKLiLFpcuP4qgw3ez3lK6S2vy5pFbCm4rE7mfUZn6QnSDp+yPpfk/T42Y6NiA6Y3DO+zcDhkg6TtJKis2LDwD63UnZcSnoysAr4abnfqZJ2k3QYxTv235jtZHO1+N4P/OmQ9feU2357juNjgeqqhAawcsWOWuIev89NtcS9c3pFLXEB9kg331gm1eKzvUPSWcAmYAWwzvZ1ks4FttjeALwN+Iikt1Kk09eWozWuk/RpYCuwA3jTbD26MHfiO9T2NUMucoukQ+f5s0XEcjPBtzJsb6QYotK/7l19n7cCM+5Ay21/AfxF1XPNlfhWzbJt96oniYhlqKFTTlUxVwN/s6TfH1xZviFxVT2XFBGt0dL5+OZq8b0F+J+SXsNDiW4tsBJ4eZ0XFhHNN8Fe3UU1a+Kz/RPguZJewEOvcf0v21+q/coiovmWY+LrsX05cHnN1xIRLSI3c+aVKjKAOSLG18C59qpoxeilSVVTk3RGuf9Nks4Yca6hcSNiiJZ2brQi8TGBamqS9gPOpnjx+Vjg7BFJbVTciBggV1uaphWJb0LV1E4CLrN9p+27gMsYXjpyVNyIGNTSFl+bn/HtVE1N0lzV1KpUWZstbkT0a2hrroo2J75RRk1RM++pa+Y8UaqsRde1NPG14lZ3hPlWU6s6dc2ouDOkylp0XYoNLb75VlPbBJwoaXXZqXFiua5q3IhYJlqR+CZRTc32ncC7Keb92gycW65D0sWSerPDjoobEYPSuVGfCVZTWwesG7L+DX2ffzYsbkQMSOdGRHRSEl9EdE4SX0R0icitbkR0TWZniYhOSosvIjonia/bdnniruz+kcmXgqyrBCTAEXuNfCllQR6z6121xI3myTO+iOieJL6I6JSGvpVRRRJfRIwtvboR0Tl5xhcR3dPSxNeK2VkiooGqzsxSMTlKOlnSDWWhsBm1biS9T9LV5XKjpJ/3bZvq27ZhrnOlxRcRYxHDpzUfK5a0AriAYiq47cBmSRtsb+3tY/utffu/GXhGX4h7bB9d9XyNavHVXUZS0jGSri2POV/SjP9vs8WNiAGTa/EdC9xse5vt+4H1FIW/RjkN+OS4l92oxEf9ZSQvLPftHTesytrQuBEx0wTLS1YtBoakQ4DDgC/1rV4laYukKyTNWRmxUYmvzjKS5ba9bX+9nKz0YwwvHTkqbkQMmq64wJoyMfWWMwcizacY2KnAZ2xP9a072PZa4NXA+yU9frbLbsMzvkmVkTyw/Dy4ftCo428b3LG/ytru++9V/SeKWA7mNwPzHWViGqVqMTAoEt9Os6zb/lH55zZJX6Z4/nfLqJM1qsU3T/MtI1n1N0rl3zz9VdZW7rv7yAuNWLYm94xvM3C4pMMkraRIbjN6ZyU9EVhNUYOnt261pN3Kz2uA44Gtg8f2a0Pim1QZye3l58H1g+bzmyei0yb1jM/2DuAsisqH1wOftn2dpHMlvbRv19OA9eXjqp4nA1skfRu4HDivvzd4mDbc6vbKPZ7HzDKSZ0laT9GRcXd5K7wJ+G99HRonAu+0faekX5QlJ68ETgf++4jzzYhb1w8X0WoTHMBseyNFlcT+de8a+H7OkOO+Bjx1PudqVOIry0j+BsWD0O0UvbPnAZ8uS0reCryq3H0j8BKKMpK/Al4HRRlJSb0yktBXRhL4A4qe492Bfy4XJL2xPPZDo+JGxEx5ZW0CFqGM5BbgqCHrP1QlbkT0yewsEdE1IrOzREQXpcUXEV0jtzPzJfFFxHjyjC8iuii9uh0nXGtFtDrsscv9S30J0XZJfBHRNWnxRUS3OMNZIqKL0uKLiC4RudWNiC7KOL6I6Jq0+CKiW1o8gHlJJiJdqmpqo84x5PqGxo2InWm62tI0SzUD8yUsTTW1Ued40BxxI6JPEt88LGE1tVHn6Dc07gJ/5IjlxxSdG1WWhmnSM77FqKY26hz9Ktf3jOi6dG7Up65qavM938wdU14yuq6lia9JVdYWo5raqHP0q1xlrb+85G77rprzB4xYTnoDmCdRZW2xNSnx9aqpwcxqaqeXvbvH8VDVs03AiWVNzdUU1dQ2ldt+Iem4sjf39IFYw87Rb2jcif6kEctB1ed7ecZXWKpqaqPOIWkt8Ebbb5gjbkT0aWKPbRVLkviWsJraz0acYwvwhrniRsTOmngbW0UbOjciookMTLcz8yXxRcT42pn3kvgiYny51Y2I7mlgj20VTRrOEhEtM8lxfJJOlnRDObnIsPfo3yfp6nK5UdLP+7bNa2KRtPgiYiwyaEKdG5JWABcAJ1C8RLBZ0gbbW3v72H5r3/5vBp5Rfu5NLLKW4qnjVeWxd406XxLfhOwis2rFAxOPu/eu9008Zs/jdhv24krEPExuHN+xwM22twFIWk8xqcjWEfufRpHsoG9ikfLY3sQinxx1siS+iBibqj/jWyNpS9/3i2xf1Pd92OQgzx56TukQ4DDgS7McO+vEIkl8ETGe+c3AfIfttbNsn8/kIqcCn7E9NcaxQDo3ImJsE31Xt/LkIBSJr/82dj7HAkl8EbEAE+zV3QwcLukwSSspktuGGeeTngisBr7et3reE4vkVjcixjehcXy2d0g6iyJhrQDW2b5O0rnAFtu9JHgasL58h7937LwnFknii4jxGDQ1uQHMtjdSzMbUv+5dA9/PGXHsvCYWqe1Wt2mV1GY7x8B1D40bEUO44tIwdT7ju4RmVVIbeo4hRsWNiAGyKy1NU1via2AltVHneNAccSNiUEtnYF7sXt2dqpwBtVdSq3COfrPFjYh+pnhzo8rSME3p3FiMSmpVjplX3P4qa3s8es85Th+xvIhm3sZWsdgtvqWspFZlkONscWfor7K2KlXWootyq1vJUlZSG3WOB80RNyL6GZhytaVharvVbVoltVHnKK/1attHzxE3Iga09Va3tsTXwEpqs53j6L7PQ+NGxBBJfBHRLc18fldFEl9EjMck8UVEBzVwjF4VSXwRMbZ0bkREtxiYameTL4kvIsaUzo3Ou/PffnbHPxy37vsVd18D3FHDZcwr7qjpaRYad54a8d+iAXHHiP33dcQ9pPr5SeLrOtuPrLqvpC1zFF4ZS9vi1hm7bXHrjF3nNSfxRUS3GJhQQfHFlsQXEWMyOJ0bUd1Fc+/Sibh1xm5b3Dpj1xO3xb26ckvv0SNiae2zcn8/d/9TK+176fbzr6rtOeMY0uKL2kn6c+A1FDNg3wFcBdxNMYnrSooZc37P9q8kXQLcAzyJoofxdRTTiz0HuNL2a8uYvwQuAF4E3AX8KfBe4GDgLbY3SDoU+DjQmyX2LNtfq/en7ZiWNpxSUDxqJWkt8ErgGcArgN5v/c/ZfpbtpwPXA6/vO2w18JvAW4EvAO8DngI8VVJvJp09gS/bPgb4BfAe4ATg5cC55T63AyfYfibwu8D5tfyQnVVxEtIGJse0+KJuzwM+b/seAElfKNcfJek9wL7AXhSTzvZ8wbYlXQv8xPa15bHXAYcCVwP3A5eW+18L3Gf7gfKYQ8v1DwP+tkyWU8AR9fyIHWVgup3P+JL4om6j6hJfArzM9rclvZZi0tqe+8o/p/s+9773/s4+4IceUD+4n+1pSb193gr8BHg6xd3NvWP/FDFcA1tzVeRWN+r2r8BvS1olaS/gP5TrHw7cJulhFM//6rAPcJvtaeD3gBU1nae7cqsbMZPtzZI2AN8Gvg9soejY+HPgynLdtRSJcNI+CHxW0quAy4F/r+Ec3WXjqamlvoqxZDhL1E7SXrZ/KWkP4KvAmba/udTXFQuzz66P9HP2flmlfTfddfGcw1kknQx8gKJlfrHt84bs8zvAORRPGL9t+9Xl+imKX6AAt9p+6WznSosvFsNFko4EVgEfTdJbRibUcJK0gmJ40gkUZV43S9pge2vfPocD7wSOt32XpEf1hbinv3bOXJL4ona938qxzNiT7NU9FrjZ9jYASeuBU4Ctffv8PnCB7buK0/v2GVEqSudGRIxvcp0bB1IMcO/ZXq7rdwRwhKT/K+mK8ta4Z5WkLeX6Oe+/0+KLiLG5eotvjaQtfd8vst3/DvGwYU+DGXNX4HCKoU8HAf9H0lG2fw4cbPtHkh4HfEnStbZvGXUxSXwRMaZ5DVW5Y47Oje3AY/u+HwT8aMg+V9h+APiupBsoEuFm2z8CsL1N0pcp3hQamfhyqxsR4zEwNVVtmdtm4HBJh0laCZwKbBjY5x+BFwBIWkNx67tN0mpJu/WtP56dnw3OkBZfRIzFgCc0EantHZLOonh1cQWwzvZ1ks4FttjeUG47UdJWilcQ/9j2zyQ9F/iwpGmKxtx5/b3Bw2QcX0SMZW/t5+N2PbHSvpft+FSmpYqI5WFSLb7FlhZfRIxF0qUUFdyquMP2yXPvtjiS+CKic9KrGxGdk8QXEZ2TxBcRnZPEFxGdk8QXEZ2TxBcRnZPEFxGdk8QXEZ2TxBcRnfP/ARQGDWLLIZuiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best combination of parameters found:  {'C': 10.0, 'gamma': 0.1}\n",
      "\n",
      "Acc. 10-fold cross on train data=  0.9469710144927538\n"
     ]
    }
   ],
   "source": [
    "# Values we will test for each parameter. When observin results, consider the limits of the \n",
    "# values tested and increase them if necessary \n",
    "gammas = [0.000001,0.00001, 0.0001,0.001,0.01,0.1,1,10]\n",
    "Cs = np.logspace(-1, 6, num=8, base=10.0)\n",
    "\n",
    "param_grid = {'C': Cs, 'gamma' : gammas}\n",
    "grid_search = GridSearchCV(SVC(), param_grid, cv=10)\n",
    "grid_search.fit(X_train,y_train)\n",
    "parval=grid_search.best_params_\n",
    "\n",
    "# We'll show in a grid, the accuracy for each combination of parameters tester\n",
    "scores = grid_search.cv_results_['mean_test_score']\n",
    "scores = np.array(scores).reshape(len(param_grid['C']), len(param_grid['gamma']))\n",
    "\n",
    "plt.matshow(scores)\n",
    "plt.xlabel('gamma')\n",
    "plt.ylabel('C')\n",
    "plt.colorbar()\n",
    "plt.xticks(np.arange(len(param_grid['gamma'])), param_grid['gamma'],rotation='vertical')\n",
    "plt.yticks(np.arange(len(param_grid['C'])), param_grid['C'])\n",
    "plt.show()\n",
    "parval=grid_search.best_params_\n",
    "print(\"\\nBest combination of parameters found: \",parval)\n",
    "\n",
    "\n",
    "cvacc = cross_val_score(SVC(C=parval['C'], gamma=parval['gamma']) , X=X_train,  y=y_train, cv=10, scoring='accuracy')\n",
    "print('\\nAcc. 10-fold cross on train data= ', cvacc.mean())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This picture show for each combination of parameters the accuracy obtained in a 10-fold cross-validation. Notice the relation between C and gamma. \n",
    "\n",
    "Let's see the performance of the best parameters found on the test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix on test set:\n",
      " [[34  4]\n",
      " [ 0 68]]\n",
      "\n",
      "Accuracy on test set:  0.9622641509433962\n",
      "\n",
      "Number of supports:  61 ( 9 of them have slacks)\n",
      "Prop. of supports:  0.24897959183673468\n"
     ]
    }
   ],
   "source": [
    "# Let's apply the best combination of parameters found to the test set\n",
    "\n",
    "knc = SVC(C=parval['C'], gamma=parval['gamma']) \n",
    "knc.fit(X_train, y_train)\n",
    "pred=knc.predict(X_test)\n",
    "\n",
    "print(\"Confusion matrix on test set:\\n\",sklearn.metrics.confusion_matrix(y_test, pred))\n",
    "print(\"\\nAccuracy on test set: \",sklearn.metrics.accuracy_score(y_test, pred))\n",
    "print(\"\\nNumber of supports: \",np.sum(knc.n_support_), \"(\",np.sum(np.abs(knc.dual_coef_)==parval['C']) ,\"of them have slacks)\")\n",
    "print(\"Prop. of supports: \",np.sum(knc.n_support_)/X_train.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Wow. Best performance obtained so far using any method, even better than results obtained with meta-methods. \n",
    "\n",
    "**Observations about number and percentage of supports vectors:** \n",
    "It is know that percentage of supports of an SVM is a lower bound for the leave-one-out error. In general, an SVM with a lot of supports will be an overfitted SVM. A percentage of supports higher than 50% should be considered suspicious. If this happens, try to use other kernels. As a rule of thumb, a good SVM has a percentatge of supports vectors about 20-40% of the data (but that depends on a lot of things).\n",
    "\n",
    "In our case all SVM have a low number of supports. And notice that the machine with a higher performance is the one with a lower number of supports (24.9%). That's not a coincidence but something common in SVMs."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Appendix: Performance of meta-methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix on test set:\n",
      " [[34  4]\n",
      " [ 1 67]]\n",
      "\n",
      "Accuracy on test set:  0.9528301886792453\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "clf=ExtraTreesClassifier(n_estimators=200,random_state=1).fit(X_train, y_train)\n",
    "pred=clf.predict(X_test)\n",
    "print(\"Confusion matrix on test set:\\n\",sklearn.metrics.confusion_matrix(y_test, pred))\n",
    "print(\"\\nAccuracy on test set: \",sklearn.metrics.accuracy_score(y_test, pred))\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix on test set:\n",
      " [[34  4]\n",
      " [ 2 66]]\n",
      "\n",
      "Accuracy on test set:  0.9433962264150944\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "\n",
    "clf=AdaBoostClassifier(n_estimators=200,random_state=1).fit(X_train, y_train)\n",
    "pred=clf.predict(X_test)\n",
    "print(\"Confusion matrix on test set:\\n\",sklearn.metrics.confusion_matrix(y_test, pred))\n",
    "print(\"\\nAccuracy on test set: \",sklearn.metrics.accuracy_score(y_test, pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix on test set:\n",
      " [[34  4]\n",
      " [ 2 66]]\n",
      "\n",
      "Accuracy on test set:  0.9433962264150944\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import BaggingClassifier\n",
    "\n",
    "clf=BaggingClassifier(n_estimators=200,max_features=0.35,random_state=1).fit(X_train, y_train)\n",
    "pred=clf.predict(X_test)\n",
    "print(\"Confusion matrix on test set:\\n\",sklearn.metrics.confusion_matrix(y_test, pred))\n",
    "print(\"\\nAccuracy on test set: \",sklearn.metrics.accuracy_score(y_test, pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}