svm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read the CSV and Perform Basic Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>koi_disposition</th>\n",
       "      <th>koi_fpflag_nt</th>\n",
       "      <th>koi_fpflag_ss</th>\n",
       "      <th>koi_fpflag_co</th>\n",
       "      <th>koi_fpflag_ec</th>\n",
       "      <th>koi_period</th>\n",
       "      <th>koi_period_err1</th>\n",
       "      <th>koi_period_err2</th>\n",
       "      <th>koi_time0bk</th>\n",
       "      <th>koi_time0bk_err1</th>\n",
       "      <th>...</th>\n",
       "      <th>koi_steff_err2</th>\n",
       "      <th>koi_slogg</th>\n",
       "      <th>koi_slogg_err1</th>\n",
       "      <th>koi_slogg_err2</th>\n",
       "      <th>koi_srad</th>\n",
       "      <th>koi_srad_err1</th>\n",
       "      <th>koi_srad_err2</th>\n",
       "      <th>ra</th>\n",
       "      <th>dec</th>\n",
       "      <th>koi_kepmag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CONFIRMED</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9.488036</td>\n",
       "      <td>2.775000e-05</td>\n",
       "      <td>-2.775000e-05</td>\n",
       "      <td>170.538750</td>\n",
       "      <td>0.002160</td>\n",
       "      <td>...</td>\n",
       "      <td>-81.0</td>\n",
       "      <td>4.467</td>\n",
       "      <td>0.064</td>\n",
       "      <td>-0.096</td>\n",
       "      <td>0.927</td>\n",
       "      <td>0.105</td>\n",
       "      <td>-0.061</td>\n",
       "      <td>291.93423</td>\n",
       "      <td>48.141651</td>\n",
       "      <td>15.347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CONFIRMED</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>54.418383</td>\n",
       "      <td>2.479000e-04</td>\n",
       "      <td>-2.479000e-04</td>\n",
       "      <td>162.513840</td>\n",
       "      <td>0.003520</td>\n",
       "      <td>...</td>\n",
       "      <td>-81.0</td>\n",
       "      <td>4.467</td>\n",
       "      <td>0.064</td>\n",
       "      <td>-0.096</td>\n",
       "      <td>0.927</td>\n",
       "      <td>0.105</td>\n",
       "      <td>-0.061</td>\n",
       "      <td>291.93423</td>\n",
       "      <td>48.141651</td>\n",
       "      <td>15.347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>FALSE POSITIVE</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19.899140</td>\n",
       "      <td>1.494000e-05</td>\n",
       "      <td>-1.494000e-05</td>\n",
       "      <td>175.850252</td>\n",
       "      <td>0.000581</td>\n",
       "      <td>...</td>\n",
       "      <td>-176.0</td>\n",
       "      <td>4.544</td>\n",
       "      <td>0.044</td>\n",
       "      <td>-0.176</td>\n",
       "      <td>0.868</td>\n",
       "      <td>0.233</td>\n",
       "      <td>-0.078</td>\n",
       "      <td>297.00482</td>\n",
       "      <td>48.134129</td>\n",
       "      <td>15.436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>FALSE POSITIVE</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.736952</td>\n",
       "      <td>2.630000e-07</td>\n",
       "      <td>-2.630000e-07</td>\n",
       "      <td>170.307565</td>\n",
       "      <td>0.000115</td>\n",
       "      <td>...</td>\n",
       "      <td>-174.0</td>\n",
       "      <td>4.564</td>\n",
       "      <td>0.053</td>\n",
       "      <td>-0.168</td>\n",
       "      <td>0.791</td>\n",
       "      <td>0.201</td>\n",
       "      <td>-0.067</td>\n",
       "      <td>285.53461</td>\n",
       "      <td>48.285210</td>\n",
       "      <td>15.597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CONFIRMED</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.525592</td>\n",
       "      <td>3.761000e-06</td>\n",
       "      <td>-3.761000e-06</td>\n",
       "      <td>171.595550</td>\n",
       "      <td>0.001130</td>\n",
       "      <td>...</td>\n",
       "      <td>-211.0</td>\n",
       "      <td>4.438</td>\n",
       "      <td>0.070</td>\n",
       "      <td>-0.210</td>\n",
       "      <td>1.046</td>\n",
       "      <td>0.334</td>\n",
       "      <td>-0.133</td>\n",
       "      <td>288.75488</td>\n",
       "      <td>48.226200</td>\n",
       "      <td>15.509</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  koi_disposition  koi_fpflag_nt  koi_fpflag_ss  koi_fpflag_co  koi_fpflag_ec  \\\n",
       "0       CONFIRMED              0              0              0              0   \n",
       "1       CONFIRMED              0              0              0              0   \n",
       "2  FALSE POSITIVE              0              1              0              0   \n",
       "3  FALSE POSITIVE              0              1              0              0   \n",
       "4       CONFIRMED              0              0              0              0   \n",
       "\n",
       "   koi_period  koi_period_err1  koi_period_err2  koi_time0bk  \\\n",
       "0    9.488036     2.775000e-05    -2.775000e-05   170.538750   \n",
       "1   54.418383     2.479000e-04    -2.479000e-04   162.513840   \n",
       "2   19.899140     1.494000e-05    -1.494000e-05   175.850252   \n",
       "3    1.736952     2.630000e-07    -2.630000e-07   170.307565   \n",
       "4    2.525592     3.761000e-06    -3.761000e-06   171.595550   \n",
       "\n",
       "   koi_time0bk_err1  ...  koi_steff_err2  koi_slogg  koi_slogg_err1  \\\n",
       "0          0.002160  ...           -81.0      4.467           0.064   \n",
       "1          0.003520  ...           -81.0      4.467           0.064   \n",
       "2          0.000581  ...          -176.0      4.544           0.044   \n",
       "3          0.000115  ...          -174.0      4.564           0.053   \n",
       "4          0.001130  ...          -211.0      4.438           0.070   \n",
       "\n",
       "   koi_slogg_err2  koi_srad  koi_srad_err1  koi_srad_err2         ra  \\\n",
       "0          -0.096     0.927          0.105         -0.061  291.93423   \n",
       "1          -0.096     0.927          0.105         -0.061  291.93423   \n",
       "2          -0.176     0.868          0.233         -0.078  297.00482   \n",
       "3          -0.168     0.791          0.201         -0.067  285.53461   \n",
       "4          -0.210     1.046          0.334         -0.133  288.75488   \n",
       "\n",
       "         dec  koi_kepmag  \n",
       "0  48.141651      15.347  \n",
       "1  48.141651      15.347  \n",
       "2  48.134129      15.436  \n",
       "3  48.285210      15.597  \n",
       "4  48.226200      15.509  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"cumulative.csv\")\n",
    "df = df.drop(columns=[\"rowid\", \"kepid\", \"kepoi_name\", \"kepler_name\", \"koi_pdisposition\", \"koi_score\", \"koi_tce_delivname\"])\n",
    "# Drop the null columns where all values are null\n",
    "df = df.dropna(axis='columns', how='all')\n",
    "# Drop the null rows\n",
    "df = df.dropna()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create a Train Test Split\n",
    "\n",
    "Use `koi_disposition` for the y values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape:  (8744, 40) (8744,)\n"
     ]
    }
   ],
   "source": [
    "X = df.drop(\"koi_disposition\", axis=1)\n",
    "y = df[\"koi_disposition\"]\n",
    "#.values.reshape(-1, 1)\n",
    "\n",
    "print(\"Shape: \", X.shape, y.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>koi_fpflag_nt</th>\n",
       "      <th>koi_fpflag_ss</th>\n",
       "      <th>koi_fpflag_co</th>\n",
       "      <th>koi_fpflag_ec</th>\n",
       "      <th>koi_period</th>\n",
       "      <th>koi_period_err1</th>\n",
       "      <th>koi_period_err2</th>\n",
       "      <th>koi_time0bk</th>\n",
       "      <th>koi_time0bk_err1</th>\n",
       "      <th>koi_time0bk_err2</th>\n",
       "      <th>...</th>\n",
       "      <th>koi_steff_err2</th>\n",
       "      <th>koi_slogg</th>\n",
       "      <th>koi_slogg_err1</th>\n",
       "      <th>koi_slogg_err2</th>\n",
       "      <th>koi_srad</th>\n",
       "      <th>koi_srad_err1</th>\n",
       "      <th>koi_srad_err2</th>\n",
       "      <th>ra</th>\n",
       "      <th>dec</th>\n",
       "      <th>koi_kepmag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6274</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>492.252990</td>\n",
       "      <td>0.011050</td>\n",
       "      <td>-0.011050</td>\n",
       "      <td>520.860000</td>\n",
       "      <td>0.01800</td>\n",
       "      <td>-0.01800</td>\n",
       "      <td>...</td>\n",
       "      <td>-152.0</td>\n",
       "      <td>4.478</td>\n",
       "      <td>0.055</td>\n",
       "      <td>-0.283</td>\n",
       "      <td>0.936</td>\n",
       "      <td>0.383</td>\n",
       "      <td>-0.089</td>\n",
       "      <td>292.13519</td>\n",
       "      <td>38.926491</td>\n",
       "      <td>13.465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3541</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>49.567408</td>\n",
       "      <td>0.000187</td>\n",
       "      <td>-0.000187</td>\n",
       "      <td>169.542270</td>\n",
       "      <td>0.00322</td>\n",
       "      <td>-0.00322</td>\n",
       "      <td>...</td>\n",
       "      <td>-111.0</td>\n",
       "      <td>4.474</td>\n",
       "      <td>0.059</td>\n",
       "      <td>-0.110</td>\n",
       "      <td>0.927</td>\n",
       "      <td>0.122</td>\n",
       "      <td>-0.066</td>\n",
       "      <td>290.67313</td>\n",
       "      <td>38.691010</td>\n",
       "      <td>13.597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8659</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>452.100930</td>\n",
       "      <td>0.054070</td>\n",
       "      <td>-0.054070</td>\n",
       "      <td>460.488400</td>\n",
       "      <td>0.05070</td>\n",
       "      <td>-0.05070</td>\n",
       "      <td>...</td>\n",
       "      <td>-214.0</td>\n",
       "      <td>4.456</td>\n",
       "      <td>0.066</td>\n",
       "      <td>-0.281</td>\n",
       "      <td>0.956</td>\n",
       "      <td>0.370</td>\n",
       "      <td>-0.103</td>\n",
       "      <td>292.10245</td>\n",
       "      <td>37.566910</td>\n",
       "      <td>15.107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4851</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>18.645329</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>-0.000002</td>\n",
       "      <td>147.207512</td>\n",
       "      <td>0.00007</td>\n",
       "      <td>-0.00007</td>\n",
       "      <td>...</td>\n",
       "      <td>-175.0</td>\n",
       "      <td>4.554</td>\n",
       "      <td>0.038</td>\n",
       "      <td>-0.200</td>\n",
       "      <td>0.845</td>\n",
       "      <td>0.251</td>\n",
       "      <td>-0.079</td>\n",
       "      <td>287.63705</td>\n",
       "      <td>42.530819</td>\n",
       "      <td>15.511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1387</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5.184990</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>-0.000012</td>\n",
       "      <td>131.881400</td>\n",
       "      <td>0.00185</td>\n",
       "      <td>-0.00185</td>\n",
       "      <td>...</td>\n",
       "      <td>-131.0</td>\n",
       "      <td>4.680</td>\n",
       "      <td>0.059</td>\n",
       "      <td>-0.027</td>\n",
       "      <td>0.560</td>\n",
       "      <td>0.045</td>\n",
       "      <td>-0.056</td>\n",
       "      <td>295.90656</td>\n",
       "      <td>45.809120</td>\n",
       "      <td>14.744</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      koi_fpflag_nt  koi_fpflag_ss  koi_fpflag_co  koi_fpflag_ec  koi_period  \\\n",
       "6274              0              0              0              0  492.252990   \n",
       "3541              0              0              0              0   49.567408   \n",
       "8659              1              0              0              0  452.100930   \n",
       "4851              0              1              0              0   18.645329   \n",
       "1387              0              0              0              0    5.184990   \n",
       "\n",
       "      koi_period_err1  koi_period_err2  koi_time0bk  koi_time0bk_err1  \\\n",
       "6274         0.011050        -0.011050   520.860000           0.01800   \n",
       "3541         0.000187        -0.000187   169.542270           0.00322   \n",
       "8659         0.054070        -0.054070   460.488400           0.05070   \n",
       "4851         0.000002        -0.000002   147.207512           0.00007   \n",
       "1387         0.000012        -0.000012   131.881400           0.00185   \n",
       "\n",
       "      koi_time0bk_err2  ...  koi_steff_err2  koi_slogg  koi_slogg_err1  \\\n",
       "6274          -0.01800  ...          -152.0      4.478           0.055   \n",
       "3541          -0.00322  ...          -111.0      4.474           0.059   \n",
       "8659          -0.05070  ...          -214.0      4.456           0.066   \n",
       "4851          -0.00007  ...          -175.0      4.554           0.038   \n",
       "1387          -0.00185  ...          -131.0      4.680           0.059   \n",
       "\n",
       "      koi_slogg_err2  koi_srad  koi_srad_err1  koi_srad_err2         ra  \\\n",
       "6274          -0.283     0.936          0.383         -0.089  292.13519   \n",
       "3541          -0.110     0.927          0.122         -0.066  290.67313   \n",
       "8659          -0.281     0.956          0.370         -0.103  292.10245   \n",
       "4851          -0.200     0.845          0.251         -0.079  287.63705   \n",
       "1387          -0.027     0.560          0.045         -0.056  295.90656   \n",
       "\n",
       "            dec  koi_kepmag  \n",
       "6274  38.926491      13.465  \n",
       "3541  38.691010      13.597  \n",
       "8659  37.566910      15.107  \n",
       "4851  42.530819      15.511  \n",
       "1387  45.809120      14.744  \n",
       "\n",
       "[5 rows x 40 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use sklearn's `train_test_split` to split the data into training and testing\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, random_state=1)\n",
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pre-processing\n",
    "\n",
    "Scale the data using the MinMaxScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder, MinMaxScaler\n",
    "#from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "X_scaler = MinMaxScaler().fit(X_train)\n",
    "X_train_scaled = X_scaler.transform(X_train)\n",
    "X_test_scaled = X_scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 2, 0, ..., 2, 1, 2])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Label-encode data set\n",
    "label_encoder = LabelEncoder()\n",
    "label_encoder.fit(y_train)\n",
    "encoded_y_train = label_encoder.transform(y_train)\n",
    "encoded_y_test = label_encoder.transform(y_test)\n",
    "encoded_y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 1.],\n",
       "       [0., 0., 1.],\n",
       "       [1., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 1.],\n",
       "       [0., 1., 0.],\n",
       "       [0., 0., 1.]], dtype=float32)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert encoded labels to one-hot-encoding\n",
    "y_train_categorical = to_categorical(encoded_y_train)\n",
    "y_test_categorical = to_categorical(encoded_y_test)\n",
    "y_train_categorical\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the Support Vector Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/lpinelec/anaconda3/envs/pandas/lib/python3.6/site-packages/sklearn/utils/validation.py:724: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
       "    decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
       "    kernel='linear', max_iter=-1, probability=False, random_state=None,\n",
       "    shrinking=True, tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import SVC \n",
    "model2 = SVC(kernel='linear')\n",
    "model2.fit(X_train_scaled, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Data Score: 0.8502592253735896\n",
      "Testing Data Score: 0.838975297346752\n"
     ]
    }
   ],
   "source": [
    "print(f\"Training Data Score: {model2.score(X_train_scaled, y_train)}\")\n",
    "print(f\"Testing Data Score: {model2.score(X_test_scaled, y_test)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                precision    recall  f1-score   support\n",
      "\n",
      "     CANDIDATE       0.73      0.56      0.63       523\n",
      "     CONFIRMED       0.68      0.80      0.73       594\n",
      "FALSE POSITIVE       0.98      1.00      0.99      1069\n",
      "\n",
      "      accuracy                           0.84      2186\n",
      "     macro avg       0.80      0.78      0.79      2186\n",
      "  weighted avg       0.84      0.84      0.83      2186\n",
      "\n"
     ]
    }
   ],
   "source": [
    "target_names = [\"CANDIDATE\", \"CONFIRMED\", \"FALSE POSITIVE\"]\n",
    "# Calculate classification report\n",
    "from sklearn.metrics import classification_report\n",
    "predictions = model2.predict(X_test_scaled)\n",
    "print(classification_report(y_test, predictions,\n",
    "                            target_names=target_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "k: 1, Train/Test Score: 1.000/0.382\n",
      "k: 3, Train/Test Score: 0.664/0.387\n",
      "k: 5, Train/Test Score: 0.589/0.380\n",
      "k: 7, Train/Test Score: 0.572/0.404\n",
      "k: 9, Train/Test Score: 0.556/0.421\n",
      "k: 11, Train/Test Score: 0.537/0.423\n",
      "k: 13, Train/Test Score: 0.530/0.439\n",
      "k: 15, Train/Test Score: 0.524/0.450\n",
      "k: 17, Train/Test Score: 0.518/0.451\n",
      "k: 19, Train/Test Score: 0.512/0.457\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxddZ3/8dcne5o03ZJuaUtXCmWRQkF2F7Qt4ACCsqi44NBBQcQRHFB/iMyoKOOIC6Mig4AiCLIqhYIoVJClhbJ1gy7QpgvdmzZNm6Wf3x/nJLm5vUlO0tzc7f18PO7jnv1+egnnc8/3e873Y+6OiIjkrrxUByAiIqmlRCAikuOUCEREcpwSgYhIjlMiEBHJcQWpDqC7KisrfezYsakOQ0Qko7z88sub3L0q0bqMSwRjx45l/vz5qQ5DRCSjmNm7Ha1T05CISI5TIhARyXFKBCIiOU6JQEQkxykRiIjkuKQlAjO7zcw2mNmbHaw3M/uZmS0zs9fN7MhkxfLQgjWccMPfGHf1o5xww994aMGaZH2UiEjGSeYVwe3AzE7WnwpMCl+zgF8mI4iHFqzhmgfeYM22ehxYs62eax54Q8lARCSUtETg7nOBLZ1sciZwpwdeAAaa2YjejuPGOUupb2xut6y+sZkb5yzt7Y8SEclIqewjqAZWx8zXhMv2YWazzGy+mc3fuHFjtz5k7bb6bi0XEck1qUwElmBZwio57n6Lu09z92lVVQmfkO7QyIGl3VouIpJrUpkIaoDRMfOjgLW9/SFXzZhMaWF+u2WlhflcNWNyb3+UiEhGSmUieAT4bHj30LHAdndf19sfctbUan5w9mFUx1wBfGPmZM6amrAVSkQk5yRt0Dkzuxv4IFBpZjXAd4BCAHf/FTAbOA1YBuwCvpCsWM6aWs1ZU6tZvnEnp/z4GfIsUauUiEhuSloicPcLuljvwKXJ+vxEJlSVM3FoOXMWrudzx4/ty48WEUlbOfdk8fQpw3hx5Ra21jWkOhQRkbSQc4lgxiHDad7rPLVkQ6pDERFJCzmXCA4fNYARA0qYs3B9qkMREUkLOZcIzIzpU4bxj7c3Ut/Q3PUOIiJZLucSAQTNQ7sb9/LMW917SllEJBvlZCI4ZtxgBpQW8oSah0REcjMRFOTnccrBQ/nr4vdobN6b6nBERFIqJxMBBM1DtbubeHFFZwOkiohkv5xNBCdPqqKkME93D4lIzsvZRFBalM8HDqziyUXvsXdvwkFPRURyQs4mAgiah9bX7ub1NdtTHYqISMrkdCI45aBh5OeZmodEJKfldCIY0K+QY8cPViIQkZyW04kAguahFRvrWLZhR6pDERFJiZxPBNOnDAdgzsL3UhyJiEhq5HwiGD6ghPeNHqinjEUkZ+V8IgCYccgwXqvZzrrt9akORUSkzykREPQTADyh5iERyUFKBAQlLCdUlenuIRHJSUoEoRmHDFcJSxHJSUoEIZWwFJFcpUQQailhqbuHRCTXKBGEWkpYzlUJSxHJMUlNBGY208yWmtkyM7s6wfoDzOwpM3vdzJ42s1HJjKcrKmEpIrkoaYnAzPKBm4FTgSnABWY2JW6z/wbudPfDgeuBHyQrniiOVglLEclBybwiOAZY5u4r3L0BuAc4M26bKcBT4fTfE6zvU4UqYSkiOSiZiaAaWB0zXxMui/UacE44/XGgv5kNiT+Qmc0ys/lmNn/jxuQ226iEpYjkmmQmAkuwLL4U2JXAB8xsAfABYA3QtM9O7re4+zR3n1ZVVdX7kcZoKWH5xCI1D4lIbkhmIqgBRsfMjwLWxm7g7mvd/Wx3nwp8K1yW0nJhLSUsn1ioEpYikhuSmQjmAZPMbJyZFQHnA4/EbmBmlWbWEsM1wG1JjCcylbAUkVyStETg7k3AZcAcYDFwr7svNLPrzeyMcLMPAkvN7C1gGPC9ZMXTHR8+aKhKWIpIzihI5sHdfTYwO27ZtTHTfwL+lMwYemJgv6LWEpb/MfOgVIcjIpJUerK4AyphKSK5QomgAyphKSK5QomgAyphKSK5QomgEyphKSK5QImgEy3NQyphKSLZrMtEYGalZnaNmf0qnJ9oZqcmP7TUmzhUJSxFJPtFuSK4jWC4iBPD+bXA95MWUZpRCUsRyXZREsEkd/8+0Ajg7rtIPI5QVlIJSxHJdlESQYOZlRAOGGdm44Cc+XmsEpYiku2iJILrgceBUWZ2B0HdgGuSGlUaUQlLEcl2nSYCMzOCmgGfBC4GHgSOcfenOtsv20xXCUsRyWKdJgJ3d+Av7r7R3R9294fcPecay49RCUsRyWJRmoZeMrMjkx5JGlMJSxHJZlESwYkEyWCpmb1iZgvM7JVkB5ZuVMJSRLJVlGGoz0p6FBkgtoTliZMqUx2OiEiv6fKKwN2XA6XAR8NXSbgsp6iEpYhkqyhDTFwG3AuMCV/3mtmXkx1YOpo+RSUsRST7RGkamkVwy+hOADP7PvBP4H+TGVg6OuXgthKWR4wemOpwRER6RZTOYiMcXiLUSA4NMRErtoSliEi2iJIIfge8YGbfNrNvE1wN3JHcsNKXSliKSLaJ0ln8I4LmoV1APXCJu/93sgNLVyphKSLZJkpn8dHAYnf/H3f/MbDEzKYlP7T0pBKWIpJtojQN3UJwNdCiDvh1csLJDNOnqISliGSPKIkgz91bx1UIpwujHNzMZoZPJC8zs6sTrB9jZn8Pn1Z+3cxOix566sw4RCUsRSR7REkEK83sS2aWb2Z5ZnYp8E5XO5lZPnAzcCowBbjAzKbEbfZt4F53nwqcT4bckqoSliKSTaIkgn8DTgHeAzYAHyAYkrorxwDL3H2FuzcA9wBnxm3jQEU4PYCgDGZGUAlLEckWUe4aes/dP+HuleHrXHeP0iZSDayOma8Jl8W6DviMmdUAs4GvJDqQmc0ys/lmNn/jxvSoCdBSwvJvKmEpIhmuw0RgZheZ2cRw2szsFjPbHI5AekSEYyd66Cx+kJ4LgNvdfRRwGvA7M9snJne/xd2nufu0qqqqCB+dfIePGsDwihI1D4lIxuvsiuDfgXfD6fOAowna+r8J/CzCsWuA0THzo9i36eeLBOMY4e7PAyVARgztaWZMP0QlLEUk83WWCJrcvWVoiX8B7gibiR4HyiMcex4wyczGmVkRQWfwI3HbrCLof8DMDiZIBOnR9hPBDJWwFJEs0FkicDMbZmbFBCfrv8asK+3qwO7eBFwGzAEWE9wdtNDMrjezM8LNvg5cbGavAXcDnw/LY2YElbAUkWzQ2eij1wEtlcgec/c3AczsJGBllIO7+2yCTuDYZdfGTC8CTuhGvGmltYTloqCEZWF+lJuwRETSS4dnLnd/GBgHHOHuX4hZ9SpBM4/QVsLypZUqYSkimanTn7Du3uDuG+OW7XD32uSGlTlaSljq7iERyVRqy9hPpUX5nDxJJSxFJHMpEfSCGYeohKWIZK4ow1DfY2YzzCwnq5JFEVvCUkQk00S5IrgduAh4y8z+q+VpY2mjEpYiksmijDX0uLufRzCI3Hrg72Y218wuNLPObj/NKW0lLHemOhQRkW6J1EdgZoOATwEXAq8TFKY5Hng8eaFllrYSlroqEJHMEqWP4F6CgvWDgXPc/XR3v8vdvwQMSXaAmUIlLEUkU0W5IrgVmOLu/+nuNbErwoIyElIJSxHJRFESwXiCojFA0ExkZrOSF1LmUglLEclEURLBJe6+rWXG3bcCX0peSJlLJSxFJBNFSQT5sTNh4ZhIxetzkUpYikimiZIInjSzu83sA2Z2MnAX7YeklhgqYSkimSZKIriK4K6hrxHUD3gWuDKZQWUylbAUkUzT5QNh7t4M/Dx8SRdaSljeO3819Q3NlBbld72TiEgKRXmOYEI43tDrZvZWy6svgstUKmEpIpkk6lhDvwUMOJWg2Pw9SYwp46mEpYhkkiiJoJ+7zwFw9+Xu/m3gQ8kNK7O1lrBcHJSwFBFJZ1ESwZ5wCOrlZnaJmf0LMDTJcWU8lbAUkUwRJRF8DSgHLicoNP+vBMNSSydUwlJEMkWnicDM8oGPh3WKV7n7he5+prs/10fxZSyVsBSRTNFV8fpmgjoE0gMqYSkimSBK09ArZvaAmV1gZme0vKIc3MxmmtlSM1tmZlcnWP8TM3s1fL1lZtsSHSdTqYSliGSCKBXGhgF1wGkxyxx4pLOdwmalm4GPAjXAPDN7xN0XtR7E/Wsx238FyKphrWNLWP7HzINSHY6ISEJRniy+sIfHPgZY5u4rAMzsHuBMYFEH218AfKeHn5W2ZhwynGsfXsiyDTuZOLQ81eGIiOyjy0RgZrckWu7uXdUkqAZWx8zXAO/v4DMOAMYBf+tg/SxgFsCYMWO6+Nj08tEpw7j24YXMWbieiUMnpjocEZF9ROkjeCrm9RzBMwR7IuxnCZZ1dPvM+cCfws7pfXdyv8Xdp7n7tKqqqggfnT5GDCjlfaMG6CljEUlbUZqG/hg7b2a/A56McOwaYHTM/ChgbQfbng9cGuGYGWn6IcO5cc5S1m2vZ8SA0lSHIyLSTpQrgnjjgAMibDcPmGRm48ysiOBkv08Hs5lNBgYBz/cgloygEpYiks6ijD661cy2hK9tBFcD3+xqP3dvAi4D5gCLgXvdfaGZXR93++kFwD3unrVPXamEpYiksyi3j1bGTO/tzgnb3WcDs+OWXRs3f13U42WyGYcM59dzV7BtVwMD+xWlOhwRkVZRmoZOB8rdvdnd3cwGmtnHkh1YtmkpYfnUYpWwFJH0EiURXO/urWMkuPs24D+TF1J2OqxaJSxFJD1FSQSJtonSpCQx8vKMiUPLeGLRe4y7+lFOuOFvPLRgTarDEhGJPNbQj8zsADMbY2Y3AguSHVi2eWjBGl5auRUIHqZYs62eax54Q8lARFIuSiK4LNzuYYLbPx34cjKDykY3zllKQ1y1svrGZm6cszRFEYmIBKI8ULYTuLIPYslqa7fVd2u5iEhfifIcweNmNjBmfpCZPZrcsLLPyIGJnyiu7F/cx5GIiLQXpWloWHinEADuvhUYmbyQstNVMyZTWpi/z/JNO/bwy6eX06wqZiKSIlESwV4zG9UyY2aZNfxnmjhrajU/OPswqgeWYkD1wFK+//FDmXnocH74+BI+fesLaiYSkZSwrh4UNrPTgf+lbYjoDwFfcvfHkhxbQtOmTfP58+en4qOTwt257+UarntkIQV5xvfPPoyPHa4LLhHpXWb2srtPS7SuyysCd3+UoMhMy11Dx6QqCWQjM+PcaaOZfflJjK8q57I/LODr977Gjt2NqQ5NRHJE1NFHdwOrgPeAiWZ2fPJCyk1jK8u475LjuPzDE3lwQQ2n/+xZXn53a6rDEpEcEOWuoYuAfxI0Df0wfP9+kuPKSYX5efz79Mnc+2/Hsdedc3/9PDf99S2a4p4/EBHpTVGuCL4GTAPecfeTgKOAdUmNKsdNGzuY2V89iTPeN5Kb/vo25/76eVZt3pXqsEQkS0VJBLvdvR7AzIrcfSFwUHLDkoqSQn5y3hH89PwjeHvDTk772T+4/+Uasrhsg4ikSJREsC58oOzPwBwzu5+gr0D6wJlHVPPYV09iyogKvn7fa3zl7gVs36WOZBHpPV3ePtpuY7NTgAHAo+4epYB9r8u220ejat7r/OqZ5fzkybcY2r+YH597BMdNGJLqsEQkQ+zX7aOx3P0pd38gVUkgl+XnGZd+aCL3f+l4igvz+dStL/DDx5fQ0KSOZBHZPz0pXi8p9L7RA/nLV07kvGmj+eXTyznnl/9k+cadqQ5LRDKYEkEGKisu4IZzDudXnzmK1Vt38bGfPcsfXlyljmQR6RElggw289DhzLniZI46YBDffPANZv3uZbbUNaQ6LBHJMFEeKNtqZlviXivN7D4zG5v8EKUzwypKuPOiY/j26QfzzNKNzLhpLnPf2pjqsEQkg0S5Ivg58P+ACcBE4NvA7cBDwG+TFplElpdn/OtJ43no0hMYWFrIZ297iev/vIjdjc2pDk1EMkCURDDd3W92963uvsXd/xc41d3vAgZ3tqOZzTSzpWa2zMyu7mCbc81skZktNLM/9ODfIKEpIyv481dO5HPHHcBtz63krJufY8n62lSHJSJpLlIfgZmdHTdt4WyH9y6aWT5wM3AqMAW4wMymxG0zCbgGOMHdDwGu6Fb0so+Swny+e+ah/PbzR7Np5x7O+MVz/Pa5lepIFpEORUkEnwEuDvsGNgMXAxeaWT86P3EfAyxz9xXu3gDcA5wZt83FwM1h1TPcfUO3/wWS0IcOGsrjV5zMSRMr+e6fF/H5385jw47dqQ5LRNJQlOL1ywh+1SfyTCe7VgOrY+ZrgPfHbXMggJk9B+QD17n74/EHMrNZwCyAMWNUIC2qyvJibv3cNH7/4ir+6y+LmHnTP/jhOYdTt6eJG+csZe22ekYOLOWqGZM5a2p1qsMVkRTpMhGYWSVwETA2dnt3n9XVrgmWxbdPFACTgA8Co4B/mNmhsTWSw8+6BbgFgiEmuopZ2pgZFx57AMeNH8zld7/KxXfOJz/PWmskr9lWzzUPvAGgZCCSo6I0DT0MDAOeBZ6KeXWlBhgdMz8KWJtgm4fdvdHdVwJLCRKD9LKJQ/vz4KXHU15c0JoEWtQ3NnPjnKUpikxEUq3LKwKgzN2/3oNjzwMmmdk4YA1wPvCpuG0eAi4Abg+vPA4EVvTgsySC4oJ86vY0JVy3Zls9F985n4lDy5lQVR6+l9G/pLCPoxSRvhYlETxmZtPd/YnuHNjdm8zsMmAOQfv/be6+0MyuB+a7+yPhuulmtghoBq5y983d/DdIN4wcWMqabfX7LC8pzOOdTXU8vXQDjc1tVwzDK0pak8LEoeVMGBokiaryYswStf6JSKbpchhqM9tKMPT0LqCBoO3f3b3TZwiSJVeHoe4tDy1YwzUPvEF9zMNmpYX5/ODswzhrajWNzXtZtWUXyzbsZNmGnSzfsJPlG4Ppuoa2fSpKCtpdPbS8Rg3qR36eEoRIuulsGOooiSA/0XJ3T8ljq0oE+++hBWu6fdeQu7O+dndrgljWmiDq2LSzbVTyooI8xleWBVcOMUliXGUZJYXt/5R6EoeI9EyPEoGZTXL3t83s8ETr3f31XowxMiWC9LNtV0PrVUOQIOpYtmEnq7fuouXPywxGD+rXmhhq6xt5cMEa9sTUU4i9MhGR3tVZIuisj+Bq4IsETwfHc+DkXohNssDAfkUcdcBgjjqgfWvh7sZmVmysY9nGtiuI5Rt28uyyTQkL6tQ3NnPtw29SVJDHuMoyxg4po7Qo4QWpiPSiKE1Dhe7e2NWyvqIrgszXvNeZ+M3Z+zxUksiIASVBUqgsY3yYHMZVlTF6UD+KCjSKukhUPb0iaPEicGSEZSKR5OdZh3cvjRhQwm8+O42Vm+p4Z1MdKzfVsXJzHbPfWMe2XW2/PfIMRg/uFySGyvavkQNL1WEt0g0dJgIzGwqMAErN7DDanhSuAPr1QWySxa6aMTnh3Uv/MfMgDq0ewKHVA/bZZ2tdAys3xySI8DX/nS3t7mgqys9jzJB+7ZLD2CFljK8qY2j/fW97Vae15LrOrghOJxhaYhRBP0HL/z07COoTiPRYy4m2OyfgQWVFDCor4sgxg9otd3c27tjTlhw217FyYx3vbK7jmbc2tuuP6FeU3+4qYnPdHh54pa3TWkNuSC6K0kdwrrvf20fxdEl9BNIdzXudddvrW5uaVsQ0Oa3eWr/PcBstBvUr5A8XH8uEqnL1RUhW2N8+gqFmVuHutWb2K4K+gWvcPcp4QyIplZ9njBrUj1GD+nHSpKp26xqb93Lgtx5L2Gm9dVcjp/70HxTmGxOqyjl4RAUHj+jPQcMrOHhEBVX9i/vmHyDSB6Ikglnu/gszm07QTPQlgpFAj0pqZCJJVpif12Gn9dD+xXzr9INZsn4Hi9fV8vzyzTy4YE3r+sryojApBMnhoBH9mTi0nOIC3e4qmSdKImj5wXQq8Ft3f9nMdK0sWaGjTutvnnYwZx5R3a6S0pa6Bpasr2XJuiA5LFm/gzuef7e1D6IgL7h6OGhEfw4eUcFBw4P3RB3UIukkSiJ4zcxmE4wM+i0zK2ffugIiGak7ndaDy4o4fkIlx0+obF3W1LyXdzbXsXjdDpasr2Xxuh3MW7mFh19d226/g4b3b72COHhEBROHlrcbckN3LkkqRR1r6CiCspNbwuGiR7v7gr4IMJ46iyUTbN/VyOL1tSxZV9uaJJa+t4PdjcHVQ36eMb6yjINGVLB3716eXLSBhmYNtyHJs1+dxe7ebGbjgY8C3wNKiVj0XiRXDehXyLHjh3Ds+CGty5r3Ou9srmNJ69VDLa+8uzVhH0V9YzPfuP91Hn9zPRWlBVSUFFJRWkhFSQED+hXGzBdSUVrAgNJCSgvz97sJSlcmuSnKFcEvgELgZHc/2MwGA3Pc/ei+CDCerggk24y7+tEO21oPHFZObX0Ttbsb2dXQ+YC/BXnWmiwqSgsZEJMoWhNHzPqKkkIGxKx7/M31nQ5RLpltf28fPd7djzSzBQBh81BRr0YoksM6unOpemApT3ztA63zDU172bG7kdrdTdTWN1K7u5Ha+ia2t063Lavd3cj2+kbWbqtv3X5PgoH+ulLf2Mw3H3yDJet30L81gbRPJP1LgumSwrxe6xTXlUnfipIIGsO7hBzAzIYA3f+LEpGEOrpz6aoZk9ttV1SQx5DyYoaU9+wZht2NzezYHZ842pLKjx5PXLd6V0Mztz27sl0fRiKF+dauCat//NVIwmXBfP+SQsqKgqat+OJJeto7+Toba6jA3ZsIhpe4H6gys+8C5wLf7aP4RLJeT4bb6ImSwnxKCvM7fBjurhdWdXhl8tzVH2Z3Y3O7K44dcVcmwbK26dr6RtbX7qa2Ptg2NtElkp9n9C8pYMfupn2e+K5vbObbD73Jqi276FeUT3lxAf2KCygvzqdfUUEwH7O8X2E+eb0w8GCuXJl0VpjmFXc/Mpw+BPgIwXhDf3X3N/suxPbURyCSHF2VMd1f8U1bO3Y3JWzSuvP5d/f7syAYV6qsuICy1vcCyorzgwRSVEC/4jBxFLUllLLiYJuy4gJeXLGZm/76dtYUT+ppH0FrOnX3hcDC3g5MRNJHsq9MojZtPbV4Q4dXJs9c9UHqGprZ1dBE3Z4m6vY0B+8NLe+Jl+9qaGLnniY21zXw7pZd7NrTtn0Hw00lVN/YzJX3vcbvX3g3ps+krbkrUXNYRWkh/UsK9uup82RfmXSWCKrM7N87Wunu/9NrUYhIWjhranXKf+121mdSkJ/HgNI8BpQW9spnuTu7G/eyMyZZ7GpoZueeJr7w23kJ92na6xQX5rFpZwMrNtW19rV0NIBhi+KCvH062xMnk/ad8f9ctonvP7ak9RmUZPSZdJYI8oFyYq4MRESSra/6TADMjNKi/LAkavsrlepO7ua661+PbbfM3alvbG7XP9K+HyWmPyWc3lbfyOotu1qbxbrqjI9V39jMjXOW9kkiWOfu1+/Pwc1sJvBTgqRyq7vfELf+88CNQMtoXr9w91v35zNFJPOl+5VJPDOjX1HQ3zB8QEmPPi++M74lgVx+d+JBHNYmSFI9FamPoCfCoSluJngiuQaYZ2aPuPuiuE3/6O6X7c9niYj0tr68MoG2u7qG9m+//IePLUl4ZTJyYGmvfXZnieCU/Tz2MQTjE60AMLN7gDOB+EQgIpKWMu3KpKc6HDPI3bfs57GrgdUx8zXhsnjnmNnrZvYnMxu9n58pIpJVzppazQ/OPozqgaUYQR9Fb9/CGuXJ4p5K1LQU363+Z+Bud99jZpcAdwAf3udAZrOAWQBjxozp7ThFRNJasq9MkjmKaA0Q+wt/FLA2dgN33+zue8LZ39BB1TN3v8Xdp7n7tKqqqkSbiIhIDyUzEcwDJpnZuHCQuvOBR2I3MLMRMbNnAIuTGI+IiCSQtKYhd28ys8uAOQS3j97m7gvN7Hpgvrs/AlxuZmcATcAW4PPJikdERBLrsh5ButFYQyIi3dfZWEOqNCYikuOUCEREcpwSgYhIjlMiEBHJcUoEIiI5TolARCSdPXsTrJzbftnKucHyXqJEICKSzqqPhPs+35YMVs4N5quP7LWPUCIQkfTSB7+A0zqOvc1Qvw2218CGxVBQCid8Fe75FMy+MkgCn7wdxp3cax+ZzEHnRES6r+UXcMvJruUX8CdvT984mvbAnp3QsAP27Aind8Ke2pjpcL5lumFnuO2OmPU7oKmTgjMv/QZO/kavJgHQk8Uikm6aGmDRw/Do12HSR+DtJ+D4r8LQg8GbwfeGLw/e98Yu2xuzjbct22ebBK99tnHYtgqWPQmVk2DjUhh2KOQXxpz0w5P33sZo/7aCUiguh6JyKO4fvIrKg2Wt0zHvxeVQ1B+2LIenb4AjPwuv3tWjK4LOnizWFYGI9J09O6F2LexYG7zXroHade2X1W1s2/7N+4P3v/9XcuOyvPCVHzMdvtxh/RtQOjjYtqAYyqraTuAdntT7t19fVA75PTjlrpwLc2+E834XnPwnfbTXm4eUCEQk8OxNQXNI7Mll5VxY8wqceEXn+7rDri3BiX3HusQn+Nq1QdNIvNLBUFENFSNg5NRges8OePl2mHImLH4EPvJdGDUtwQnb2p+08xKcyBO92m3XSVXeluagaV+F+f8HH/1urzfLdGnNK+1P+uNODubXvNJrsahpSEQCsW3gsW3i5/wfVB4Yc4KPObHHnvSb97Q/nuVB+fDgBF8xMjjB9x/RdtKvGBnMF5ZGi6OXO0i7lC5x9BI1DYlIx9yDEzrA4efDXZ+EgaNhy0ooroDfnx20mcfKLw5P5tUw6ui4E3x1cJIvG9qzppA++AWcUXH0AV0RiOQCd9i1GTYvh83Lgs7Hzctg84pgunFX27aWH3S4DhoLB5wY/pqPOcH3Hwn9BnfepCJpR1cEIrlid214km850cec+Hdvb9surwAGHgBDJsK4k2DIBBg8IWjnf+wqmPbFoE38fedl3a9f2ZcSgUiqdbeTtrEetqyI+3Ufvuo2xGxoMGA0DBkPh30yONEPmRic9AeOCW6DjLVybpAEWppDxp2U0W3iEp0SgeS2/blTprd09ODS9I9bIgAAAAzRSURBVO/B0sdjmnHCk31tTfv9y4cFJ/gDZ7Sd6IdMhEHjoLAkehw51CYu7amPQHJby0n3X34KI46Ad58PfhXP+B4MPxz2NgUPGu1tDKabG8P5pphlTeF8y7LmcLuY+dZ9m9q/WvatXQOrXwza4be9Cw4Q00FbMjA8ybec6MOmnCETgnvURbqgPgIRCE66294NnhDdtBQ2vgWb3gqaWv74mfbbPnxp73625Qft8nkFwZ00eQWQV9h+vrg/bF0JVZPh4DPaN+X0G9y78YjEUCKQ7NNYD5veDk7yLSf9TW8HzSvNDW3blQ+HqgPhiE8Hbe7Ln4LJp8OhZwcPHMWfqBOdvGNf+YWJl1k+5HUxvmPLlcnJ3wg6acedrOYY6TNKBJK5dm2JOdnHvG9bRdi2EjzUNGgsVE4OHs2vnBz84h4yEUoHBtvEn4SPvSS1Dy6pk1b6mBKBpEbUTlr3oP08/mS/6a32Y9IUlMCQScEwBEd8OvilXzkZBo/vvMM0HU7C6qSVFEtqZ7GZzQR+CuQDt7r7DR1s9wngPuBod++0J1idxVki/gS8/G9w3xfg2C8HzTKtJ/23obGubb+SgcEv+soDw/fJwciQA8cE+3VXOtw1JNIHOussTloiMLN84C3go0ANMA+4wN0XxW3XH3gUKAIuUyLIAc2NsP51WPAHWHBnMCpj/Zb221RUx5zsY97LqvREq0gPpOquoWOAZe6+IgziHuBMYFHcdv8J/Ai4MomxSCrtroWaebDqBVj9AtTMbxvSoLgiSAKj3w/TLgpO9pWTdEukSB9KZiKoBlbHzNcA74/dwMymAqPd/S9m1mEiMLNZwCyAMWPGJCFU6VXb18Cq59tO/O8tDAYtszwYfhhMvRDGHAsYzP56WydtxchercMqItEkMxEkun5vbYcyszzgJ8DnuzqQu98C3AJB01AvxSe9YW9zUFd11fPBA1GrXoDtYf4vLAs6b0/+RnDiHzWt7Zd+OnTSigiQ3ERQA4yOmR8FrI2Z7w8cCjxtQZvvcOARMzujq36CjJQtnZINu2DNy8Ev/VUvwOqX2oqNlA8PTvjHXQZj3g/DDut4GGLdKSOSNpKZCOYBk8xsHLAGOB/4VMtKd98OVLbMm9nTwJW9ngTS4QTc3BR0dt77WZh5QzB++4Yl8Oev9H1B7u7aubHtpL/qeVj3WjAkAkDVwXDoOcHJf8yxwWiWUTtyE333eohKJCWSlgjcvcnMLgPmENw+epu7LzSz64H57v5Isj67nY4G9Io/ATc1BLcpNuwKOjIb6sL3XTHLu1ofu7yubTr2adYH/639597zmeDBptJBMe+DgtskO1w2KKjq1JO7ZzpLjCd8NXj6tqV9f9ULwYBnEBQiqT4Kjv8KjDkuSGYa9kAkK+TGoHMr58Ldn4KyIbC9Jhia1/Lan8hbfuVGVVAChf2gqCx87xe0iRf163z5sqdg6aMw4ZTgZFq/FXZvC97rt0J9zLQ3d/z5+UWJE0RHiaNl+brX4f6LgkQ4+v3w8h3w1+8EnbiblwXFSyCoI9vyS3/0sTDyiKBot4hkJA0619LksPTRIAlUTU5wkg7fi8qindB78vDSyrnw9++13SVz4hUdN4W4Q8POfZNDR0mjtgbeezOYbtjZeRyFZXDnmYC1JZu6jXDgzCA5jDkuuIVT9+uL5ITcSAQr5wbt3C0n4OMu7fu26O7eJWMW3GFT3D94arY7mhvbkkRr4ohLJivnwoZFMOUsOO1GKB+6//9GEclI2Z8I0uU2xb68Sya/EMqrglciK+fCG/e1JcaNS5QIRHJY9vcRpMNdQ+kkPjHGz4tIVkrJWEPJorGG9pMSo0hOUmextNH9+yISp4uySSIiku2UCEREcpwSgYhIjlMiEBHJcUoEIiI5LuNuHzWzjcC7qY6jC5XAplQHEYHi7F2ZEidkTqyKs/cc4O4JnzLNuESQCcxsfkf366YTxdm7MiVOyJxYFWffUNOQiEiOUyIQEclxSgTJcUuqA4hIcfauTIkTMidWxdkH1EcgIpLjdEUgIpLjlAhERHKcEkEPmNloM/u7mS02s4Vm9tUE23zQzLab2avh69pUxBrG8o6ZvRHGsc8Y3hb4mZktM7PXzezIFMQ4Oea7etXMas3sirhtUvadmtltZrbBzN6MWTbYzJ40s7fD90Ed7Pu5cJu3zexzKYjzRjNbEv63fdDMBnawb6d/J30Q53Vmtibmv+9pHew708yWhn+vV6cgzj/GxPiOmb3awb599n3uN3fXq5svYARwZDjdH3gLmBK3zQeBv6Q61jCWd4DKTtafBjwGGHAs8GKK480H1hM8AJMW3ylwMnAk8GbMsh8BV4fTVwM/TLDfYGBF+D4onB7Ux3FOBwrC6R8mijPK30kfxHkdcGWEv43lwHigCHgt/v+9ZMcZt/7HwLWp/j7396Urgh5w93Xu/ko4vQNYDFSnNqr9ciZwpwdeAAaa2YgUxnMKsNzd0+YJcnefC2yJW3wmcEc4fQdwVoJdZwBPuvsWd98KPAnM7Ms43f0Jd28KZ18ARiXr86Pq4PuM4hhgmbuvcPcG4B6C/w5J0VmcZmbAucDdyfr8vqJEsJ/MbCwwFXgxwerjzOw1M3vMzA7p08Dac+AJM3vZzGYlWF8NrI6ZryG1ie18Ov6fK12+U4Bh7r4Ogh8HQKLCz+n23V5EcPWXSFd/J33hsrAJ67YOmtrS6fs8CXjP3d/uYH06fJ+RKBHsBzMrB+4HrnD32rjVrxA0bbwP+DnwUF/HF+MEdz8SOBW41Mziy5FZgn1Scl+xmRUBZwD3JVidTt9pVOn03X4LaALu6mCTrv5Oku2XwATgCGAdQbNLvLT5PoEL6PxqINXfZ2RKBD1kZoUESeAud38gfr2717r7znB6NlBoZpV9HGZLLGvD9w3AgwSX17FqgNEx86OAtX0T3T5OBV5x9/fiV6TTdxp6r6UJLXzfkGCbtPhuw07qjwGf9rABO16Ev5Okcvf33L3Z3fcCv+ng89Pl+ywAzgb+2NE2qf4+u0OJoAfCtsH/Axa7+/90sM3wcDvM7BiC73pz30XZGkeZmfVvmSboOHwzbrNHgM+Gdw8dC2xvafJIgQ5/ZaXLdxrjEaDlLqDPAQ8n2GYOMN3MBoVNHdPDZX3GzGYC/wGc4e67Otgmyt9JUsX1S328g8+fB0wys3Hh1eP5BP8d+tpHgCXuXpNoZTp8n92S6t7qTHwBJxJcjr4OvBq+TgMuAS4Jt7kMWEhwV8MLwPEpinV8GMNrYTzfCpfHxmrAzQR3Y7wBTEtRrP0ITuwDYpalxXdKkJzWAY0Ev0q/CAwBngLeDt8Hh9tOA26N2fciYFn4+kIK4lxG0K7e8rf6q3DbkcDszv5O+jjO34V/f68TnNxHxMcZzp9GcKfe8lTEGS6/veXvMmbblH2f+/vSEBMiIjlOTUMiIjlOiUBEJMcpEYiI5DglAhGRHKdEICKS45QIJCuZ2djYESN78bjXm9lHutjmOjO7sq9iEtlfBakOQCSTuHsqhxPPd/fmVH2+ZC9dEUjWM7PxZrbAzI6OW/5BM3vazP4Ujtd/V8yTy0eZ2TPhgGFzYoaSuN3MPhFOnxbu96wF9Rz+EnP4KeGxV5jZ5THLC8zsjnBgtT+ZWb/wWKeEMb4RDrhWHC5/x8yuNbNngU+a2eVmtijc/54kfm2SQ5QIJKuZ2WSCMaG+4O7zEmwyFbgCmELwNOgJ4ThSPwc+4e5HAbcB34s7bgnwa+BUdz8RqIo77kEEQ1AfA3wnPCbAZOAWdz8cqAW+HB7rduA8dz+M4Er9SzHH2u3uJ7r7PQR1D6aG+1/S7S9EJAElAslmVQTj/3zG3RNWkQJecvcaDwY6exUYS3CyPhR4Mqw+9W32HcP/IGCFu68M5+PHR3rU3fe4+yaCweiGhctXu/tz4fTvCYYrmQysdPe3wuV3EBREaRE7sNnrwF1m9hmCkURF9pv6CCSbbScYY+cEgvFeEtkTM91M8P+EAQvd/bhOjp1oOOSujgv7DpnsEY5VFzN9OkGSOAP4f2Z2iLcVnRHpEV0RSDZrIKga9lkz+1Q39lsKVJnZcRAMOZ6gCM4SYHxYmAjgvIjHHtNyXIKRVp8NjzXWzCaGyy8Enonf0czygNHu/nfgG8BAoDzi54p0SFcEktXcvc7MPkbQzFPn7omGio7fpyHsEP6ZmQ0g+P/kJmKuKty93sy+DDxuZpuAlyKGtBj4nJn9mmDU0l+6+24z+wJwXzjO/TzgVwn2zQd+H8ZkwE/cfVvEzxXpkEYfFekhMyt3953hnUY3A2+7+09SHZdId6lpSKTnLg47kxcCAwjuIhLJOLoiEBHJcboiEBHJcUoEIiI5TolARCTHKRGIiOQ4JQIRkRz3/wGV2zi68mLkfAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "# Loop through different k values to see which has the highest accuracy\n",
    "# Note: We only use odd numbers because we don't want any ties\n",
    "train_scores = []\n",
    "test_scores = []\n",
    "for k in range(1, 20, 2):\n",
    "    knn = KNeighborsClassifier(n_neighbors=k)\n",
    "    knn.fit(X_train_scaled, y_train)\n",
    "    train_score = knn.score(X_train_scaled, y_train)\n",
    "    test_score = knn.score(X_test_scaled, y_test)\n",
    "    train_scores.append(train_score)\n",
    "    test_scores.append(test_score)\n",
    "    print(f\"k: {k}, Train/Test Score: {train_score:.3f}/{test_score:.3f}\")\n",
    "    \n",
    "    \n",
    "plt.plot(range(1, 20, 2), train_scores, marker='o')\n",
    "plt.plot(range(1, 20, 2), test_scores, marker=\"x\")\n",
    "plt.xlabel(\"k neighbors\")\n",
    "plt.ylabel(\"Testing accuracy Score\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameter Tuning\n",
    "\n",
    "Use `GridSearchCV` to tune the `C` and `gamma` parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the GridSearchCV model\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "param_grid = {'C': [1, 5, 10],\n",
    "              'gamma': [0.0001, 0.001, 0.01]}\n",
    "grid2 = GridSearchCV(model2, param_grid, verbose=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/lpinelec/anaconda3/envs/pandas/lib/python3.6/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 9 candidates, totalling 27 fits\n",
      "[CV] C=1, gamma=0.0001 ...............................................\n",
      "[CV] ................... C=1, gamma=0.0001, score=0.847, total=   0.7s\n",
      "[CV] C=1, gamma=0.0001 ...............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.7s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................... C=1, gamma=0.0001, score=0.838, total=   0.7s\n",
      "[CV] C=1, gamma=0.0001 ...............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    1.4s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................... C=1, gamma=0.0001, score=0.848, total=   0.7s\n",
      "[CV] C=1, gamma=0.001 ................................................\n",
      "[CV] .................... C=1, gamma=0.001, score=0.847, total=   1.0s\n",
      "[CV] C=1, gamma=0.001 ................................................\n",
      "[CV] .................... C=1, gamma=0.001, score=0.838, total=   0.6s\n",
      "[CV] C=1, gamma=0.001 ................................................\n",
      "[CV] .................... C=1, gamma=0.001, score=0.848, total=   0.7s\n",
      "[CV] C=1, gamma=0.01 .................................................\n",
      "[CV] ..................... C=1, gamma=0.01, score=0.847, total=   0.8s\n",
      "[CV] C=1, gamma=0.01 .................................................\n",
      "[CV] ..................... C=1, gamma=0.01, score=0.838, total=   0.6s\n",
      "[CV] C=1, gamma=0.01 .................................................\n",
      "[CV] ..................... C=1, gamma=0.01, score=0.848, total=   0.6s\n",
      "[CV] C=5, gamma=0.0001 ...............................................\n",
      "[CV] ................... C=5, gamma=0.0001, score=0.866, total=   0.9s\n",
      "[CV] C=5, gamma=0.0001 ...............................................\n",
      "[CV] ................... C=5, gamma=0.0001, score=0.854, total=   0.6s\n",
      "[CV] C=5, gamma=0.0001 ...............................................\n",
      "[CV] ................... C=5, gamma=0.0001, score=0.875, total=   0.6s\n",
      "[CV] C=5, gamma=0.001 ................................................\n",
      "[CV] .................... C=5, gamma=0.001, score=0.866, total=   0.6s\n",
      "[CV] C=5, gamma=0.001 ................................................\n",
      "[CV] .................... C=5, gamma=0.001, score=0.854, total=   0.6s\n",
      "[CV] C=5, gamma=0.001 ................................................\n",
      "[CV] .................... C=5, gamma=0.001, score=0.875, total=   0.6s\n",
      "[CV] C=5, gamma=0.01 .................................................\n",
      "[CV] ..................... C=5, gamma=0.01, score=0.866, total=   0.8s\n",
      "[CV] C=5, gamma=0.01 .................................................\n",
      "[CV] ..................... C=5, gamma=0.01, score=0.854, total=   0.7s\n",
      "[CV] C=5, gamma=0.01 .................................................\n",
      "[CV] ..................... C=5, gamma=0.01, score=0.875, total=   0.7s\n",
      "[CV] C=10, gamma=0.0001 ..............................................\n",
      "[CV] .................. C=10, gamma=0.0001, score=0.870, total=   0.6s\n",
      "[CV] C=10, gamma=0.0001 ..............................................\n",
      "[CV] .................. C=10, gamma=0.0001, score=0.865, total=   0.6s\n",
      "[CV] C=10, gamma=0.0001 ..............................................\n",
      "[CV] .................. C=10, gamma=0.0001, score=0.879, total=   0.7s\n",
      "[CV] C=10, gamma=0.001 ...............................................\n",
      "[CV] ................... C=10, gamma=0.001, score=0.870, total=   0.6s\n",
      "[CV] C=10, gamma=0.001 ...............................................\n",
      "[CV] ................... C=10, gamma=0.001, score=0.865, total=   0.6s\n",
      "[CV] C=10, gamma=0.001 ...............................................\n",
      "[CV] ................... C=10, gamma=0.001, score=0.879, total=   0.7s\n",
      "[CV] C=10, gamma=0.01 ................................................\n",
      "[CV] .................... C=10, gamma=0.01, score=0.870, total=   0.7s\n",
      "[CV] C=10, gamma=0.01 ................................................\n",
      "[CV] .................... C=10, gamma=0.01, score=0.865, total=   0.6s\n",
      "[CV] C=10, gamma=0.01 ................................................\n",
      "[CV] .................... C=10, gamma=0.01, score=0.879, total=   0.7s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:   18.5s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv='warn', error_score='raise-deprecating',\n",
       "             estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
       "                           decision_function_shape='ovr', degree=3,\n",
       "                           gamma='auto_deprecated', kernel='linear',\n",
       "                           max_iter=-1, probability=False, random_state=None,\n",
       "                           shrinking=True, tol=0.001, verbose=False),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid={'C': [1, 5, 10], 'gamma': [0.0001, 0.001, 0.01]},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=3)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the model with GridSearch\n",
    "grid2.fit(X_train_scaled, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'C': 10, 'gamma': 0.0001}\n",
      "0.8714547118023788\n"
     ]
    }
   ],
   "source": [
    "print(grid2.best_params_)\n",
    "print(grid2.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernel_info": {
   "name": "dev"
  },
  "kernelspec": {
   "display_name": "Python [conda env:pandas] *",
   "language": "python",
   "name": "conda-env-pandas-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "nteract": {
   "version": "0.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}