breast_cancer_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Import Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: optional dependency `torch` is not available. - skipping import of NN models.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARN: OMP_NUM_THREADS=None =>\n",
      "... If you are using openblas if you are using openblas set OMP_NUM_THREADS=1 or risk subprocess calls hanging indefinitely\n"
     ]
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import shap\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from scipy.stats import shapiro, normaltest\n",
    "from tpot import TPOTClassifier\n",
    "from hpsklearn import HyperoptEstimator, pca, min_max_scaler, standard_scaler\n",
    "from hpsklearn import xgboost_classification, random_forest, ada_boost, gradient_boosting, extra_trees\n",
    "from hpsklearn import svc, svc_linear, svc_rbf, svc_poly, svc_sigmoid, liblinear_svc\n",
    "from hpsklearn import any_classifier\n",
    "from hpsklearn import any_preprocessing\n",
    "from hyperopt import tpe, hp\n",
    "from xgboost import XGBClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from catboost import CatBoostClassifier\n",
    "import lightgbm as lgb\n",
    "import catboost as cb\n",
    "import xgboost as xgb\n",
    "import ngboost as ngb\n",
    "import optuna\n",
    "from bayes_opt import BayesianOptimization\n",
    "from skopt import BayesSearchCV\n",
    "from skopt import gp_minimize\n",
    "from skopt.space import Real, Integer, Categorical\n",
    "from skopt.utils import use_named_args\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold, StratifiedKFold, RepeatedStratifiedKFold\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import Perceptron\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.ensemble import IsolationForest\n",
    "from sklearn.neighbors import LocalOutlierFactor\n",
    "from sklearn.covariance import EllipticEnvelope\n",
    "from sklearn.svm import OneClassSVM\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import balanced_accuracy_score, accuracy_score, f1_score, recall_score\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import classification_report,confusion_matrix,plot_confusion_matrix\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler\n",
    "from sklearn.preprocessing import PowerTransformer, QuantileTransformer\n",
    "from sklearn.decomposition import PCA, TruncatedSVD\n",
    "from sklearn.feature_selection import RFE, SelectFromModel, RFECV\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import f_classif, mutual_info_classif\n",
    "from matplotlib import pyplot\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Load and Clean Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('Data/cancer_classification.csv')\n",
    "#df_train = pd.read_csv('Data/higgs_boson_training.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean radius</th>\n",
       "      <th>mean texture</th>\n",
       "      <th>mean perimeter</th>\n",
       "      <th>mean area</th>\n",
       "      <th>mean smoothness</th>\n",
       "      <th>mean compactness</th>\n",
       "      <th>mean concavity</th>\n",
       "      <th>mean concave points</th>\n",
       "      <th>mean symmetry</th>\n",
       "      <th>mean fractal dimension</th>\n",
       "      <th>...</th>\n",
       "      <th>worst texture</th>\n",
       "      <th>worst perimeter</th>\n",
       "      <th>worst area</th>\n",
       "      <th>worst smoothness</th>\n",
       "      <th>worst compactness</th>\n",
       "      <th>worst concavity</th>\n",
       "      <th>worst concave points</th>\n",
       "      <th>worst symmetry</th>\n",
       "      <th>worst fractal dimension</th>\n",
       "      <th>benign_0__mal_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>0.2419</td>\n",
       "      <td>0.07871</td>\n",
       "      <td>...</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.0869</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>0.1812</td>\n",
       "      <td>0.05667</td>\n",
       "      <td>...</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.1238</td>\n",
       "      <td>0.1866</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>0.2069</td>\n",
       "      <td>0.05999</td>\n",
       "      <td>...</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.1444</td>\n",
       "      <td>0.4245</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>0.2597</td>\n",
       "      <td>0.09744</td>\n",
       "      <td>...</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>0.05883</td>\n",
       "      <td>...</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean radius  mean texture  mean perimeter  mean area  mean smoothness  \\\n",
       "0        17.99         10.38          122.80     1001.0          0.11840   \n",
       "1        20.57         17.77          132.90     1326.0          0.08474   \n",
       "2        19.69         21.25          130.00     1203.0          0.10960   \n",
       "3        11.42         20.38           77.58      386.1          0.14250   \n",
       "4        20.29         14.34          135.10     1297.0          0.10030   \n",
       "\n",
       "   mean compactness  mean concavity  mean concave points  mean symmetry  \\\n",
       "0           0.27760          0.3001              0.14710         0.2419   \n",
       "1           0.07864          0.0869              0.07017         0.1812   \n",
       "2           0.15990          0.1974              0.12790         0.2069   \n",
       "3           0.28390          0.2414              0.10520         0.2597   \n",
       "4           0.13280          0.1980              0.10430         0.1809   \n",
       "\n",
       "   mean fractal dimension  ...  worst texture  worst perimeter  worst area  \\\n",
       "0                 0.07871  ...          17.33           184.60      2019.0   \n",
       "1                 0.05667  ...          23.41           158.80      1956.0   \n",
       "2                 0.05999  ...          25.53           152.50      1709.0   \n",
       "3                 0.09744  ...          26.50            98.87       567.7   \n",
       "4                 0.05883  ...          16.67           152.20      1575.0   \n",
       "\n",
       "   worst smoothness  worst compactness  worst concavity  worst concave points  \\\n",
       "0            0.1622             0.6656           0.7119                0.2654   \n",
       "1            0.1238             0.1866           0.2416                0.1860   \n",
       "2            0.1444             0.4245           0.4504                0.2430   \n",
       "3            0.2098             0.8663           0.6869                0.2575   \n",
       "4            0.1374             0.2050           0.4000                0.1625   \n",
       "\n",
       "   worst symmetry  worst fractal dimension  benign_0__mal_1  \n",
       "0          0.4601                  0.11890                0  \n",
       "1          0.2750                  0.08902                0  \n",
       "2          0.3613                  0.08758                0  \n",
       "3          0.6638                  0.17300                0  \n",
       "4          0.2364                  0.07678                0  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 569 entries, 0 to 568\n",
      "Data columns (total 31 columns):\n",
      " #   Column                   Non-Null Count  Dtype  \n",
      "---  ------                   --------------  -----  \n",
      " 0   mean radius              569 non-null    float64\n",
      " 1   mean texture             569 non-null    float64\n",
      " 2   mean perimeter           569 non-null    float64\n",
      " 3   mean area                569 non-null    float64\n",
      " 4   mean smoothness          569 non-null    float64\n",
      " 5   mean compactness         569 non-null    float64\n",
      " 6   mean concavity           569 non-null    float64\n",
      " 7   mean concave points      569 non-null    float64\n",
      " 8   mean symmetry            569 non-null    float64\n",
      " 9   mean fractal dimension   569 non-null    float64\n",
      " 10  radius error             569 non-null    float64\n",
      " 11  texture error            569 non-null    float64\n",
      " 12  perimeter error          569 non-null    float64\n",
      " 13  area error               569 non-null    float64\n",
      " 14  smoothness error         569 non-null    float64\n",
      " 15  compactness error        569 non-null    float64\n",
      " 16  concavity error          569 non-null    float64\n",
      " 17  concave points error     569 non-null    float64\n",
      " 18  symmetry error           569 non-null    float64\n",
      " 19  fractal dimension error  569 non-null    float64\n",
      " 20  worst radius             569 non-null    float64\n",
      " 21  worst texture            569 non-null    float64\n",
      " 22  worst perimeter          569 non-null    float64\n",
      " 23  worst area               569 non-null    float64\n",
      " 24  worst smoothness         569 non-null    float64\n",
      " 25  worst compactness        569 non-null    float64\n",
      " 26  worst concavity          569 non-null    float64\n",
      " 27  worst concave points     569 non-null    float64\n",
      " 28  worst symmetry           569 non-null    float64\n",
      " 29  worst fractal dimension  569 non-null    float64\n",
      " 30  benign_0__mal_1          569 non-null    int64  \n",
      "dtypes: float64(30), int64(1)\n",
      "memory usage: 137.9 KB\n"
     ]
    }
   ],
   "source": [
    "df_train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(569, 31)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "# calculate duplicates\n",
    "dups = df_train.duplicated()\n",
    "#print(dups)\n",
    "# report if there are any duplicates\n",
    "print(dups.any())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    357\n",
       "0    212\n",
       "Name: benign_0__mal_1, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[\"benign_0__mal_1\"].value_counts()\n",
    "#df_train[\"Label\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Malignant    357\n",
      "Benign       212\n",
      "Name: benign_0__mal_1, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "class_names = {0:'Benign', 1:'Malignant'}\n",
    "print(df_train.benign_0__mal_1.value_counts().rename(index = class_names))\n",
    "#df_train['Label'] = df_train['Label'].map({'b':0,'s':1})\n",
    "#df_train[\"Label\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.6839622641509433\n"
     ]
    }
   ],
   "source": [
    "Malignant = df_train[df_train['benign_0__mal_1']==1]\n",
    "Benign = df_train[df_train['benign_0__mal_1']==0]\n",
    "outlier_fraction = len(Malignant)/float(len(Benign))\n",
    "print (outlier_fraction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "X = df_train.drop(['benign_0__mal_1'],axis=1)\n",
    "y = df_train['benign_0__mal_1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#y = LabelEncoder().fit_transform(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Train Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, stratify=y, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(398, 30)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(171, 30)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find Best Algorithm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### Find Best Algorithm with Best Params Using HyperoptEstimator AutoML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-28 17:19:40.251060\n",
      "100%|██████████| 1/1 [00:00<00:00,  3.10trial/s, best loss: 0.375]\n",
      "100%|██████████| 2/2 [00:00<00:00,  3.55trial/s, best loss: 0.125]\n",
      "100%|██████████| 3/3 [00:00<00:00,  3.72trial/s, best loss: 0.11250000000000004]\n",
      "100%|██████████| 4/4 [00:01<00:00,  1.45s/trial, best loss: 0.03749999999999998]\n",
      "100%|██████████| 5/5 [00:26<00:00, 26.62s/trial, best loss: 0.03749999999999998]\n",
      "100%|██████████| 6/6 [00:00<00:00,  3.12trial/s, best loss: 0.03749999999999998]\n",
      "100%|██████████| 7/7 [00:00<00:00,  3.49trial/s, best loss: 0.03749999999999998]\n",
      "100%|██████████| 8/8 [00:00<00:00,  1.52trial/s, best loss: 0.03749999999999998]\n",
      "100%|██████████| 9/9 [00:05<00:00,  5.45s/trial, best loss: 0.012499999999999956]\n",
      "100%|██████████| 10/10 [00:04<00:00,  4.99s/trial, best loss: 0.012499999999999956]\n",
      "100%|██████████| 11/11 [00:00<00:00,  1.50trial/s, best loss: 0.012499999999999956]\n",
      "100%|██████████| 12/12 [00:00<00:00,  2.86trial/s, best loss: 0.012499999999999956]\n",
      "100%|██████████| 13/13 [00:02<00:00,  2.53s/trial, best loss: 0.012499999999999956]\n",
      "100%|██████████| 14/14 [00:02<00:00,  2.07s/trial, best loss: 0.012499999999999956]\n",
      "100%|██████████| 15/15 [00:00<00:00,  1.34trial/s, best loss: 0.012499999999999956]\n",
      "100%|██████████| 16/16 [00:00<00:00,  2.85trial/s, best loss: 0.012499999999999956]\n",
      "100%|██████████| 17/17 [00:00<00:00,  3.51trial/s, best loss: 0.012499999999999956]\n",
      "100%|██████████| 18/18 [00:01<00:00,  1.75s/trial, best loss: 0.0]\n",
      "100%|██████████| 19/19 [00:00<00:00,  1.34trial/s, best loss: 0.0]\n",
      "100%|██████████| 20/20 [00:00<00:00,  3.52trial/s, best loss: 0.0]\n",
      "100%|██████████| 21/21 [00:04<00:00,  4.56s/trial, best loss: 0.0]\n",
      "100%|██████████| 22/22 [00:06<00:00,  6.89s/trial, best loss: 0.0]\n",
      "100%|██████████| 23/23 [00:03<00:00,  3.62s/trial, best loss: 0.0]\n",
      "100%|██████████| 24/24 [00:00<00:00,  2.51trial/s, best loss: 0.0]\n",
      "100%|██████████| 25/25 [00:11<00:00, 11.36s/trial, best loss: 0.0]\n",
      "100%|██████████| 26/26 [00:05<00:00,  5.69s/trial, best loss: 0.0]\n",
      "100%|██████████| 27/27 [00:05<00:00,  5.66s/trial, best loss: 0.0]\n",
      "100%|██████████| 28/28 [00:01<00:00,  1.93s/trial, best loss: 0.0]\n",
      "100%|██████████| 29/29 [00:01<00:00,  1.35s/trial, best loss: 0.0]\n",
      "100%|██████████| 30/30 [00:06<00:00,  6.74s/trial, best loss: 0.0]\n",
      "100%|██████████| 31/31 [00:08<00:00,  8.14s/trial, best loss: 0.0]\n",
      "100%|██████████| 32/32 [00:00<00:00,  2.06trial/s, best loss: 0.0]\n",
      "100%|██████████| 33/33 [00:05<00:00,  5.05s/trial, best loss: 0.0]\n",
      "100%|██████████| 34/34 [00:00<00:00,  1.89trial/s, best loss: 0.0]\n",
      "100%|██████████| 35/35 [00:02<00:00,  2.45s/trial, best loss: 0.0]\n",
      "100%|██████████| 36/36 [00:00<00:00,  1.14trial/s, best loss: 0.0]\n",
      "100%|██████████| 37/37 [00:00<00:00,  1.07trial/s, best loss: 0.0]\n",
      "100%|██████████| 38/38 [00:00<00:00,  1.64trial/s, best loss: 0.0]\n",
      "100%|██████████| 39/39 [00:00<00:00,  1.62trial/s, best loss: 0.0]\n",
      "100%|██████████| 40/40 [00:01<00:00,  1.24s/trial, best loss: 0.0]\n",
      "100%|██████████| 41/41 [00:00<00:00,  1.24trial/s, best loss: 0.0]\n",
      "100%|██████████| 42/42 [00:01<00:00,  1.81s/trial, best loss: 0.0]\n",
      "100%|██████████| 43/43 [00:10<00:00, 10.62s/trial, best loss: 0.0]\n",
      "100%|██████████| 44/44 [00:06<00:00,  6.43s/trial, best loss: 0.0]\n",
      "100%|██████████| 45/45 [00:12<00:00, 12.52s/trial, best loss: 0.0]\n",
      "100%|██████████| 46/46 [00:05<00:00,  5.76s/trial, best loss: 0.0]\n",
      "100%|██████████| 47/47 [00:06<00:00,  6.07s/trial, best loss: 0.0]\n",
      "100%|██████████| 48/48 [00:00<00:00,  1.14trial/s, best loss: 0.0]\n",
      "100%|██████████| 49/49 [00:00<00:00,  1.63trial/s, best loss: 0.0]\n",
      "100%|██████████| 50/50 [00:04<00:00,  4.53s/trial, best loss: 0.0]\n",
      "Execution time :  0:03:05.734596\n",
      "Job Ended at: 2020-11-28 17:22:45.985656\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "accuracy = 0\n",
    "best_model = None\n",
    "#for i in range(100):\n",
    "    #print (i)\n",
    "model = HyperoptEstimator(  classifier= any_classifier('cla'), \n",
    "                                preprocessing= any_preprocessing('pre'), \n",
    "                                algo=tpe.suggest, \n",
    "                                max_evals=50, \n",
    "                                trial_timeout=5000)\n",
    "    # perform the search\n",
    "model.fit(X_train, y_train)\n",
    "acc = model.score(X_test, y_test)\n",
    "    #if acc > accuracy:\n",
    "        #accuracy = acc\n",
    "        #best_model = model\n",
    "        #print (accuracy)\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))\n",
    "print (f\"Job Ended at: {fin_time}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.953\n",
      "{'learner': ExtraTreesClassifier(bootstrap=False, ccp_alpha=0.0, class_weight=None,\n",
      "                     criterion='gini', max_depth=4,\n",
      "                     max_features=0.7975833610618567, max_leaf_nodes=None,\n",
      "                     max_samples=None, min_impurity_decrease=0.0,\n",
      "                     min_impurity_split=None, min_samples_leaf=1,\n",
      "                     min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
      "                     n_estimators=626, n_jobs=1, oob_score=False,\n",
      "                     random_state=4, verbose=False, warm_start=False), 'preprocs': (StandardScaler(copy=True, with_mean=True, with_std=True),), 'ex_preprocs': ()}\n"
     ]
    }
   ],
   "source": [
    "#summarize the best model\n",
    "best_model = model.best_model()\n",
    "print(\"Accuracy: %.4f\" % best_model.score(X_test, y_test))\n",
    "print(best_model.best_model())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "model = XGBClassifier(base_score=0.5, booster='gbtree',\n",
    "              colsample_bylevel=0.50029110018247, colsample_bynode=1,\n",
    "              colsample_bytree=0.9371080486138555, gamma=0.00014999673883672553,\n",
    "              gpu_id=-1, importance_type='gain', interaction_constraints='',\n",
    "              learning_rate=0.15017743323882513, max_delta_step=0, max_depth=10,\n",
    "              min_child_weight=1, missing=None, monotone_constraints='()',\n",
    "              n_estimators=1000, n_jobs=0, num_parallel_tree=1,\n",
    "              objective='multi:softprob', random_state=4,\n",
    "              reg_alpha=6.402887715271464e-05, reg_lambda=1.0516334360090362,\n",
    "              scale_pos_weight=1, seed=4, subsample=0.9995865341563067,\n",
    "              tree_method='exact', validate_parameters=1, verbosity=None)\n",
    "\n",
    "model.fit(X_train, y_train,verbose=False,\n",
    "            early_stopping_rounds=10\n",
    "            #eval_set=[(X_valid,y_valid)]\n",
    "         )\n",
    "\n",
    "prediction = model.predict(X_test)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "print(classification_report(y_test, prediction))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### Find Best Algorithm with Best Params Using TPOT AutoML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "ss = StandardScaler()\n",
    "X_train_scaled = ss.fit_transform(X_train)\n",
    "X_test_scaled = ss.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-28 17:24:17.836220\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3989504d60db4703b12d7ae6e0ef22ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Optimization Progress'), FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Generation 1 - Current best internal CV score: 0.9756837606837607\n",
      "\n",
      "Generation 2 - Current best internal CV score: 0.9756837606837607\n",
      "\n",
      "Generation 3 - Current best internal CV score: 0.9773290598290598\n",
      "\n",
      "Generation 4 - Current best internal CV score: 0.9781837606837607\n",
      "\n",
      "Generation 5 - Current best internal CV score: 0.9832051282051283\n",
      "\n",
      "Generation 6 - Current best internal CV score: 0.9832051282051283\n",
      "\n",
      "Generation 7 - Current best internal CV score: 0.9832051282051283\n",
      "\n",
      "Generation 8 - Current best internal CV score: 0.9840384615384615\n",
      "\n",
      "Generation 9 - Current best internal CV score: 0.9840384615384615\n",
      "\n",
      "Generation 10 - Current best internal CV score: 0.9840384615384615\n",
      "\n",
      "Generation 11 - Current best internal CV score: 0.9840384615384615\n",
      "\n",
      "Generation 12 - Current best internal CV score: 0.9840384615384615\n",
      "\n",
      "Generation 13 - Current best internal CV score: 0.9840384615384615\n",
      "\n",
      "121.65 minutes have elapsed. TPOT will close down.\n",
      "TPOT closed during evaluation in one generation.\n",
      "WARNING: TPOT may not provide a good pipeline if TPOT is stopped/interrupted in a early generation.\n",
      "\n",
      "\n",
      "TPOT closed prematurely. Will use the current best pipeline.\n",
      "\n",
      "Best pipeline: MLPClassifier(SGDClassifier(XGBClassifier(StandardScaler(ZeroCount(input_matrix)), learning_rate=1.0, max_depth=10, min_child_weight=16, n_estimators=100, nthread=1, subsample=0.2), alpha=0.01, eta0=0.1, fit_intercept=False, l1_ratio=0.5, learning_rate=invscaling, loss=log, penalty=elasticnet, power_t=0.0), alpha=0.001, learning_rate_init=0.001)\n",
      "Execution time :  2:01:41.879451\n",
      "Job Ended at: 2020-11-28 19:25:59.715671\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "\n",
    "# define model evaluation\n",
    "cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "# define search\n",
    "#model = TPOTClassifier(generations=5, population_size=50, cv=cv, scoring='accuracy', \n",
    "                       #verbosity=2, random_state=1, n_jobs=-1)\n",
    "tpot = TPOTClassifier(max_time_mins=120, cv=cv, scoring='accuracy', \n",
    "                       verbosity=2, random_state=1, n_jobs=-1)\n",
    "# perform the search\n",
    "tpot.fit(X_train, y_train)\n",
    "# export the best model\n",
    "#tpot.export('tpot_Otto_best_model.py')\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))\n",
    "print (f\"Job Ended at: {fin_time}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "         steps=[('zerocount', ZeroCount()),\n",
       "                ('standardscaler',\n",
       "                 StandardScaler(copy=True, with_mean=True, with_std=True)),\n",
       "                ('stackingestimator-1',\n",
       "                 StackingEstimator(estimator=XGBClassifier(base_score=0.5,\n",
       "                                                           booster='gbtree',\n",
       "                                                           colsample_bylevel=1,\n",
       "                                                           colsample_bynode=1,\n",
       "                                                           colsample_bytree=1,\n",
       "                                                           gamma=0, gpu_id=-1,\n",
       "                                                           importance_type='gain',\n",
       "                                                           interaction_constraints=''...\n",
       "                               batch_size='auto', beta_1=0.9, beta_2=0.999,\n",
       "                               early_stopping=False, epsilon=1e-08,\n",
       "                               hidden_layer_sizes=(100,),\n",
       "                               learning_rate='constant',\n",
       "                               learning_rate_init=0.001, max_fun=15000,\n",
       "                               max_iter=200, momentum=0.9, n_iter_no_change=10,\n",
       "                               nesterovs_momentum=True, power_t=0.5,\n",
       "                               random_state=1, shuffle=True, solver='adam',\n",
       "                               tol=0.0001, validation_fraction=0.1,\n",
       "                               verbose=False, warm_start=False))],\n",
       "         verbose=False)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tpot.fitted_pipeline_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9707602339181286"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tpot.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### Find Best Algorithm with Best Params Using PyCaret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from pycaret.classification import * \n",
    "from pycaret.datasets import get_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "##### PyCaret Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Description</th>        <th class=\"col_heading level0 col1\" >Value</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row0_col0\" class=\"data row0 col0\" >session_id</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row0_col1\" class=\"data row0 col1\" >123</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row1_col0\" class=\"data row1 col0\" >Target</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row1_col1\" class=\"data row1 col1\" >benign_0__mal_1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row2_col0\" class=\"data row2 col0\" >Target Type</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row2_col1\" class=\"data row2 col1\" >Binary</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row3_col0\" class=\"data row3 col0\" >Label Encoded</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row3_col1\" class=\"data row3 col1\" >0: 0, 1: 1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row4_col0\" class=\"data row4 col0\" >Original Data</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row4_col1\" class=\"data row4 col1\" >(569, 31)</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row5_col0\" class=\"data row5 col0\" >Missing Values</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row5_col1\" class=\"data row5 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row6_col0\" class=\"data row6 col0\" >Numeric Features</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row6_col1\" class=\"data row6 col1\" >30</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row7_col0\" class=\"data row7 col0\" >Categorical Features</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row7_col1\" class=\"data row7 col1\" >0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row8_col0\" class=\"data row8 col0\" >Ordinal Features</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row8_col1\" class=\"data row8 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row9_col0\" class=\"data row9 col0\" >High Cardinality Features</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row9_col1\" class=\"data row9 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row10\" class=\"row_heading level0 row10\" >10</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row10_col0\" class=\"data row10 col0\" >High Cardinality Method</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row10_col1\" class=\"data row10 col1\" >None</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row11\" class=\"row_heading level0 row11\" >11</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row11_col0\" class=\"data row11 col0\" >Transformed Train Set</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row11_col1\" class=\"data row11 col1\" >(398, 29)</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row12\" class=\"row_heading level0 row12\" >12</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row12_col0\" class=\"data row12 col0\" >Transformed Test Set</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row12_col1\" class=\"data row12 col1\" >(171, 29)</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row13\" class=\"row_heading level0 row13\" >13</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row13_col0\" class=\"data row13 col0\" >Shuffle Train-Test</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row13_col1\" class=\"data row13 col1\" >True</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row14\" class=\"row_heading level0 row14\" >14</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row14_col0\" class=\"data row14 col0\" >Stratify Train-Test</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row14_col1\" class=\"data row14 col1\" >True</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row15\" class=\"row_heading level0 row15\" >15</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row15_col0\" class=\"data row15 col0\" >Fold Generator</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row15_col1\" class=\"data row15 col1\" >StratifiedKFold</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row16\" class=\"row_heading level0 row16\" >16</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row16_col0\" class=\"data row16 col0\" >Fold Number</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row16_col1\" class=\"data row16 col1\" >10</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row17\" class=\"row_heading level0 row17\" >17</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row17_col0\" class=\"data row17 col0\" >CPU Jobs</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row17_col1\" class=\"data row17 col1\" >-1</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row18\" class=\"row_heading level0 row18\" >18</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row18_col0\" class=\"data row18 col0\" >Use GPU</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row18_col1\" class=\"data row18 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row19\" class=\"row_heading level0 row19\" >19</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row19_col0\" class=\"data row19 col0\" >Log Experiment</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row19_col1\" class=\"data row19 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row20\" class=\"row_heading level0 row20\" >20</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row20_col0\" class=\"data row20 col0\" >Experiment Name</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row20_col1\" class=\"data row20 col1\" >clf-default-name</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row21\" class=\"row_heading level0 row21\" >21</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row21_col0\" class=\"data row21 col0\" >USI</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row21_col1\" class=\"data row21 col1\" >24c0</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row22\" class=\"row_heading level0 row22\" >22</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row22_col0\" class=\"data row22 col0\" >Imputation Type</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row22_col1\" class=\"data row22 col1\" >simple</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row23\" class=\"row_heading level0 row23\" >23</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row23_col0\" class=\"data row23 col0\" >Iterative Imputation Iteration</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row23_col1\" class=\"data row23 col1\" >None</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row24\" class=\"row_heading level0 row24\" >24</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row24_col0\" class=\"data row24 col0\" >Numeric Imputer</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row24_col1\" class=\"data row24 col1\" >mean</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row25\" class=\"row_heading level0 row25\" >25</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row25_col0\" class=\"data row25 col0\" >Iterative Imputation Numeric Model</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row25_col1\" class=\"data row25 col1\" >None</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row26\" class=\"row_heading level0 row26\" >26</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row26_col0\" class=\"data row26 col0\" >Categorical Imputer</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row26_col1\" class=\"data row26 col1\" >constant</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row27\" class=\"row_heading level0 row27\" >27</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row27_col0\" class=\"data row27 col0\" >Iterative Imputation Categorical Model</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row27_col1\" class=\"data row27 col1\" >None</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row28\" class=\"row_heading level0 row28\" >28</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row28_col0\" class=\"data row28 col0\" >Unknown Categoricals Handling</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row28_col1\" class=\"data row28 col1\" >least_frequent</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row29\" class=\"row_heading level0 row29\" >29</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row29_col0\" class=\"data row29 col0\" >Normalize</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row29_col1\" class=\"data row29 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row30\" class=\"row_heading level0 row30\" >30</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row30_col0\" class=\"data row30 col0\" >Normalize Method</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row30_col1\" class=\"data row30 col1\" >None</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row31\" class=\"row_heading level0 row31\" >31</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row31_col0\" class=\"data row31 col0\" >Transformation</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row31_col1\" class=\"data row31 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row32\" class=\"row_heading level0 row32\" >32</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row32_col0\" class=\"data row32 col0\" >Transformation Method</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row32_col1\" class=\"data row32 col1\" >None</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row33\" class=\"row_heading level0 row33\" >33</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row33_col0\" class=\"data row33 col0\" >PCA</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row33_col1\" class=\"data row33 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row34\" class=\"row_heading level0 row34\" >34</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row34_col0\" class=\"data row34 col0\" >PCA Method</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row34_col1\" class=\"data row34 col1\" >None</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row35\" class=\"row_heading level0 row35\" >35</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row35_col0\" class=\"data row35 col0\" >PCA Components</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row35_col1\" class=\"data row35 col1\" >None</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row36\" class=\"row_heading level0 row36\" >36</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row36_col0\" class=\"data row36 col0\" >Ignore Low Variance</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row36_col1\" class=\"data row36 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row37\" class=\"row_heading level0 row37\" >37</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row37_col0\" class=\"data row37 col0\" >Combine Rare Levels</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row37_col1\" class=\"data row37 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row38\" class=\"row_heading level0 row38\" >38</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row38_col0\" class=\"data row38 col0\" >Rare Level Threshold</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row38_col1\" class=\"data row38 col1\" >None</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row39\" class=\"row_heading level0 row39\" >39</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row39_col0\" class=\"data row39 col0\" >Numeric Binning</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row39_col1\" class=\"data row39 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row40\" class=\"row_heading level0 row40\" >40</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row40_col0\" class=\"data row40 col0\" >Remove Outliers</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row40_col1\" class=\"data row40 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row41\" class=\"row_heading level0 row41\" >41</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row41_col0\" class=\"data row41 col0\" >Outliers Threshold</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row41_col1\" class=\"data row41 col1\" >None</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row42\" class=\"row_heading level0 row42\" >42</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row42_col0\" class=\"data row42 col0\" >Remove Multicollinearity</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row42_col1\" class=\"data row42 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row43\" class=\"row_heading level0 row43\" >43</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row43_col0\" class=\"data row43 col0\" >Multicollinearity Threshold</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row43_col1\" class=\"data row43 col1\" >None</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row44\" class=\"row_heading level0 row44\" >44</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row44_col0\" class=\"data row44 col0\" >Clustering</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row44_col1\" class=\"data row44 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row45\" class=\"row_heading level0 row45\" >45</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row45_col0\" class=\"data row45 col0\" >Clustering Iteration</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row45_col1\" class=\"data row45 col1\" >None</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row46\" class=\"row_heading level0 row46\" >46</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row46_col0\" class=\"data row46 col0\" >Polynomial Features</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row46_col1\" class=\"data row46 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row47\" class=\"row_heading level0 row47\" >47</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row47_col0\" class=\"data row47 col0\" >Polynomial Degree</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row47_col1\" class=\"data row47 col1\" >None</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row48\" class=\"row_heading level0 row48\" >48</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row48_col0\" class=\"data row48 col0\" >Trignometry Features</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row48_col1\" class=\"data row48 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row49\" class=\"row_heading level0 row49\" >49</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row49_col0\" class=\"data row49 col0\" >Polynomial Threshold</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row49_col1\" class=\"data row49 col1\" >None</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row50\" class=\"row_heading level0 row50\" >50</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row50_col0\" class=\"data row50 col0\" >Group Features</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row50_col1\" class=\"data row50 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row51\" class=\"row_heading level0 row51\" >51</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row51_col0\" class=\"data row51 col0\" >Feature Selection</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row51_col1\" class=\"data row51 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row52\" class=\"row_heading level0 row52\" >52</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row52_col0\" class=\"data row52 col0\" >Features Selection Threshold</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row52_col1\" class=\"data row52 col1\" >None</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row53\" class=\"row_heading level0 row53\" >53</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row53_col0\" class=\"data row53 col0\" >Feature Interaction</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row53_col1\" class=\"data row53 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row54\" class=\"row_heading level0 row54\" >54</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row54_col0\" class=\"data row54 col0\" >Feature Ratio</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row54_col1\" class=\"data row54 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row55\" class=\"row_heading level0 row55\" >55</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row55_col0\" class=\"data row55 col0\" >Interaction Threshold</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row55_col1\" class=\"data row55 col1\" >None</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row56\" class=\"row_heading level0 row56\" >56</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row56_col0\" class=\"data row56 col0\" >Fix Imbalance</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row56_col1\" class=\"data row56 col1\" >False</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3level0_row57\" class=\"row_heading level0 row57\" >57</th>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row57_col0\" class=\"data row57 col0\" >Fix Imbalance Method</td>\n",
       "                        <td id=\"T_474f8dc0_316b_11eb_9749_44032c87eac3row57_col1\" class=\"data row57 col1\" >SMOTE</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fd465d455c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time :  0:00:14.740240\n",
      "Job Ended at: 2020-11-28 17:17:19.416215\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "#fold_strategy = 'kfold' Default = 'stratifiedkfold'\n",
    "clf = setup(data = df_train, target = 'benign_0__mal_1', session_id=123,\n",
    "            data_split_stratify = True,\n",
    "            #normalize = True, \n",
    "            #normalize_method = 'minmax',\n",
    "            #transformation = True, \n",
    "            #transformation_method = 'quantile',\n",
    "            #pca = True,\n",
    "            #pca_method = 'incremental',\n",
    "            #feature_selection = True,\n",
    "            #feature_selection_threshold = 0.5,\n",
    "            #feature_selection_method = 'boruta',\n",
    "            #remove_outliers = True,\n",
    "            #outliers_threshold = 0.05,\n",
    "            #fix_imbalance = True\n",
    "           )\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))\n",
    "print (f\"Job Ended at: {fin_time}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "##### Finding optimal model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_7b4bb70c_316b_11eb_9749_44032c87eac3 th {\n",
       "          text-align: left;\n",
       "    }#T_7b4bb70c_316b_11eb_9749_44032c87eac3row0_col0,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row0_col2,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row0_col3,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row0_col5,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row0_col7,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row1_col0,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row1_col2,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row1_col3,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row1_col4,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row1_col5,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row1_col6,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row2_col0,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row2_col1,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row2_col2,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row2_col3,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row2_col4,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row2_col6,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row2_col7,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row3_col0,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row3_col1,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row3_col3,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row3_col4,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row3_col5,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row3_col6,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row3_col7,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row4_col0,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row4_col1,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row4_col2,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row4_col3,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row4_col4,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row4_col5,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row4_col6,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row4_col7,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row5_col0,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row5_col1,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row5_col2,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row5_col4,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row5_col5,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row5_col6,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row5_col7,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row6_col0,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row6_col1,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row6_col2,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row6_col3,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row6_col4,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row6_col5,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row6_col6,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row6_col7,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row7_col0,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row7_col1,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row7_col2,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row7_col3,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row7_col4,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row7_col5,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row7_col6,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row7_col7,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row8_col0,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row8_col1,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row8_col2,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row8_col3,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row8_col4,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row8_col5,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row8_col6,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row8_col7,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row9_col0,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row9_col1,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row9_col2,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row9_col3,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row9_col4,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row9_col5,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row9_col6,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row9_col7,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row10_col0,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row10_col1,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row10_col2,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row10_col3,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row10_col4,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row10_col5,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row10_col6,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row10_col7,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row11_col0,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row11_col1,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row11_col2,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row11_col3,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row11_col4,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row11_col5,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row11_col6,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row11_col7,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row12_col0,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row12_col1,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row12_col2,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row12_col3,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row12_col4,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row12_col5,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row12_col6,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row12_col7,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row13_col0,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row13_col1,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row13_col2,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row13_col3,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row13_col4,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row13_col5,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row13_col6,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row13_col7,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row14_col0,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row14_col1,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row14_col2,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row14_col3,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row14_col4,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row14_col5,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row14_col6,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row14_col7{\n",
       "            text-align:  left;\n",
       "            text-align:  left;\n",
       "        }#T_7b4bb70c_316b_11eb_9749_44032c87eac3row0_col1,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row0_col4,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row0_col6,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row1_col1,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row1_col7,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row2_col5,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row3_col2,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row5_col3{\n",
       "            text-align:  left;\n",
       "            text-align:  left;\n",
       "            background-color:  yellow;\n",
       "        }#T_7b4bb70c_316b_11eb_9749_44032c87eac3row0_col8,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row1_col8,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row2_col8,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row3_col8,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row4_col8,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row6_col8,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row7_col8,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row8_col8,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row9_col8,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row10_col8,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row11_col8,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row12_col8,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row13_col8,#T_7b4bb70c_316b_11eb_9749_44032c87eac3row14_col8{\n",
       "            text-align:  left;\n",
       "            text-align:  left;\n",
       "            background-color:  lightgrey;\n",
       "        }#T_7b4bb70c_316b_11eb_9749_44032c87eac3row5_col8{\n",
       "            text-align:  left;\n",
       "            text-align:  left;\n",
       "            background-color:  yellow;\n",
       "            background-color:  lightgrey;\n",
       "        }</style><table id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Model</th>        <th class=\"col_heading level0 col1\" >Accuracy</th>        <th class=\"col_heading level0 col2\" >AUC</th>        <th class=\"col_heading level0 col3\" >Recall</th>        <th class=\"col_heading level0 col4\" >Prec.</th>        <th class=\"col_heading level0 col5\" >F1</th>        <th class=\"col_heading level0 col6\" >Kappa</th>        <th class=\"col_heading level0 col7\" >MCC</th>        <th class=\"col_heading level0 col8\" >TT (Sec)</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3level0_row0\" class=\"row_heading level0 row0\" >et</th>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row0_col0\" class=\"data row0 col0\" >Extra Trees Classifier</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row0_col1\" class=\"data row0 col1\" >0.9675</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row0_col2\" class=\"data row0 col2\" >0.9885</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row0_col3\" class=\"data row0 col3\" >0.9760</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row0_col4\" class=\"data row0 col4\" >0.9730</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row0_col5\" class=\"data row0 col5\" >0.9743</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row0_col6\" class=\"data row0 col6\" >0.9302</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row0_col7\" class=\"data row0 col7\" >0.9310</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row0_col8\" class=\"data row0 col8\" >0.2510</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3level0_row1\" class=\"row_heading level0 row1\" >lightgbm</th>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row1_col0\" class=\"data row1 col0\" >Light Gradient Boosting Machine</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row1_col1\" class=\"data row1 col1\" >0.9675</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row1_col2\" class=\"data row1 col2\" >0.9885</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row1_col3\" class=\"data row1 col3\" >0.9840</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row1_col4\" class=\"data row1 col4\" >0.9656</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row1_col5\" class=\"data row1 col5\" >0.9743</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row1_col6\" class=\"data row1 col6\" >0.9301</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row1_col7\" class=\"data row1 col7\" >0.9321</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row1_col8\" class=\"data row1 col8\" >0.0770</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3level0_row2\" class=\"row_heading level0 row2\" >rf</th>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row2_col0\" class=\"data row2 col0\" >Random Forest Classifier</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row2_col1\" class=\"data row2 col1\" >0.9674</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row2_col2\" class=\"data row2 col2\" >0.9886</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row2_col3\" class=\"data row2 col3\" >0.9880</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row2_col4\" class=\"data row2 col4\" >0.9618</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row2_col5\" class=\"data row2 col5\" >0.9745</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row2_col6\" class=\"data row2 col6\" >0.9292</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row2_col7\" class=\"data row2 col7\" >0.9307</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row2_col8\" class=\"data row2 col8\" >0.2770</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3level0_row3\" class=\"row_heading level0 row3\" >catboost</th>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row3_col0\" class=\"data row3 col0\" >CatBoost Classifier</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row3_col1\" class=\"data row3 col1\" >0.9649</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row3_col2\" class=\"data row3 col2\" >0.9899</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row3_col3\" class=\"data row3 col3\" >0.9840</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row3_col4\" class=\"data row3 col4\" >0.9619</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row3_col5\" class=\"data row3 col5\" >0.9725</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row3_col6\" class=\"data row3 col6\" >0.9241</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row3_col7\" class=\"data row3 col7\" >0.9256</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row3_col8\" class=\"data row3 col8\" >4.8040</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3level0_row4\" class=\"row_heading level0 row4\" >gbc</th>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row4_col0\" class=\"data row4 col0\" >Gradient Boosting Classifier</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row4_col1\" class=\"data row4 col1\" >0.9574</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row4_col2\" class=\"data row4 col2\" >0.9826</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row4_col3\" class=\"data row4 col3\" >0.9760</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row4_col4\" class=\"data row4 col4\" >0.9576</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row4_col5\" class=\"data row4 col5\" >0.9663</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row4_col6\" class=\"data row4 col6\" >0.9082</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row4_col7\" class=\"data row4 col7\" >0.9098</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row4_col8\" class=\"data row4 col8\" >0.1520</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3level0_row5\" class=\"row_heading level0 row5\" >lda</th>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row5_col0\" class=\"data row5 col0\" >Linear Discriminant Analysis</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row5_col1\" class=\"data row5 col1\" >0.9574</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row5_col2\" class=\"data row5 col2\" >0.9872</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row5_col3\" class=\"data row5 col3\" >1.0000</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row5_col4\" class=\"data row5 col4\" >0.9387</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row5_col5\" class=\"data row5 col5\" >0.9678</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row5_col6\" class=\"data row5 col6\" >0.9056</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row5_col7\" class=\"data row5 col7\" >0.9117</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row5_col8\" class=\"data row5 col8\" >0.0210</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3level0_row6\" class=\"row_heading level0 row6\" >xgboost</th>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row6_col0\" class=\"data row6 col0\" >Extreme Gradient Boosting</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row6_col1\" class=\"data row6 col1\" >0.9524</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row6_col2\" class=\"data row6 col2\" >0.9864</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row6_col3\" class=\"data row6 col3\" >0.9680</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row6_col4\" class=\"data row6 col4\" >0.9577</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row6_col5\" class=\"data row6 col5\" >0.9623</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row6_col6\" class=\"data row6 col6\" >0.8975</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row6_col7\" class=\"data row6 col7\" >0.8993</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row6_col8\" class=\"data row6 col8\" >0.0960</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3level0_row7\" class=\"row_heading level0 row7\" >qda</th>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row7_col0\" class=\"data row7 col0\" >Quadratic Discriminant Analysis</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row7_col1\" class=\"data row7 col1\" >0.9522</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row7_col2\" class=\"data row7 col2\" >0.9885</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row7_col3\" class=\"data row7 col3\" >0.9560</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row7_col4\" class=\"data row7 col4\" >0.9689</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row7_col5\" class=\"data row7 col5\" >0.9619</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row7_col6\" class=\"data row7 col6\" >0.8978</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row7_col7\" class=\"data row7 col7\" >0.8994</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row7_col8\" class=\"data row7 col8\" >0.0280</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3level0_row8\" class=\"row_heading level0 row8\" >ridge</th>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row8_col0\" class=\"data row8 col0\" >Ridge Classifier</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row8_col1\" class=\"data row8 col1\" >0.9499</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row8_col2\" class=\"data row8 col2\" >0.0000</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row8_col3\" class=\"data row8 col3\" >0.9960</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row8_col4\" class=\"data row8 col4\" >0.9309</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row8_col5\" class=\"data row8 col5\" >0.9620</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row8_col6\" class=\"data row8 col6\" >0.8890</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row8_col7\" class=\"data row8 col7\" >0.8945</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row8_col8\" class=\"data row8 col8\" >0.0250</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3level0_row9\" class=\"row_heading level0 row9\" >ada</th>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row9_col0\" class=\"data row9 col0\" >Ada Boost Classifier</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row9_col1\" class=\"data row9 col1\" >0.9499</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row9_col2\" class=\"data row9 col2\" >0.9881</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row9_col3\" class=\"data row9 col3\" >0.9800</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row9_col4\" class=\"data row9 col4\" >0.9444</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row9_col5\" class=\"data row9 col5\" >0.9614</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row9_col6\" class=\"data row9 col6\" >0.8902</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row9_col7\" class=\"data row9 col7\" >0.8935</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row9_col8\" class=\"data row9 col8\" >0.0850</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3level0_row10\" class=\"row_heading level0 row10\" >lr</th>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row10_col0\" class=\"data row10 col0\" >Logistic Regression</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row10_col1\" class=\"data row10 col1\" >0.9498</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row10_col2\" class=\"data row10 col2\" >0.9895</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row10_col3\" class=\"data row10 col3\" >0.9640</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row10_col4\" class=\"data row10 col4\" >0.9581</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row10_col5\" class=\"data row10 col5\" >0.9602</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row10_col6\" class=\"data row10 col6\" >0.8921</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row10_col7\" class=\"data row10 col7\" >0.8948</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row10_col8\" class=\"data row10 col8\" >1.1050</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3level0_row11\" class=\"row_heading level0 row11\" >knn</th>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row11_col0\" class=\"data row11 col0\" >K Neighbors Classifier</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row11_col1\" class=\"data row11 col1\" >0.9423</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row11_col2\" class=\"data row11 col2\" >0.9660</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row11_col3\" class=\"data row11 col3\" >0.9680</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row11_col4\" class=\"data row11 col4\" >0.9436</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row11_col5\" class=\"data row11 col5\" >0.9549</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row11_col6\" class=\"data row11 col6\" >0.8749</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row11_col7\" class=\"data row11 col7\" >0.8783</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row11_col8\" class=\"data row11 col8\" >0.0580</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3level0_row12\" class=\"row_heading level0 row12\" >nb</th>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row12_col0\" class=\"data row12 col0\" >Naive Bayes</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row12_col1\" class=\"data row12 col1\" >0.9347</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row12_col2\" class=\"data row12 col2\" >0.9823</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row12_col3\" class=\"data row12 col3\" >0.9680</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row12_col4\" class=\"data row12 col4\" >0.9326</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row12_col5\" class=\"data row12 col5\" >0.9494</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row12_col6\" class=\"data row12 col6\" >0.8578</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row12_col7\" class=\"data row12 col7\" >0.8612</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row12_col8\" class=\"data row12 col8\" >0.0280</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3level0_row13\" class=\"row_heading level0 row13\" >dt</th>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row13_col0\" class=\"data row13 col0\" >Decision Tree Classifier</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row13_col1\" class=\"data row13 col1\" >0.9247</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row13_col2\" class=\"data row13 col2\" >0.9171</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row13_col3\" class=\"data row13 col3\" >0.9480</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row13_col4\" class=\"data row13 col4\" >0.9348</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row13_col5\" class=\"data row13 col5\" >0.9406</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row13_col6\" class=\"data row13 col6\" >0.8379</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row13_col7\" class=\"data row13 col7\" >0.8407</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row13_col8\" class=\"data row13 col8\" >0.0260</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3level0_row14\" class=\"row_heading level0 row14\" >svm</th>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row14_col0\" class=\"data row14 col0\" >SVM - Linear Kernel</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row14_col1\" class=\"data row14 col1\" >0.8872</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row14_col2\" class=\"data row14 col2\" >0.0000</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row14_col3\" class=\"data row14 col3\" >0.8920</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row14_col4\" class=\"data row14 col4\" >0.9375</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row14_col5\" class=\"data row14 col5\" >0.8984</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row14_col6\" class=\"data row14 col6\" >0.7684</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row14_col7\" class=\"data row14 col7\" >0.7919</td>\n",
       "                        <td id=\"T_7b4bb70c_316b_11eb_9749_44032c87eac3row14_col8\" class=\"data row14 col8\" >0.0290</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fd465d5ceb8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time :  0:01:16.600605\n",
      "Job Ended at: 2020-11-28 17:18:46.497164\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "#top3_models = compare_models(n_select = 3)\n",
    "best = compare_models()\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))\n",
    "print (f\"Job Ended at: {fin_time}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "best_model = finalize_model(best)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))\n",
    "print (f\"Job Ended at: {fin_time}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "print (best_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "##### Tune Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "#tuned_model = tune_model(best_model, optimize = 'AUC') # 'Recall'\n",
    "tuned_model = tune_model(best_model, optimize = 'F1')\n",
    "print(tuned_model)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))\n",
    "print (f\"Job Ended at: {fin_time}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "##### Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "predictions = predict_model(tuned_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find Best Params of Gradient Boosting "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### XGBoost Hyperparameter Optimization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "##### skopt Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#xgb_params[\"n_estimators\"]=(10, 100),\n",
    "#xgb_params['min_child_weight']=(1, 20),  \n",
    "#xgb_params['scale_pos_weight']= (1.0, 16.0),  For Imbalanced\n",
    "\n",
    "# define search space\n",
    "xgb_params = dict()\n",
    "#params['booster'] = ['gbtree', 'dart']\n",
    "#params[\"grow_policy\"]= [\"depthwise\", \"lossguide\"]\n",
    "xgb_params['learning_rate'] = (1e-7, 1.0, 'log-uniform')\n",
    "xgb_params['min_child_weight'] = (1e-16, 1e5, 'log-uniform')\n",
    "xgb_params['subsample'] = (0.5, 1.0)\n",
    "xgb_params['max_depth'] = (2, 20)\n",
    "xgb_params['colsample_bylevel'] = (0.5, 1.0)\n",
    "xgb_params['colsample_bytree'] = (0.5, 1.0)\n",
    "xgb_params['colsample_bynode'] = (0.5, 1.0)\n",
    "#xgb_params['max_delta_step'] = (0.1, 10)\n",
    "#xgb_params['bagging_temperature'] = (0.5, 1.0)\n",
    "#xgb_params['random_strength'] = (0.0, 100)\n",
    "xgb_params['reg_alpha'] = (0.0, 100)\n",
    "xgb_params['reg_lambda'] = (0.0, 100)\n",
    "#xgb_params['gamma'] = (0.0, 100)\n",
    "\n",
    "dart_param = dict()\n",
    "dart_param[\"sample_type\"]= [\"uniform\", \"weighted\"],\n",
    "dart_param[\"normalize_type\"]= [\"tree\", \"forest\"],\n",
    "dart_param[\"rate_drop\"]= (1e-8, 1.0),\n",
    "dart_param[\"skip_drop\"]= (1e-8, 1.0)\n",
    "           \n",
    "#if xgb_params[\"booster\"] == \"dart\":\n",
    "     #xgb_params.update(dart_param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#For Multi-Classification - multi:softmax\n",
    "#For Binary Classificayion - binary:logistic\n",
    "xgb_model = XGBClassifier(\n",
    "        objective='multi:softmax',\n",
    "        booster='dart'\n",
    "        n_estimators=10000, # use large n_estimators deliberately to make use of the early stopping\n",
    "        n_jobs=-1\n",
    "    )\n",
    "fit_params = {\n",
    "    'early_stopping_rounds':30,  \n",
    "    'eval_set':[(X_test, y_test)],\n",
    "    'verbose':0\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# define evaluation\n",
    "#xgb_cv = RepeatedStratifiedKFold(n_splits=5, n_repeats=3, random_state=101)\n",
    "# define the search\n",
    "xgb_search = BayesSearchCV(estimator=xgb_model, search_spaces=xgb_params, fit_params=fit_params,\n",
    "                           cv=5, n_iter=50,\n",
    "                           verbose=0, n_jobs=-1,\n",
    "                           )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# callback handler\n",
    "def on_step(optim_result):\n",
    "    score = xgb_search.best_score_\n",
    "    print (f\"{datetime.now()}: Best Score- {score:.4f}\")\n",
    "    if score >= 0.98:\n",
    "        print('Interrupting!')\n",
    "        return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "# perform the search\n",
    "xgb_search.fit(X_train, y_train, callback=on_step)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "##### Bayesian Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "xgb_pbounds = {\n",
    "    'learning_rate': (1e-7, 1.0),\n",
    "    'min_child_weight': (1e-16, 1e5),\n",
    "    'max_depth': (2,20),\n",
    "    'subsample': (0.3, 1.0),\n",
    "    'colsample_bylevel': (0.3, 1.0), \n",
    "    'colsample_bytree': (0.3, 1.0),\n",
    "    'colsample_bynode': (0.3, 1.0),\n",
    "    'reg_alpha': (0, 100),\n",
    "    'reg_lambda': (0, 100)\n",
    "} "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def xgboost_hyper_param(learning_rate,\n",
    "                        min_child_weight,\n",
    "                        max_depth,\n",
    "                        subsample,\n",
    "                        colsample_bylevel, \n",
    "                        colsample_bytree,\n",
    "                        colsample_bynode,\n",
    "                        reg_alpha,\n",
    "                        reg_lambda):\n",
    "\n",
    "    max_depth = int(max_depth)\n",
    "\n",
    "    clf = XGBClassifier(\n",
    "        objective= \"multi:softmax\",\n",
    "        booster = \"dart\",\n",
    "        n_estimators = 10000,\n",
    "        learning_rate=learning_rate,\n",
    "        min_child_weight= min_child_weight,\n",
    "        max_depth=max_depth,\n",
    "        subsample= subsample,  \n",
    "        colsample_bylevel= colsample_bylevel, \n",
    "        colsample_bytree= colsample_bytree,\n",
    "        colsample_bynode= colsample_bynode,\n",
    "        reg_alpha= reg_alpha,\n",
    "        reg_lambda= reg_lambda\n",
    "        )\n",
    "    \n",
    "    fit_params = {\n",
    "        'early_stopping_rounds':30,\n",
    "        'eval_set':[(X_test, y_test)],\n",
    "        'verbose':0\n",
    "    }\n",
    "\n",
    "    #cv = RepeatedStratifiedKFold(n_splits=5, n_repeats=3, random_state=101)\n",
    "    cv_result = np.mean(cross_val_score(estimator=clf, X=X_train, y=y_train, fit_params=fit_params, \n",
    "                                        cv=5, scoring='accuracy', verbose = 0))\n",
    "    return cv_result\n",
    "\n",
    "xgb_optimizer = BayesianOptimization(\n",
    "    f=xgboost_hyper_param,\n",
    "    pbounds=xgb_pbounds,\n",
    "    #verbose=2,\n",
    "    random_state=1,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "init_points = 8 \n",
    "n_iter = 50 #The more steps the more likely to find a good maximum you are.\n",
    "xgb_optimizer.maximize(n_iter=n_iter, init_points=init_points)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}