higgs_boson_classification_100.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'tpot'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-7dc200b286eb>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpreprocessing\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mLabelEncoder\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mscipy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstats\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mshapiro\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnormaltest\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 10\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtpot\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mTPOTClassifier\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     11\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mhpsklearn\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mHyperoptEstimator\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpca\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmin_max_scaler\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstandard_scaler\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mhpsklearn\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mxgboost_classification\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrandom_forest\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mada_boost\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgradient_boosting\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mextra_trees\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'tpot'"
     ]
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import shap\n",
    "import pickle\n",
    "import ppscore as pps\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from scipy.stats import shapiro, normaltest\n",
    "from tpot import TPOTClassifier\n",
    "from hpsklearn import HyperoptEstimator, pca, min_max_scaler, standard_scaler\n",
    "from hpsklearn import xgboost_classification, random_forest, ada_boost, gradient_boosting, extra_trees\n",
    "from hpsklearn import svc, svc_linear, svc_rbf, svc_poly, svc_sigmoid, liblinear_svc\n",
    "from hpsklearn import any_classifier\n",
    "from hpsklearn import any_preprocessing\n",
    "from hyperopt import tpe, hp\n",
    "from xgboost import XGBClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from catboost import CatBoostClassifier\n",
    "import lightgbm as lgb\n",
    "import catboost as cb\n",
    "import xgboost as xgb\n",
    "import optuna\n",
    "from bayes_opt import BayesianOptimization\n",
    "from skopt import BayesSearchCV\n",
    "from skopt import gp_minimize\n",
    "from skopt.space import Real, Integer, Categorical\n",
    "from skopt.utils import use_named_args\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold, StratifiedKFold, RepeatedStratifiedKFold\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import Perceptron\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.ensemble import IsolationForest\n",
    "from sklearn.neighbors import LocalOutlierFactor\n",
    "from sklearn.covariance import EllipticEnvelope\n",
    "from sklearn.svm import OneClassSVM\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import balanced_accuracy_score, accuracy_score, f1_score, recall_score\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import classification_report,confusion_matrix,plot_confusion_matrix\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler\n",
    "from sklearn.preprocessing import PowerTransformer, QuantileTransformer\n",
    "from sklearn.decomposition import PCA, TruncatedSVD\n",
    "from sklearn.feature_selection import RFE, SelectFromModel, RFECV\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import f_classif, mutual_info_classif\n",
    "from matplotlib import pyplot\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and Clean Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('Data/higgs_boson_training.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EventId</th>\n",
       "      <th>DER_mass_MMC</th>\n",
       "      <th>DER_mass_transverse_met_lep</th>\n",
       "      <th>DER_mass_vis</th>\n",
       "      <th>DER_pt_h</th>\n",
       "      <th>DER_deltaeta_jet_jet</th>\n",
       "      <th>DER_mass_jet_jet</th>\n",
       "      <th>DER_prodeta_jet_jet</th>\n",
       "      <th>DER_deltar_tau_lep</th>\n",
       "      <th>DER_pt_tot</th>\n",
       "      <th>...</th>\n",
       "      <th>PRI_jet_num</th>\n",
       "      <th>PRI_jet_leading_pt</th>\n",
       "      <th>PRI_jet_leading_eta</th>\n",
       "      <th>PRI_jet_leading_phi</th>\n",
       "      <th>PRI_jet_subleading_pt</th>\n",
       "      <th>PRI_jet_subleading_eta</th>\n",
       "      <th>PRI_jet_subleading_phi</th>\n",
       "      <th>PRI_jet_all_pt</th>\n",
       "      <th>Weight</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100000</td>\n",
       "      <td>138.470</td>\n",
       "      <td>51.655</td>\n",
       "      <td>97.827</td>\n",
       "      <td>27.980</td>\n",
       "      <td>0.91</td>\n",
       "      <td>124.711</td>\n",
       "      <td>2.666</td>\n",
       "      <td>3.064</td>\n",
       "      <td>41.928</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>67.435</td>\n",
       "      <td>2.150</td>\n",
       "      <td>0.444</td>\n",
       "      <td>46.062</td>\n",
       "      <td>1.24</td>\n",
       "      <td>-2.475</td>\n",
       "      <td>113.497</td>\n",
       "      <td>0.002653</td>\n",
       "      <td>s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100001</td>\n",
       "      <td>160.937</td>\n",
       "      <td>68.768</td>\n",
       "      <td>103.235</td>\n",
       "      <td>48.146</td>\n",
       "      <td>-999.00</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>3.473</td>\n",
       "      <td>2.078</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>46.226</td>\n",
       "      <td>0.725</td>\n",
       "      <td>1.158</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>-999.00</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>46.226</td>\n",
       "      <td>2.233584</td>\n",
       "      <td>b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100002</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>162.172</td>\n",
       "      <td>125.953</td>\n",
       "      <td>35.635</td>\n",
       "      <td>-999.00</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>3.148</td>\n",
       "      <td>9.336</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>44.251</td>\n",
       "      <td>2.053</td>\n",
       "      <td>-2.028</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>-999.00</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>44.251</td>\n",
       "      <td>2.347389</td>\n",
       "      <td>b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100003</td>\n",
       "      <td>143.905</td>\n",
       "      <td>81.417</td>\n",
       "      <td>80.943</td>\n",
       "      <td>0.414</td>\n",
       "      <td>-999.00</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>3.310</td>\n",
       "      <td>0.414</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>-999.00</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>-0.000</td>\n",
       "      <td>5.446378</td>\n",
       "      <td>b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100004</td>\n",
       "      <td>175.864</td>\n",
       "      <td>16.915</td>\n",
       "      <td>134.805</td>\n",
       "      <td>16.405</td>\n",
       "      <td>-999.00</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>3.891</td>\n",
       "      <td>16.405</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>-999.00</td>\n",
       "      <td>-999.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>6.245333</td>\n",
       "      <td>b</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   EventId  DER_mass_MMC  DER_mass_transverse_met_lep  DER_mass_vis  DER_pt_h  \\\n",
       "0   100000       138.470                       51.655        97.827    27.980   \n",
       "1   100001       160.937                       68.768       103.235    48.146   \n",
       "2   100002      -999.000                      162.172       125.953    35.635   \n",
       "3   100003       143.905                       81.417        80.943     0.414   \n",
       "4   100004       175.864                       16.915       134.805    16.405   \n",
       "\n",
       "   DER_deltaeta_jet_jet  DER_mass_jet_jet  DER_prodeta_jet_jet  \\\n",
       "0                  0.91           124.711                2.666   \n",
       "1               -999.00          -999.000             -999.000   \n",
       "2               -999.00          -999.000             -999.000   \n",
       "3               -999.00          -999.000             -999.000   \n",
       "4               -999.00          -999.000             -999.000   \n",
       "\n",
       "   DER_deltar_tau_lep  DER_pt_tot  ...  PRI_jet_num  PRI_jet_leading_pt  \\\n",
       "0               3.064      41.928  ...            2              67.435   \n",
       "1               3.473       2.078  ...            1              46.226   \n",
       "2               3.148       9.336  ...            1              44.251   \n",
       "3               3.310       0.414  ...            0            -999.000   \n",
       "4               3.891      16.405  ...            0            -999.000   \n",
       "\n",
       "   PRI_jet_leading_eta  PRI_jet_leading_phi  PRI_jet_subleading_pt  \\\n",
       "0                2.150                0.444                 46.062   \n",
       "1                0.725                1.158               -999.000   \n",
       "2                2.053               -2.028               -999.000   \n",
       "3             -999.000             -999.000               -999.000   \n",
       "4             -999.000             -999.000               -999.000   \n",
       "\n",
       "   PRI_jet_subleading_eta  PRI_jet_subleading_phi  PRI_jet_all_pt    Weight  \\\n",
       "0                    1.24                  -2.475         113.497  0.002653   \n",
       "1                 -999.00                -999.000          46.226  2.233584   \n",
       "2                 -999.00                -999.000          44.251  2.347389   \n",
       "3                 -999.00                -999.000          -0.000  5.446378   \n",
       "4                 -999.00                -999.000           0.000  6.245333   \n",
       "\n",
       "   Label  \n",
       "0      s  \n",
       "1      b  \n",
       "2      b  \n",
       "3      b  \n",
       "4      b  \n",
       "\n",
       "[5 rows x 33 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['EventId', 'DER_mass_MMC', 'DER_mass_transverse_met_lep',\n",
       "       'DER_mass_vis', 'DER_pt_h', 'DER_deltaeta_jet_jet', 'DER_mass_jet_jet',\n",
       "       'DER_prodeta_jet_jet', 'DER_deltar_tau_lep', 'DER_pt_tot', 'DER_sum_pt',\n",
       "       'DER_pt_ratio_lep_tau', 'DER_met_phi_centrality',\n",
       "       'DER_lep_eta_centrality', 'PRI_tau_pt', 'PRI_tau_eta', 'PRI_tau_phi',\n",
       "       'PRI_lep_pt', 'PRI_lep_eta', 'PRI_lep_phi', 'PRI_met', 'PRI_met_phi',\n",
       "       'PRI_met_sumet', 'PRI_jet_num', 'PRI_jet_leading_pt',\n",
       "       'PRI_jet_leading_eta', 'PRI_jet_leading_phi', 'PRI_jet_subleading_pt',\n",
       "       'PRI_jet_subleading_eta', 'PRI_jet_subleading_phi', 'PRI_jet_all_pt',\n",
       "       'Weight', 'Label'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 250000 entries, 0 to 249999\n",
      "Data columns (total 33 columns):\n",
      " #   Column                       Non-Null Count   Dtype  \n",
      "---  ------                       --------------   -----  \n",
      " 0   EventId                      250000 non-null  int64  \n",
      " 1   DER_mass_MMC                 250000 non-null  float64\n",
      " 2   DER_mass_transverse_met_lep  250000 non-null  float64\n",
      " 3   DER_mass_vis                 250000 non-null  float64\n",
      " 4   DER_pt_h                     250000 non-null  float64\n",
      " 5   DER_deltaeta_jet_jet         250000 non-null  float64\n",
      " 6   DER_mass_jet_jet             250000 non-null  float64\n",
      " 7   DER_prodeta_jet_jet          250000 non-null  float64\n",
      " 8   DER_deltar_tau_lep           250000 non-null  float64\n",
      " 9   DER_pt_tot                   250000 non-null  float64\n",
      " 10  DER_sum_pt                   250000 non-null  float64\n",
      " 11  DER_pt_ratio_lep_tau         250000 non-null  float64\n",
      " 12  DER_met_phi_centrality       250000 non-null  float64\n",
      " 13  DER_lep_eta_centrality       250000 non-null  float64\n",
      " 14  PRI_tau_pt                   250000 non-null  float64\n",
      " 15  PRI_tau_eta                  250000 non-null  float64\n",
      " 16  PRI_tau_phi                  250000 non-null  float64\n",
      " 17  PRI_lep_pt                   250000 non-null  float64\n",
      " 18  PRI_lep_eta                  250000 non-null  float64\n",
      " 19  PRI_lep_phi                  250000 non-null  float64\n",
      " 20  PRI_met                      250000 non-null  float64\n",
      " 21  PRI_met_phi                  250000 non-null  float64\n",
      " 22  PRI_met_sumet                250000 non-null  float64\n",
      " 23  PRI_jet_num                  250000 non-null  int64  \n",
      " 24  PRI_jet_leading_pt           250000 non-null  float64\n",
      " 25  PRI_jet_leading_eta          250000 non-null  float64\n",
      " 26  PRI_jet_leading_phi          250000 non-null  float64\n",
      " 27  PRI_jet_subleading_pt        250000 non-null  float64\n",
      " 28  PRI_jet_subleading_eta       250000 non-null  float64\n",
      " 29  PRI_jet_subleading_phi       250000 non-null  float64\n",
      " 30  PRI_jet_all_pt               250000 non-null  float64\n",
      " 31  Weight                       250000 non-null  float64\n",
      " 32  Label                        250000 non-null  object \n",
      "dtypes: float64(30), int64(2), object(1)\n",
      "memory usage: 62.9+ MB\n"
     ]
    }
   ],
   "source": [
    "df_train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(250000, 33)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "# calculate duplicates\n",
    "dups = df_train.duplicated()\n",
    "#print(dups)\n",
    "# report if there are any duplicates\n",
    "print(dups.any())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b    164333\n",
       "s     85667\n",
       "Name: Label, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[\"Label\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    164333\n",
       "1     85667\n",
       "Name: Label, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train['Label'] = df_train['Label'].map({'b':0,'s':1})\n",
    "df_train[\"Label\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.521301260245964\n"
     ]
    }
   ],
   "source": [
    "s = df_train[df_train['Label']==1]\n",
    "b = df_train[df_train['Label']==0]\n",
    "outlier_fraction = len(s)/float(len(b))\n",
    "print (outlier_fraction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>ppscore</th>\n",
       "      <th>case</th>\n",
       "      <th>is_valid_score</th>\n",
       "      <th>metric</th>\n",
       "      <th>baseline_score</th>\n",
       "      <th>model_score</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>EventId</td>\n",
       "      <td>EventId</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>predict_itself</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>EventId</td>\n",
       "      <td>DER_mass_MMC</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>regression</td>\n",
       "      <td>True</td>\n",
       "      <td>mean absolute error</td>\n",
       "      <td>196.062021</td>\n",
       "      <td>321.063250</td>\n",
       "      <td>DecisionTreeRegressor()</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>EventId</td>\n",
       "      <td>DER_mass_transverse_met_lep</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>regression</td>\n",
       "      <td>True</td>\n",
       "      <td>mean absolute error</td>\n",
       "      <td>28.247978</td>\n",
       "      <td>37.901123</td>\n",
       "      <td>DecisionTreeRegressor()</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>EventId</td>\n",
       "      <td>DER_mass_vis</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>regression</td>\n",
       "      <td>True</td>\n",
       "      <td>mean absolute error</td>\n",
       "      <td>24.561323</td>\n",
       "      <td>38.002895</td>\n",
       "      <td>DecisionTreeRegressor()</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>EventId</td>\n",
       "      <td>DER_pt_h</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>regression</td>\n",
       "      <td>True</td>\n",
       "      <td>mean absolute error</td>\n",
       "      <td>41.882694</td>\n",
       "      <td>59.955338</td>\n",
       "      <td>DecisionTreeRegressor()</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1084</th>\n",
       "      <td>Label</td>\n",
       "      <td>PRI_jet_subleading_eta</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>regression</td>\n",
       "      <td>True</td>\n",
       "      <td>mean absolute error</td>\n",
       "      <td>286.913230</td>\n",
       "      <td>398.773803</td>\n",
       "      <td>DecisionTreeRegressor()</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1085</th>\n",
       "      <td>Label</td>\n",
       "      <td>PRI_jet_subleading_phi</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>regression</td>\n",
       "      <td>True</td>\n",
       "      <td>mean absolute error</td>\n",
       "      <td>286.943100</td>\n",
       "      <td>398.815250</td>\n",
       "      <td>DecisionTreeRegressor()</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1086</th>\n",
       "      <td>Label</td>\n",
       "      <td>PRI_jet_all_pt</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>regression</td>\n",
       "      <td>True</td>\n",
       "      <td>mean absolute error</td>\n",
       "      <td>65.235753</td>\n",
       "      <td>68.575408</td>\n",
       "      <td>DecisionTreeRegressor()</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1087</th>\n",
       "      <td>Label</td>\n",
       "      <td>Weight</td>\n",
       "      <td>0.343517</td>\n",
       "      <td>regression</td>\n",
       "      <td>True</td>\n",
       "      <td>mean absolute error</td>\n",
       "      <td>1.529808</td>\n",
       "      <td>1.004294</td>\n",
       "      <td>DecisionTreeRegressor()</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1088</th>\n",
       "      <td>Label</td>\n",
       "      <td>Label</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>predict_itself</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1089 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            x                            y   ppscore            case  \\\n",
       "0     EventId                      EventId  1.000000  predict_itself   \n",
       "1     EventId                 DER_mass_MMC  0.000000      regression   \n",
       "2     EventId  DER_mass_transverse_met_lep  0.000000      regression   \n",
       "3     EventId                 DER_mass_vis  0.000000      regression   \n",
       "4     EventId                     DER_pt_h  0.000000      regression   \n",
       "...       ...                          ...       ...             ...   \n",
       "1084    Label       PRI_jet_subleading_eta  0.000000      regression   \n",
       "1085    Label       PRI_jet_subleading_phi  0.000000      regression   \n",
       "1086    Label               PRI_jet_all_pt  0.000000      regression   \n",
       "1087    Label                       Weight  0.343517      regression   \n",
       "1088    Label                        Label  1.000000  predict_itself   \n",
       "\n",
       "      is_valid_score               metric  baseline_score  model_score  \\\n",
       "0               True                 None        0.000000     1.000000   \n",
       "1               True  mean absolute error      196.062021   321.063250   \n",
       "2               True  mean absolute error       28.247978    37.901123   \n",
       "3               True  mean absolute error       24.561323    38.002895   \n",
       "4               True  mean absolute error       41.882694    59.955338   \n",
       "...              ...                  ...             ...          ...   \n",
       "1084            True  mean absolute error      286.913230   398.773803   \n",
       "1085            True  mean absolute error      286.943100   398.815250   \n",
       "1086            True  mean absolute error       65.235753    68.575408   \n",
       "1087            True  mean absolute error        1.529808     1.004294   \n",
       "1088            True                 None        0.000000     1.000000   \n",
       "\n",
       "                        model  \n",
       "0                        None  \n",
       "1     DecisionTreeRegressor()  \n",
       "2     DecisionTreeRegressor()  \n",
       "3     DecisionTreeRegressor()  \n",
       "4     DecisionTreeRegressor()  \n",
       "...                       ...  \n",
       "1084  DecisionTreeRegressor()  \n",
       "1085  DecisionTreeRegressor()  \n",
       "1086  DecisionTreeRegressor()  \n",
       "1087  DecisionTreeRegressor()  \n",
       "1088                     None  \n",
       "\n",
       "[1089 rows x 9 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pps.matrix(df_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df_train.drop(['Label'],axis=1)\n",
    "y = df_train['Label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#y = LabelEncoder().fit_transform(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, stratify=y, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.30, stratify=y_train, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Data (X): (122500, 32)\n",
      "Test Data (X): (75000, 32)\n",
      "Validation Data (X): (52500, 32)\n"
     ]
    }
   ],
   "source": [
    "print(f\"Train Data (X): {X_train.shape}\")\n",
    "print(f\"Test Data (X): {X_test.shape}\")\n",
    "print(f\"Validation Data (X): {X_valid.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Data (y): (122500,)\n",
      "Test Data (y): (75000,)\n",
      "Validation Data (y): (52500,)\n"
     ]
    }
   ],
   "source": [
    "print(f\"Train Data (y): {y_train.shape}\")\n",
    "print(f\"Test Data (y): {y_test.shape}\")\n",
    "print(f\"Validation Data (y): {y_valid.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find Best Algorithm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Find Best Algorithm with Best Params Using HyperoptEstimator AutoML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-26 22:28:19.114837\n",
      "100%|██████████| 1/1 [05:46<00:00, 346.25s/trial, best loss: 0.0]\n",
      "100%|██████████| 2/2 [00:01<00:00,  1.09s/trial, best loss: 0.0]\n",
      "100%|██████████| 3/3 [01:03<00:00, 63.66s/trial, best loss: 0.0]\n",
      "100%|██████████| 4/4 [00:12<00:00, 12.31s/trial, best loss: 0.0]\n",
      "100%|██████████| 5/5 [00:05<00:00,  5.93s/trial, best loss: 0.0]\n",
      "100%|██████████| 6/6 [07:38<00:00, 458.12s/trial, best loss: 0.0]\n",
      "100%|██████████| 7/7 [03:38<00:00, 218.94s/trial, best loss: 0.0]\n",
      "100%|██████████| 8/8 [03:11<00:00, 191.30s/trial, best loss: 0.0]\n",
      "100%|██████████| 9/9 [00:00<00:00,  1.72trial/s, best loss: 0.0]\n",
      "100%|██████████| 10/10 [1:23:20<00:00, 5000.12s/trial, best loss: 0.0]\n",
      "100%|██████████| 11/11 [09:56<00:00, 596.09s/trial, best loss: 0.0]\n",
      "100%|██████████| 12/12 [16:41<00:00, 1001.33s/trial, best loss: 0.0]\n",
      "100%|██████████| 13/13 [04:35<00:00, 275.07s/trial, best loss: 0.0]\n",
      "100%|██████████| 14/14 [01:30<00:00, 90.63s/trial, best loss: 0.0]\n",
      "100%|██████████| 15/15 [01:33<00:00, 93.86s/trial, best loss: 0.0]\n",
      "100%|██████████| 16/16 [00:01<00:00,  1.13s/trial, best loss: 0.0]\n",
      "100%|██████████| 17/17 [05:56<00:00, 356.90s/trial, best loss: 0.0]\n",
      "100%|██████████| 18/18 [00:39<00:00, 39.27s/trial, best loss: 0.0]\n",
      "100%|██████████| 19/19 [00:01<00:00,  1.99s/trial, best loss: 0.0]\n",
      "100%|██████████| 20/20 [01:11<00:00, 71.65s/trial, best loss: 0.0]\n",
      "100%|██████████| 21/21 [00:01<00:00,  1.24s/trial, best loss: 0.0]\n",
      "100%|██████████| 22/22 [09:16<00:00, 556.35s/trial, best loss: 0.0]\n",
      "100%|██████████| 23/23 [00:03<00:00,  3.53s/trial, best loss: 0.0]\n",
      "100%|██████████| 24/24 [46:24<00:00, 2784.68s/trial, best loss: 0.0]\n",
      "100%|██████████| 25/25 [08:07<00:00, 487.55s/trial, best loss: 0.0]\n",
      "100%|██████████| 26/26 [02:52<00:00, 172.87s/trial, best loss: 0.0]\n",
      "100%|██████████| 27/27 [02:59<00:00, 179.33s/trial, best loss: 0.0]\n",
      "100%|██████████| 28/28 [00:00<00:00,  1.05trial/s, best loss: 0.0]\n",
      "100%|██████████| 29/29 [00:53<00:00, 53.72s/trial, best loss: 0.0]\n",
      "100%|██████████| 30/30 [00:02<00:00,  2.93s/trial, best loss: 0.0]\n",
      "100%|██████████| 31/31 [00:56<00:00, 56.75s/trial, best loss: 0.0]\n",
      "100%|██████████| 32/32 [03:41<00:00, 221.71s/trial, best loss: 0.0]\n",
      "100%|██████████| 33/33 [02:35<00:00, 155.30s/trial, best loss: 0.0]\n",
      "100%|██████████| 34/34 [21:13<00:00, 1273.85s/trial, best loss: 0.0]\n",
      "100%|██████████| 35/35 [01:02<00:00, 62.82s/trial, best loss: 0.0]\n",
      "100%|██████████| 36/36 [00:00<00:00,  1.23trial/s, best loss: 0.0]\n",
      "100%|██████████| 37/37 [00:07<00:00,  7.71s/trial, best loss: 0.0]\n",
      "100%|██████████| 38/38 [00:02<00:00,  2.91s/trial, best loss: 0.0]\n",
      "100%|██████████| 39/39 [00:00<00:00,  1.00trial/s, best loss: 0.0]\n",
      "100%|██████████| 40/40 [01:23<00:00, 83.73s/trial, best loss: 0.0]\n",
      "100%|██████████| 41/41 [00:10<00:00, 10.09s/trial, best loss: 0.0]\n",
      "100%|██████████| 42/42 [00:00<00:00,  1.06trial/s, best loss: 0.0]\n",
      "100%|██████████| 43/43 [00:04<00:00,  4.51s/trial, best loss: 0.0]\n",
      "100%|██████████| 44/44 [00:01<00:00,  1.19s/trial, best loss: 0.0]\n",
      "100%|██████████| 45/45 [00:02<00:00,  2.57s/trial, best loss: 0.0]\n",
      "100%|██████████| 46/46 [13:38<00:00, 818.29s/trial, best loss: 0.0]\n",
      " 98%|█████████▊| 46/47 [00:00<?, ?trial/s, best loss=?]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "job exception: Input contains NaN, infinity or a value too large for dtype('float64').\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 46/47 [00:02<?, ?trial/s, best loss=?]\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Input contains NaN, infinity or a value too large for dtype('float64').",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-1f63e25bb337>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     11\u001b[0m                                 trial_timeout=5000)\n\u001b[1;32m     12\u001b[0m     \u001b[0;31m# perform the search\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0;31m#if acc > accuracy:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/py_caret/lib/python3.6/site-packages/hpsklearn/estimator.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, EX_list, valid_size, n_folds, cv_shuffle, warm_start, random_state, weights)\u001b[0m\n\u001b[1;32m    785\u001b[0m                 increment = min(self.fit_increment,\n\u001b[1;32m    786\u001b[0m                                 adjusted_max_evals - len(self.trials.trials))\n\u001b[0;32m--> 787\u001b[0;31m                 \u001b[0mfit_iter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mincrement\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    788\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mfilename\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    789\u001b[0m                     \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'wb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mdump_file\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/py_caret/lib/python3.6/site-packages/hpsklearn/estimator.py\u001b[0m in \u001b[0;36mfit_iter\u001b[0;34m(self, X, y, EX_list, valid_size, n_folds, cv_shuffle, warm_start, random_state, weights, increment)\u001b[0m\n\u001b[1;32m    695\u001b[0m                               \u001b[0;31m#    so we notice them.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    696\u001b[0m                               \u001b[0mcatch_eval_exceptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 697\u001b[0;31m                               \u001b[0mreturn_argmin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;31m# -- in case no success so far\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    698\u001b[0m                              )\n\u001b[1;32m    699\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/py_caret/lib/python3.6/site-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mfmin\u001b[0;34m(fn, space, algo, max_evals, timeout, loss_threshold, trials, rstate, allow_trials_fmin, pass_expr_memo_ctrl, catch_eval_exceptions, verbose, return_argmin, points_to_evaluate, max_queue_len, show_progressbar, early_stop_fn, trials_save_file)\u001b[0m\n\u001b[1;32m    520\u001b[0m             \u001b[0mshow_progressbar\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshow_progressbar\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    521\u001b[0m             \u001b[0mearly_stop_fn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mearly_stop_fn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 522\u001b[0;31m             \u001b[0mtrials_save_file\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrials_save_file\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    523\u001b[0m         )\n\u001b[1;32m    524\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/py_caret/lib/python3.6/site-packages/hyperopt/base.py\u001b[0m in \u001b[0;36mfmin\u001b[0;34m(self, fn, space, algo, max_evals, timeout, loss_threshold, max_queue_len, rstate, verbose, pass_expr_memo_ctrl, catch_eval_exceptions, return_argmin, show_progressbar, early_stop_fn, trials_save_file)\u001b[0m\n\u001b[1;32m    697\u001b[0m             \u001b[0mshow_progressbar\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshow_progressbar\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    698\u001b[0m             \u001b[0mearly_stop_fn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mearly_stop_fn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 699\u001b[0;31m             \u001b[0mtrials_save_file\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrials_save_file\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    700\u001b[0m         )\n\u001b[1;32m    701\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/py_caret/lib/python3.6/site-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mfmin\u001b[0;34m(fn, space, algo, max_evals, timeout, loss_threshold, trials, rstate, allow_trials_fmin, pass_expr_memo_ctrl, catch_eval_exceptions, verbose, return_argmin, points_to_evaluate, max_queue_len, show_progressbar, early_stop_fn, trials_save_file)\u001b[0m\n\u001b[1;32m    551\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    552\u001b[0m     \u001b[0;31m# next line is where the fmin is actually executed\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 553\u001b[0;31m     \u001b[0mrval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexhaust\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    554\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    555\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mreturn_argmin\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/py_caret/lib/python3.6/site-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mexhaust\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    354\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mexhaust\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    355\u001b[0m         \u001b[0mn_done\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 356\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_evals\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mn_done\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mblock_until_done\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masynchronous\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    357\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrefresh\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    358\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/py_caret/lib/python3.6/site-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, N, block_until_done)\u001b[0m\n\u001b[1;32m    290\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    291\u001b[0m                     \u001b[0;31m# -- loop over trials and do the jobs directly\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 292\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mserial_evaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    293\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    294\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrefresh\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/py_caret/lib/python3.6/site-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mserial_evaluate\u001b[0;34m(self, N)\u001b[0m\n\u001b[1;32m    168\u001b[0m                 \u001b[0mctrl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbase\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCtrl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcurrent_trial\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    169\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 170\u001b[0;31m                     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdomain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspec\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mctrl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    171\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    172\u001b[0m                     \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"job exception: %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/py_caret/lib/python3.6/site-packages/hyperopt/base.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, config, ctrl, attach_attachments)\u001b[0m\n\u001b[1;32m    905\u001b[0m                 \u001b[0mprint_node_on_error\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrec_eval_print_node_on_error\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    906\u001b[0m             )\n\u001b[0;32m--> 907\u001b[0;31m             \u001b[0mrval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpyll_rval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    908\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    909\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumber\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/py_caret/lib/python3.6/site-packages/hpsklearn/estimator.py\u001b[0m in \u001b[0;36mfn_with_timeout\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    658\u001b[0m             \u001b[0;32massert\u001b[0m \u001b[0mfn_rval\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m'raise'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'return'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    659\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mfn_rval\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'raise'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 660\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mfn_rval\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    661\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    662\u001b[0m             \u001b[0;31m# -- remove potentially large objects from the rval\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Input contains NaN, infinity or a value too large for dtype('float64')."
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "accuracy = 0\n",
    "best_model = None\n",
    "#for i in range(100):\n",
    "    #print (i)\n",
    "model = HyperoptEstimator(  classifier= any_classifier('cla'), \n",
    "                                preprocessing= any_preprocessing('pre'), \n",
    "                                algo=tpe.suggest, \n",
    "                                max_evals=50, \n",
    "                                trial_timeout=5000)\n",
    "    # perform the search\n",
    "model.fit(X_train, y_train)\n",
    "acc = model.score(X_test, y_test)\n",
    "    #if acc > accuracy:\n",
    "        #accuracy = acc\n",
    "        #best_model = model\n",
    "        #print (accuracy)\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))\n",
    "print (f\"Job Ended at: {fin_time}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'learner': XGBClassifier(base_score=0.5, booster=None,\n",
      "              colsample_bylevel=0.6573364597182277, colsample_bynode=None,\n",
      "              colsample_bytree=0.5102849665204783, gamma=0.0016299786974779509,\n",
      "              gpu_id=None, importance_type='gain', interaction_constraints=None,\n",
      "              learning_rate=0.0001052843797478166, max_delta_step=0,\n",
      "              max_depth=5, min_child_weight=21, missing=nan,\n",
      "              monotone_constraints=None, n_estimators=5000, n_jobs=None,\n",
      "              num_parallel_tree=None, random_state=None,\n",
      "              reg_alpha=0.004292169845548234, reg_lambda=2.788702137746418,\n",
      "              scale_pos_weight=1, seed=1, subsample=0.9383008964042696,\n",
      "              tree_method=None, validate_parameters=None, verbosity=None), 'preprocs': (MinMaxScaler(feature_range=(0.0, 1.0)),), 'ex_preprocs': ()}\n"
     ]
    }
   ],
   "source": [
    "#summarize the best model\n",
    "print(\"Accuracy: %.3f\" % acc)\n",
    "print(model.best_model())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-27 10:53:56.897099\n",
      "Execution time :  0:07:29.147834\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "xgb_model = XGBClassifier(base_score=0.5, booster=None,\n",
    "              colsample_bylevel=0.6573364597182277, colsample_bynode=None,\n",
    "              colsample_bytree=0.5102849665204783, gamma=0.0016299786974779509,\n",
    "              gpu_id=None, importance_type='gain', interaction_constraints=None,\n",
    "              learning_rate=0.0001052843797478166, max_delta_step=0,\n",
    "              max_depth=5, min_child_weight=21, missing=None,\n",
    "              monotone_constraints=None, n_estimators=5000, n_jobs=None,\n",
    "              num_parallel_tree=None, random_state=None,\n",
    "              reg_alpha=0.004292169845548234, reg_lambda=2.788702137746418,\n",
    "              scale_pos_weight=1, seed=1, subsample=0.9383008964042696,\n",
    "              tree_method=None, validate_parameters=None, verbosity=None)\n",
    "\n",
    "pipeline = Pipeline(steps=[('norm', MinMaxScaler(feature_range=(0.0, 1.0))), ('m', xgb_model)])\n",
    "\n",
    "fit_params = {\n",
    "    'm__early_stopping_rounds':30,  \n",
    "    'm__eval_set':[(X_valid, y_valid)],\n",
    "    'm__verbose':0\n",
    "}\n",
    "\n",
    "pipeline.fit(X_train, y_train)# **fit_params)\n",
    "prediction = pipeline.predict(X_test)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     49300\n",
      "           1       1.00      1.00      1.00     25700\n",
      "\n",
      "    accuracy                           1.00     75000\n",
      "   macro avg       1.00      1.00      1.00     75000\n",
      "weighted avg       1.00      1.00      1.00     75000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, prediction))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Save Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# save the model to disk\n",
    "filename = 'finalized_higgs_boson_model_1.0.sav'\n",
    "pickle.dump(pipeline, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Load Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-27 11:12:44.504891\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     49300\n",
      "           1       1.00      1.00      1.00     25700\n",
      "\n",
      "    accuracy                           1.00     75000\n",
      "   macro avg       1.00      1.00      1.00     75000\n",
      "weighted avg       1.00      1.00      1.00     75000\n",
      "\n",
      "Execution time :  0:00:06.633323\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "# load the model from disk\n",
    "loaded_model = pickle.load(open(filename, 'rb'))\n",
    "prediction = loaded_model.predict(X_test)\n",
    "print(classification_report(y_test, prediction))\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.3 64-bit ('base': conda)",
   "language": "python",
   "name": "python38364bitbaseconda5399315a2a954424bee177201f1b943d"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}