Otto_Group_Product_Classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mhossain/anaconda3/lib/python3.8/site-packages/tpot/builtins/__init__.py:36: UserWarning: Warning: optional dependency `torch` is not available. - skipping import of NN models.\n",
      "  warnings.warn(\"Warning: optional dependency `torch` is not available. - skipping import of NN models.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARN: OMP_NUM_THREADS=None =>\n",
      "... If you are using openblas if you are using openblas set OMP_NUM_THREADS=1 or risk subprocess calls hanging indefinitely\n"
     ]
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from tpot import TPOTClassifier\n",
    "from hpsklearn import HyperoptEstimator, pca, min_max_scaler, standard_scaler\n",
    "from hpsklearn import xgboost_classification, random_forest, ada_boost, gradient_boosting, extra_trees\n",
    "from hpsklearn import svc, svc_linear, svc_rbf, svc_poly, svc_sigmoid, liblinear_svc\n",
    "from hpsklearn import any_classifier\n",
    "from hpsklearn import any_preprocessing\n",
    "from hyperopt import tpe, hp\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold, StratifiedKFold, RepeatedStratifiedKFold\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import Perceptron\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.ensemble import IsolationForest\n",
    "from sklearn.neighbors import LocalOutlierFactor\n",
    "from sklearn.covariance import EllipticEnvelope\n",
    "from sklearn.svm import OneClassSVM\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import accuracy_score, f1_score, recall_score, classification_report,confusion_matrix,plot_confusion_matrix\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler\n",
    "from sklearn.preprocessing import PowerTransformer, QuantileTransformer\n",
    "from sklearn.decomposition import PCA, TruncatedSVD\n",
    "from sklearn.feature_selection import RFE, SelectFromModel, RFECV\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import f_classif, mutual_info_classif\n",
    "from matplotlib import pyplot\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Load and Clean Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('Data/otto_group_Product_train.csv')\n",
    "#df_test = pd.read_csv('Data/otto_group_Product_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>feat_1</th>\n",
       "      <th>feat_2</th>\n",
       "      <th>feat_3</th>\n",
       "      <th>feat_4</th>\n",
       "      <th>feat_5</th>\n",
       "      <th>feat_6</th>\n",
       "      <th>feat_7</th>\n",
       "      <th>feat_8</th>\n",
       "      <th>feat_9</th>\n",
       "      <th>...</th>\n",
       "      <th>feat_85</th>\n",
       "      <th>feat_86</th>\n",
       "      <th>feat_87</th>\n",
       "      <th>feat_88</th>\n",
       "      <th>feat_89</th>\n",
       "      <th>feat_90</th>\n",
       "      <th>feat_91</th>\n",
       "      <th>feat_92</th>\n",
       "      <th>feat_93</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 95 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  feat_1  feat_2  feat_3  feat_4  feat_5  feat_6  feat_7  feat_8  feat_9  \\\n",
       "0   1       1       0       0       0       0       0       0       0       0   \n",
       "1   2       0       0       0       0       0       0       0       1       0   \n",
       "2   3       0       0       0       0       0       0       0       1       0   \n",
       "3   4       1       0       0       1       6       1       5       0       0   \n",
       "4   5       0       0       0       0       0       0       0       0       0   \n",
       "\n",
       "   ...  feat_85  feat_86  feat_87  feat_88  feat_89  feat_90  feat_91  \\\n",
       "0  ...        1        0        0        0        0        0        0   \n",
       "1  ...        0        0        0        0        0        0        0   \n",
       "2  ...        0        0        0        0        0        0        0   \n",
       "3  ...        0        1        2        0        0        0        0   \n",
       "4  ...        1        0        0        0        0        1        0   \n",
       "\n",
       "   feat_92  feat_93   target  \n",
       "0        0        0  Class_1  \n",
       "1        0        0  Class_1  \n",
       "2        0        0  Class_1  \n",
       "3        0        0  Class_1  \n",
       "4        0        0  Class_1  \n",
       "\n",
       "[5 rows x 95 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 61878 entries, 0 to 61877\n",
      "Data columns (total 95 columns):\n",
      " #   Column   Non-Null Count  Dtype \n",
      "---  ------   --------------  ----- \n",
      " 0   id       61878 non-null  int64 \n",
      " 1   feat_1   61878 non-null  int64 \n",
      " 2   feat_2   61878 non-null  int64 \n",
      " 3   feat_3   61878 non-null  int64 \n",
      " 4   feat_4   61878 non-null  int64 \n",
      " 5   feat_5   61878 non-null  int64 \n",
      " 6   feat_6   61878 non-null  int64 \n",
      " 7   feat_7   61878 non-null  int64 \n",
      " 8   feat_8   61878 non-null  int64 \n",
      " 9   feat_9   61878 non-null  int64 \n",
      " 10  feat_10  61878 non-null  int64 \n",
      " 11  feat_11  61878 non-null  int64 \n",
      " 12  feat_12  61878 non-null  int64 \n",
      " 13  feat_13  61878 non-null  int64 \n",
      " 14  feat_14  61878 non-null  int64 \n",
      " 15  feat_15  61878 non-null  int64 \n",
      " 16  feat_16  61878 non-null  int64 \n",
      " 17  feat_17  61878 non-null  int64 \n",
      " 18  feat_18  61878 non-null  int64 \n",
      " 19  feat_19  61878 non-null  int64 \n",
      " 20  feat_20  61878 non-null  int64 \n",
      " 21  feat_21  61878 non-null  int64 \n",
      " 22  feat_22  61878 non-null  int64 \n",
      " 23  feat_23  61878 non-null  int64 \n",
      " 24  feat_24  61878 non-null  int64 \n",
      " 25  feat_25  61878 non-null  int64 \n",
      " 26  feat_26  61878 non-null  int64 \n",
      " 27  feat_27  61878 non-null  int64 \n",
      " 28  feat_28  61878 non-null  int64 \n",
      " 29  feat_29  61878 non-null  int64 \n",
      " 30  feat_30  61878 non-null  int64 \n",
      " 31  feat_31  61878 non-null  int64 \n",
      " 32  feat_32  61878 non-null  int64 \n",
      " 33  feat_33  61878 non-null  int64 \n",
      " 34  feat_34  61878 non-null  int64 \n",
      " 35  feat_35  61878 non-null  int64 \n",
      " 36  feat_36  61878 non-null  int64 \n",
      " 37  feat_37  61878 non-null  int64 \n",
      " 38  feat_38  61878 non-null  int64 \n",
      " 39  feat_39  61878 non-null  int64 \n",
      " 40  feat_40  61878 non-null  int64 \n",
      " 41  feat_41  61878 non-null  int64 \n",
      " 42  feat_42  61878 non-null  int64 \n",
      " 43  feat_43  61878 non-null  int64 \n",
      " 44  feat_44  61878 non-null  int64 \n",
      " 45  feat_45  61878 non-null  int64 \n",
      " 46  feat_46  61878 non-null  int64 \n",
      " 47  feat_47  61878 non-null  int64 \n",
      " 48  feat_48  61878 non-null  int64 \n",
      " 49  feat_49  61878 non-null  int64 \n",
      " 50  feat_50  61878 non-null  int64 \n",
      " 51  feat_51  61878 non-null  int64 \n",
      " 52  feat_52  61878 non-null  int64 \n",
      " 53  feat_53  61878 non-null  int64 \n",
      " 54  feat_54  61878 non-null  int64 \n",
      " 55  feat_55  61878 non-null  int64 \n",
      " 56  feat_56  61878 non-null  int64 \n",
      " 57  feat_57  61878 non-null  int64 \n",
      " 58  feat_58  61878 non-null  int64 \n",
      " 59  feat_59  61878 non-null  int64 \n",
      " 60  feat_60  61878 non-null  int64 \n",
      " 61  feat_61  61878 non-null  int64 \n",
      " 62  feat_62  61878 non-null  int64 \n",
      " 63  feat_63  61878 non-null  int64 \n",
      " 64  feat_64  61878 non-null  int64 \n",
      " 65  feat_65  61878 non-null  int64 \n",
      " 66  feat_66  61878 non-null  int64 \n",
      " 67  feat_67  61878 non-null  int64 \n",
      " 68  feat_68  61878 non-null  int64 \n",
      " 69  feat_69  61878 non-null  int64 \n",
      " 70  feat_70  61878 non-null  int64 \n",
      " 71  feat_71  61878 non-null  int64 \n",
      " 72  feat_72  61878 non-null  int64 \n",
      " 73  feat_73  61878 non-null  int64 \n",
      " 74  feat_74  61878 non-null  int64 \n",
      " 75  feat_75  61878 non-null  int64 \n",
      " 76  feat_76  61878 non-null  int64 \n",
      " 77  feat_77  61878 non-null  int64 \n",
      " 78  feat_78  61878 non-null  int64 \n",
      " 79  feat_79  61878 non-null  int64 \n",
      " 80  feat_80  61878 non-null  int64 \n",
      " 81  feat_81  61878 non-null  int64 \n",
      " 82  feat_82  61878 non-null  int64 \n",
      " 83  feat_83  61878 non-null  int64 \n",
      " 84  feat_84  61878 non-null  int64 \n",
      " 85  feat_85  61878 non-null  int64 \n",
      " 86  feat_86  61878 non-null  int64 \n",
      " 87  feat_87  61878 non-null  int64 \n",
      " 88  feat_88  61878 non-null  int64 \n",
      " 89  feat_89  61878 non-null  int64 \n",
      " 90  feat_90  61878 non-null  int64 \n",
      " 91  feat_91  61878 non-null  int64 \n",
      " 92  feat_92  61878 non-null  int64 \n",
      " 93  feat_93  61878 non-null  int64 \n",
      " 94  target   61878 non-null  object\n",
      "dtypes: int64(94), object(1)\n",
      "memory usage: 44.8+ MB\n"
     ]
    }
   ],
   "source": [
    "df_train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(61878, 95)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id         0\n",
       "feat_1     0\n",
       "feat_2     0\n",
       "feat_3     0\n",
       "feat_4     0\n",
       "          ..\n",
       "feat_90    0\n",
       "feat_91    0\n",
       "feat_92    0\n",
       "feat_93    0\n",
       "target     0\n",
       "Length: 95, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "null_columns = {}\n",
    "\n",
    "all_columns = df_train.isnull().sum().sort_values(ascending=False)\n",
    "for item in all_columns.index:\n",
    "    if all_columns[item] > 0:\n",
    "        null_columns[item] = 100* all_columns[item]/len(df)\n",
    "        \n",
    "null_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "# calculate duplicates\n",
    "dups = df_train.duplicated()\n",
    "#print(dups)\n",
    "# report if there are any duplicates\n",
    "print(dups.any())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "target         9\n",
       "feat_6         9\n",
       "feat_5        15\n",
       "feat_21       15\n",
       "feat_37       18\n",
       "           ...  \n",
       "feat_90       91\n",
       "feat_74      101\n",
       "feat_19      105\n",
       "feat_73      115\n",
       "id         61878\n",
       "Length: 95, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# summarize the number of unique values in each column\n",
    "df_train.nunique().sort_values(ascending=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Class_1', 'Class_1', 'Class_1', ..., 'Class_9', 'Class_9',\n",
       "       'Class_9'], dtype=object)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[\"target\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Class_2    16122\n",
       "Class_6    14135\n",
       "Class_8     8464\n",
       "Class_3     8004\n",
       "Class_9     4955\n",
       "Class_7     2839\n",
       "Class_5     2739\n",
       "Class_4     2691\n",
       "Class_1     1929\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[\"target\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "X = df_train.drop(['id', 'target'],axis=1)\n",
    "y = df_train['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "y = LabelEncoder().fit_transform(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Remove correlated features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "#### Identify Highly Correlated Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Create correlation matrix\n",
    "corr_matrix = X.corr().abs()\n",
    "print(); print(corr_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Select upper triangle of correlation matrix\n",
    "upper_tri = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))\n",
    "print(); print(upper_tri)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Find index of feature columns with correlation greater than 0.95\n",
    "to_drop = [column for column in upper_tri.columns if any(upper_tri[column] > 0.80)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "print(to_drop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "len(to_drop)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "#### Drop Marked Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Drop features \n",
    "if len(to_drop) > 0:\n",
    "    X = X.drop(X[to_drop], axis=1)\n",
    "    print (X.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Train Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.20, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Find Best Algorithm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "#### Find Best Algorithm with Best Params Using HyperoptEstimator AutoML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "preproc = hp.choice('myprepros_name', \n",
    "                    [\n",
    "                        [min_max_scaler('myprepros_name.only_norm')],\n",
    "                        [standard_scaler('myprepros_name.only_std_scaler')],\n",
    "                        [pca('myprepros_name.only_pca')],\n",
    "                        [pca('myprepros_name.pca'), min_max_scaler('myprepros_name.norm')],\n",
    "                        [min_max_scaler('myprepros_name.first_norm'), standard_scaler('myprepros_name.second_std_scaler')],\n",
    "                        []\n",
    "                    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "clf = hp.choice( 'model_name',\n",
    "            [ random_forest ('model_name.random_forest'),\n",
    "            ada_boost ('model_name.ada_boost'),\n",
    "            gradient_boosting ('model_name.gradient_boosting'),\n",
    "            xgboost_classification ('model_name.xgboost_classification'),\n",
    "            extra_trees('model_name.extra_trees'),\n",
    "            svc_linear('model_name.svc_linear'),\n",
    "            svc_rbf('model_name.svc_rbf'),\n",
    "            svc('model_name.svc')]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#preprocessing=[min_max_scaler('norm_scaler'), pca('my_pca') ]\n",
    "#classifier=xgboost_classification('xgboost')\n",
    "#preprocessing=any_preprocessing('pre')\n",
    "#preprocessing = preproc\n",
    "#classifier=any_classifier('cla')\n",
    "#classifier=clf\n",
    "\n",
    "#01. 100 times with classifier=any_classifier('cla') & preprocessing=any_preprocessing('pre')\n",
    "\n",
    "#02. 100 times with classifier=clf & preprocessing = preproc\n",
    "\n",
    "#03. 100 times with classifier=xgboost_classification('xgboost') & preprocessing = preproc\n",
    "\n",
    "#04. 100 times with classifier=xgboost_classification('xgboost') & preprocessing = any_preprocessing('pre')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-04 10:56:12.812090\n",
      "100%|██████████████████████████████████████████████████████████████| 1/1 [03:32<00:00, 212.86s/trial, best loss: 0.2234566342633506]\n",
      "100%|███████████████████████████████████████████████████████████████| 2/2 [01:29<00:00, 44.73s/trial, best loss: 0.2234566342633506]\n",
      "100%|███████████████████████████████████████████████████████████████| 3/3 [00:04<00:00,  1.60s/trial, best loss: 0.2234566342633506]\n",
      "100%|███████████████████████████████████████████████████████████████| 4/4 [04:32<00:00, 68.23s/trial, best loss: 0.2234566342633506]\n",
      "100%|███████████████████████████████████████████████████████████████| 5/5 [00:26<00:00,  5.37s/trial, best loss: 0.2234566342633506]\n",
      "100%|█████████████████████████████████████████████████████████████| 6/6 [27:00<00:00, 270.03s/trial, best loss: 0.20388839792955438]\n",
      "100%|██████████████████████████████████████████████████████████████| 7/7 [09:02<00:00, 77.45s/trial, best loss: 0.20388839792955438]\n",
      "100%|██████████████████████████████████████████████████████████████| 8/8 [00:09<00:00,  1.18s/trial, best loss: 0.20388839792955438]\n",
      "100%|██████████████████████████████████████████████████████████████| 9/9 [00:42<00:00,  4.69s/trial, best loss: 0.20388839792955438]\n",
      "100%|████████████████████████████████████████████████████████████| 10/10 [00:41<00:00,  4.16s/trial, best loss: 0.20388839792955438]\n",
      "100%|████████████████████████████████████████████████████████████| 11/11 [08:29<00:00, 46.29s/trial, best loss: 0.20388839792955438]\n",
      "100%|████████████████████████████████████████████████████████████| 12/12 [00:28<00:00,  2.41s/trial, best loss: 0.20388839792955438]\n",
      "100%|████████████████████████████████████████████████████████████| 13/13 [00:08<00:00,  1.57trial/s, best loss: 0.20388839792955438]\n",
      "100%|███████████████████████████████████████████████████████████| 14/14 [27:25<00:00, 117.53s/trial, best loss: 0.18116399444514586]\n",
      "100%|████████████████████████████████████████████████████████████| 15/15 [00:08<00:00,  1.77trial/s, best loss: 0.18116399444514586]\n",
      "100%|████████████████████████████████████████████████████████████| 16/16 [01:13<00:00,  4.61s/trial, best loss: 0.18116399444514586]\n",
      "100%|████████████████████████████████████████████████████████████| 17/17 [01:15<00:00,  4.42s/trial, best loss: 0.18116399444514586]\n",
      "100%|████████████████████████████████████████████████████████████| 18/18 [01:43<00:00,  5.73s/trial, best loss: 0.18116399444514586]\n",
      "100%|███████████████████████████████████████████████████████████| 19/19 [35:34<00:00, 112.36s/trial, best loss: 0.18116399444514586]\n",
      "100%|████████████████████████████████████████████████████████████| 20/20 [00:07<00:00,  2.78trial/s, best loss: 0.18116399444514586]\n",
      "100%|████████████████████████████████████████████████████████████| 21/21 [00:10<00:00,  2.08trial/s, best loss: 0.18116399444514586]\n",
      "100%|████████████████████████████████████████████████████████████| 22/22 [48:33<00:00, 132.42s/trial, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 23/23 [02:11<00:00,  5.71s/trial, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 24/24 [01:20<00:00,  3.36s/trial, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 25/25 [17:21<00:00, 41.65s/trial, best loss: 0.1785128140386315]\n",
      "100%|████████████████████████████████████████████████████████████| 26/26 [52:54<00:00, 122.08s/trial, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 27/27 [41:57<00:00, 93.24s/trial, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 28/28 [15:48<00:00, 33.86s/trial, best loss: 0.1785128140386315]\n",
      "100%|██████████████████████████████████████████████████████████| 29/29 [1:08:48<00:00, 142.37s/trial, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 30/30 [00:04<00:00,  6.72trial/s, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 31/31 [00:04<00:00,  6.42trial/s, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 32/32 [00:04<00:00,  7.94trial/s, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 33/33 [40:19<00:00, 73.31s/trial, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 34/34 [00:08<00:00,  3.86trial/s, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 35/35 [05:55<00:00, 10.16s/trial, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 36/36 [00:35<00:00,  1.01trial/s, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 37/37 [30:43<00:00, 49.83s/trial, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 38/38 [30:12<00:00, 47.70s/trial, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 39/39 [04:04<00:00,  6.28s/trial, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 40/40 [46:56<00:00, 70.42s/trial, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 41/41 [00:09<00:00,  4.20trial/s, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 42/42 [14:43<00:00, 21.04s/trial, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 43/43 [01:09<00:00,  1.61s/trial, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 44/44 [01:10<00:00,  1.59s/trial, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 45/45 [00:12<00:00,  3.52trial/s, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 46/46 [41:19<00:00, 53.90s/trial, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 47/47 [03:50<00:00,  4.91s/trial, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 48/48 [00:07<00:00,  6.31trial/s, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 49/49 [01:11<00:00,  1.46s/trial, best loss: 0.1785128140386315]\n",
      "100%|█████████████████████████████████████████████████████████████| 50/50 [03:36<00:00,  4.32s/trial, best loss: 0.1785128140386315]\n",
      "[20:56:21] WARNING: C:\\Users\\Administrator\\workspace\\xgboost-win64_release_1.1.0\\src\\learner.cc:480: \n",
      "Parameters: { scale_pos_weight } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      "Execution time :  11:02:40.515554\n",
      "Job Ended at: 2020-11-04 21:58:53.327644\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "accuracy = 0\n",
    "best_model = None\n",
    "#for i in range(100):\n",
    "    #print (i)\n",
    "model = HyperoptEstimator(  classifier= any_classifier('cla'), \n",
    "                                preprocessing= any_preprocessing('pre'), \n",
    "                                algo=tpe.suggest, \n",
    "                                max_evals=50, \n",
    "                                trial_timeout=5000)\n",
    "    # perform the search\n",
    "model.fit(X_train, y_train)\n",
    "acc = model.score(X_test, y_test)\n",
    "    #if acc > accuracy:\n",
    "        #accuracy = acc\n",
    "        #best_model = model\n",
    "        #print (accuracy)\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))\n",
    "print (f\"Job Ended at: {fin_time}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.827\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy: %.3f\" % acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.827\n",
      "{'learner': XGBClassifier(base_score=0.5, booster='gbtree',\n",
      "              colsample_bylevel=0.9783479287357171, colsample_bynode=1,\n",
      "              colsample_bytree=0.8174662246984075, gamma=0.6596707384106874,\n",
      "              gpu_id=-1, importance_type='gain', interaction_constraints='',\n",
      "              learning_rate=0.025933472296031868, max_delta_step=0, max_depth=9,\n",
      "              min_child_weight=1, missing=nan, monotone_constraints='()',\n",
      "              n_estimators=3600, n_jobs=0, num_parallel_tree=1,\n",
      "              objective='multi:softprob', random_state=0,\n",
      "              reg_alpha=0.05907593535724862, reg_lambda=1.0398714478065672,\n",
      "              scale_pos_weight=1, seed=0, subsample=0.5081679750943192,\n",
      "              tree_method='exact', validate_parameters=1, verbosity=None), 'preprocs': (StandardScaler(with_mean=False),), 'ex_preprocs': ()}\n"
     ]
    }
   ],
   "source": [
    "#summarize the best model\n",
    "print(\"Accuracy: %.3f\" % acc)\n",
    "print(model.best_model())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[21:26:53] WARNING: C:\\Users\\Administrator\\workspace\\xgboost-win64_release_1.1.0\\src\\learner.cc:480: \n",
      "Parameters: { scale_pos_weight } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      "Execution time :  0:14:02.334549\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "model = XGBClassifier(base_score=0.5, booster='gbtree',\n",
    "              colsample_bylevel=0.50029110018247, colsample_bynode=1,\n",
    "              colsample_bytree=0.9371080486138555, gamma=0.00014999673883672553,\n",
    "              gpu_id=-1, importance_type='gain', interaction_constraints='',\n",
    "              learning_rate=0.15017743323882513, max_delta_step=0, max_depth=10,\n",
    "              min_child_weight=1, missing=None, monotone_constraints='()',\n",
    "              n_estimators=1000, n_jobs=0, num_parallel_tree=1,\n",
    "              objective='multi:softprob', random_state=4,\n",
    "              reg_alpha=6.402887715271464e-05, reg_lambda=1.0516334360090362,\n",
    "              scale_pos_weight=1, seed=4, subsample=0.9995865341563067,\n",
    "              tree_method='exact', validate_parameters=1, verbosity=None)\n",
    "\n",
    "model.fit(X_train, y_train,verbose=False,\n",
    "            early_stopping_rounds=10,\n",
    "            eval_set=[(X_valid,y_valid)])\n",
    "\n",
    "prediction = model.predict(X_test)\n",
    "#print(f\"F-measure of XGB: {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.74      0.57      0.64       587\n",
      "           1       0.77      0.84      0.80      4800\n",
      "           2       0.62      0.58      0.60      2361\n",
      "           3       0.78      0.57      0.66       787\n",
      "           4       0.97      0.98      0.97       845\n",
      "           5       0.95      0.95      0.95      4290\n",
      "           6       0.77      0.69      0.72       846\n",
      "           7       0.90      0.94      0.92      2490\n",
      "           8       0.88      0.88      0.88      1558\n",
      "\n",
      "    accuracy                           0.83     18564\n",
      "   macro avg       0.82      0.78      0.79     18564\n",
      "weighted avg       0.83      0.83      0.82     18564\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, prediction))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "#### Find Best Params With RandomizedSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "estimators = []\n",
    "estimators.append(('norm', MinMaxScaler()))\n",
    "estimators.append(('xgb', XGBClassifier(objective='binary:logistic')))\n",
    "pipeline = Pipeline(estimators)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "parameters = {\n",
    "                'xgb__n_estimators': [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000],\n",
    "                'xgb__max_depth': [3, 4, 5, 6, 7, 8, 9, 10],\n",
    "                'xgb__learning_rate': [0.0001, 0.001, 0.01, 0.02, 0.1, 0.2, 0.3], \n",
    "                'xgb__subsample' : [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 1.0],\n",
    "                'xgb__colsample_bytree' : [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 1.0],\n",
    "                'xgb__colsample_bylevel' : [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 1.0],\n",
    "                'xgb__gamma': [0],\n",
    "                'xgb__reg_lambda': [0, 1.0, 3.0, 5.0, 7.0, 10.0, 12.0],\n",
    "                'xgb__scale_pos_weight': [1, 2, 3, 4, 5, 6]\n",
    "              }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-05 13:03:58.276701\n",
      "[14:53:25] WARNING: C:\\Users\\Administrator\\workspace\\xgboost-win64_release_1.1.0\\src\\learner.cc:480: \n",
      "Parameters: { scale_pos_weight } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      "Execution time :  2:07:10.255337\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "kfold = KFold(n_splits=5, random_state=None, shuffle=True)\n",
    "xgb_grid = RandomizedSearchCV(pipeline,\n",
    "                        parameters,\n",
    "                        cv = kfold,\n",
    "                        n_jobs = -1,\n",
    "                        verbose=False)\n",
    "xgb_grid.fit(X_train, y_train)\n",
    "grid_predictions = xgb_grid.predict(X_test)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.69      0.56      0.62       399\n",
      "           1       0.74      0.84      0.79      3178\n",
      "           2       0.60      0.53      0.56      1561\n",
      "           3       0.72      0.50      0.59       538\n",
      "           4       0.98      0.98      0.98       565\n",
      "           5       0.94      0.94      0.94      2884\n",
      "           6       0.74      0.67      0.70       552\n",
      "           7       0.89      0.92      0.91      1674\n",
      "           8       0.86      0.85      0.86      1025\n",
      "\n",
      "    accuracy                           0.81     12376\n",
      "   macro avg       0.80      0.76      0.77     12376\n",
      "weighted avg       0.81      0.81      0.81     12376\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, grid_predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'xgb__subsample': 0.5,\n",
       " 'xgb__scale_pos_weight': 1,\n",
       " 'xgb__reg_lambda': 3.0,\n",
       " 'xgb__n_estimators': 800,\n",
       " 'xgb__max_depth': 3,\n",
       " 'xgb__learning_rate': 0.3,\n",
       " 'xgb__gamma': 0,\n",
       " 'xgb__colsample_bytree': 0.7,\n",
       " 'xgb__colsample_bylevel': 0.4}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_grid.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "print(classification_report(y_test, grid_predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "#### Find Best Algorithm with Best Params Using TPOT AutoML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "ss = StandardScaler()\n",
    "X_train_scaled = ss.fit_transform(X_train)\n",
    "X_test_scaled = ss.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-11 17:28:06.824817\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b1a9710724b444eb62fde6554179e9a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Optimization Progress', style=ProgressStyle(description_w…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "66.78 minutes have elapsed. TPOT will close down.\n",
      "TPOT closed during evaluation in one generation.\n",
      "WARNING: TPOT may not provide a good pipeline if TPOT is stopped/interrupted in a early generation.\n",
      "\n",
      "\n",
      "TPOT closed prematurely. Will use the current best pipeline.\n",
      "\n",
      "Best pipeline: SGDClassifier(input_matrix, alpha=0.01, eta0=0.01, fit_intercept=True, l1_ratio=1.0, learning_rate=constant, loss=modified_huber, penalty=elasticnet, power_t=50.0)\n",
      "Execution time :  1:06:51.567657\n",
      "Job Ended at: 2020-11-11 18:34:58.392474\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "\n",
    "# define model evaluation\n",
    "cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "# define search\n",
    "#model = TPOTClassifier(generations=5, population_size=50, cv=cv, scoring='accuracy', \n",
    "                       #verbosity=2, random_state=1, n_jobs=-1)\n",
    "tpot = TPOTClassifier(max_time_mins=120, cv=cv, scoring='accuracy', \n",
    "                       verbosity=2, random_state=1, n_jobs=-1)\n",
    "# perform the search\n",
    "tpot.fit(X_train_scaled, y_train)\n",
    "# export the best model\n",
    "tpot.export('tpot_Otto_best_model.py')\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))\n",
    "print (f\"Job Ended at: {fin_time}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('sgdclassifier',\n",
       "                 SGDClassifier(alpha=0.01, eta0=0.01, l1_ratio=1.0,\n",
       "                               learning_rate='constant', loss='modified_huber',\n",
       "                               penalty='elasticnet', power_t=50.0,\n",
       "                               random_state=1))])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tpot.fitted_pipeline_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7024886877828054"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tpot.score(X_test_scaled, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.pipeline import make_pipeline, make_union\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from tpot.builtins import StackingEstimator\n",
    "from tpot.export_utils import set_param_recursive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-05 13:01:40.215947\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.50      0.36      0.42       399\n",
      "           1       0.65      0.88      0.75      3178\n",
      "           2       0.55      0.28      0.37      1561\n",
      "           3       0.65      0.30      0.41       538\n",
      "           4       0.96      0.96      0.96       565\n",
      "           5       0.93      0.90      0.91      2884\n",
      "           6       0.63      0.59      0.61       552\n",
      "           7       0.85      0.87      0.86      1674\n",
      "           8       0.79      0.85      0.82      1025\n",
      "\n",
      "    accuracy                           0.76     12376\n",
      "   macro avg       0.72      0.67      0.68     12376\n",
      "weighted avg       0.75      0.76      0.74     12376\n",
      "\n",
      "Execution time :  0:00:01.606661\n",
      "Job Ended at: 2020-11-05 13:01:41.822608\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "# Average CV score on the training set was: 0.7517570704095573\n",
    "exported_pipeline = make_pipeline(\n",
    "    StackingEstimator(estimator=MultinomialNB(alpha=100.0, fit_prior=False)),\n",
    "    DecisionTreeClassifier(criterion=\"entropy\", max_depth=10, min_samples_leaf=12, min_samples_split=7)\n",
    ")\n",
    "# Fix random state for all the steps in exported pipeline\n",
    "set_param_recursive(exported_pipeline.steps, 'random_state', 1)\n",
    "\n",
    "exported_pipeline.fit(X_train, y_train)\n",
    "prediction = exported_pipeline.predict(X_test)\n",
    "print(classification_report(y_test, prediction))\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))\n",
    "print (f\"Job Ended at: {fin_time}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Define Base Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# evaluate a model\n",
    "def evaluate_model(X, y, model):\n",
    "    # define evaluation procedure\n",
    "    cv = RepeatedStratifiedKFold(n_splits=5, n_repeats=3, random_state=1)\n",
    "    # evaluate model  'accuracy'  'f1_micro'\n",
    "    scores = cross_val_score(model, X, y, scoring='accuracy', cv=cv, n_jobs=-1)\n",
    "    #print('F-measure: %.3f' % score)\n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "model = XGBClassifier(base_score=0.5, booster='gbtree',\n",
    "              colsample_bylevel=0.50029110018247, colsample_bynode=1,\n",
    "              colsample_bytree=0.9371080486138555, gamma=0.00014999673883672553,\n",
    "              gpu_id=-1, importance_type='gain', interaction_constraints='',\n",
    "              learning_rate=0.15017743323882513, max_delta_step=0, max_depth=10,\n",
    "              min_child_weight=1, missing=None, monotone_constraints='()',\n",
    "              n_estimators=1000, n_jobs=0, num_parallel_tree=1,\n",
    "              objective='multi:softprob', random_state=4,\n",
    "              reg_alpha=6.402887715271464e-05, reg_lambda=1.0516334360090362,\n",
    "              scale_pos_weight=1, seed=4, subsample=0.9995865341563067,\n",
    "              tree_method='exact', validate_parameters=1, verbosity=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Evaluate Base Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "model.fit(X_train, y_train)\n",
    "predictions = model.predict(X_test)\n",
    "print(classification_report(y_test, predictions))\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### ANOVA F-test Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# feature selection\n",
    "def select_Anova_features(X_train, y_train, X_test, k):\n",
    "    # configure to select all features\n",
    "    fs = SelectKBest(score_func=f_classif, k=k)\n",
    "    # learn relationship from training data\n",
    "    fs.fit(X_train, y_train)\n",
    "    # transform train input data\n",
    "    X_train_fs = fs.transform(X_train)\n",
    "    # transform test input data\n",
    "    X_test_fs = fs.transform(X_test)\n",
    "    return X_train_fs, X_test_fs, fs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# feature selection\n",
    "X_train_fs, X_test_fs, fs = select_Anova_features(X_train, y_train, X_test, 'all')\n",
    "scores = sorted(fs.scores_,reverse=True)\n",
    "# what are scores for the features\n",
    "for i in range(len(scores)):\n",
    "    print('Feature %d: %f' % (i, scores[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "# define the evaluation method\n",
    "cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "# define the pipeline to evaluate\n",
    "fs = SelectKBest(score_func=f_classif)\n",
    "pipeline = Pipeline(steps=[('anova',fs), ('xgb', model)])\n",
    "# define the grid\n",
    "grid = dict()\n",
    "grid['anova__k'] = [i+1 for i in range(X.shape[1])]\n",
    "# define the grid search\n",
    "# RandomizedSearchCV/GridSearchCV\n",
    "search = RandomizedSearchCV(pipeline, grid, scoring='accuracy', n_jobs=-1, cv=cv)\n",
    "# perform the search\n",
    "results = search.fit(X, y)\n",
    "# summarize best\n",
    "print('Best Mean Accuracy: %.3f' % results.best_score_)\n",
    "print('Best Config: %s' % results.best_params_)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "anova_best_features = results.best_params_['']\n",
    "anova_best_features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Evaluate ANOVA Feature Selection + Normalization + XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "# feature selection\n",
    "X_train_fs, X_test_fs, fs = select_Anova_features(X_train, y_train, X_test, anova_best_features)\n",
    "# fit the model\n",
    "estimators = []\n",
    "estimators.append(('norm', MinMaxScaler()))\n",
    "estimators.append(('xgb', model))\n",
    "pipeline = Pipeline(estimators)\n",
    "\n",
    "pipeline.fit(X_train_fs, y_train)\n",
    "predictions = pipeline.predict(X_test_fs)\n",
    "print(classification_report(y_test, predictions))\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Evaluate ANOVA Feature Selection + Outlier + Normalization + XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Evaluate ANOVA Feature Selection + Outlier + Normalization + PowerTransformer + XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Mutual Information Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# feature selection\n",
    "def select_Mutual_features(X_train, y_train, X_test, k):\n",
    "    # configure to select all features\n",
    "    fs = SelectKBest(score_func=mutual_info_classif, k=k)\n",
    "    # learn relationship from training data\n",
    "    fs.fit(X_train, y_train)\n",
    "    # transform train input data\n",
    "    X_train_fs = fs.transform(X_train)\n",
    "    # transform test input data\n",
    "    X_test_fs = fs.transform(X_test)\n",
    "    return X_train_fs, X_test_fs, fs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "# define the evaluation method\n",
    "cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "# define the pipeline to evaluate\n",
    "fs = SelectKBest(score_func=mutual_info_classif)\n",
    "pipeline = Pipeline(steps=[('mutual',fs), ('xgb', model)])\n",
    "# define the grid\n",
    "grid = dict()\n",
    "grid['mutual__k'] = [i+1 for i in range(X.shape[1])]\n",
    "# define the grid search\n",
    "# RandomizedSearchCV/GridSearchCV\n",
    "search = GridSearchCV(pipeline, grid, scoring='accuracy', n_jobs=-1, cv=cv)\n",
    "# perform the search\n",
    "results = search.fit(X, y)\n",
    "# summarize best\n",
    "print('Best Mean Accuracy: %.3f' % results.best_score_)\n",
    "print('Best Config: %s' % results.best_params_)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "mutual_best_features = results.best_params_['k']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "mutual_best_features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Evaluate Mutual Information Feature Selection + Normalization + XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "# feature selection\n",
    "X_train_fs, X_test_fs, fs = select_Mutual_features(X_train, y_train, X_test, mutual_best_features)\n",
    "# fit the model\n",
    "estimators = []\n",
    "estimators.append(('norm', MinMaxScaler()))\n",
    "estimators.append(('xgb', model))\n",
    "pipeline = Pipeline(estimators)\n",
    "\n",
    "pipeline.fit(X_train_fs, y_train)\n",
    "predictions = pipeline.predict(X_test_fs)\n",
    "print(classification_report(y_test, predictions))\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Evaluate Mutual Information Feature Selection + Outlier + Normalization + XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Evaluate Mutual Information Feature Selection + Outlier + Normalization + PowerTransformer + XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate RFE (Recursive Feature Elimination) Feature Selection + Normalization + XGB"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Find Optimal number of features (RFE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "rfc = RandomForestClassifier(random_state=101)\n",
    "rfecv = RFECV(estimator=rfc, step=1, cv=RepeatedStratifiedKFold(10), scoring='accuracy')\n",
    "#X_train_scaled, y_train\n",
    "rfecv.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "print('Optimal number of features: {}'.format(rfecv.n_features_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "print(np.where(rfecv.support_ == False)[0])\n",
    "X_best_features = X.drop(X.columns[np.where(rfecv.support_ == False)[0]], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "rfecv.estimator_.feature_importances_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "dset = pd.DataFrame()\n",
    "dset['attr'] = X.columns\n",
    "dset['importance'] = rfecv.estimator_.feature_importances_\n",
    "dset = dset.sort_values(by='importance', ascending=False)\n",
    "\n",
    "plt.figure(figsize=(16, 10))\n",
    "plt.barh(y=dset['attr'], width=dset['importance'], color='#1976D2')\n",
    "plt.title('RFECV - Feature importances', fontsize=20, fontweight='bold', pad=20)\n",
    "plt.xlabel('Importance', fontsize=14, labelpad=20)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# get a list of models to evaluate\n",
    "def get_models():\n",
    "    models = dict()\n",
    "    # lr\n",
    "    #rfe = RFECV(estimator=LogisticRegression())\n",
    "    #models['lr'] = Pipeline(steps=[('norm', MinMaxScaler()),('s',rfe),('m',model)])\n",
    "    # perceptron\n",
    "    #rfe = RFECV(estimator=Perceptron())\n",
    "    #model = DecisionTreeClassifier()\n",
    "    #models['per'] = Pipeline(steps=[('norm', MinMaxScaler()),('s',rfe),('m',model)])\n",
    "    \n",
    "    # abc\n",
    "    #rfe = RFECV(estimator=AdaBoostClassifier())\n",
    "    #models['abc'] = Pipeline(steps=[('norm', MinMaxScaler()),('s',rfe),('m',model)])\n",
    "\n",
    "    # rf\n",
    "    rfe = RFECV(estimator=RandomForestClassifier())\n",
    "    models['rf'] = Pipeline(steps=[('norm', MinMaxScaler()),('s',rfe),('m',model)])\n",
    "    # gbm\n",
    "    rfe = RFECV(estimator=GradientBoostingClassifier())\n",
    "    models['gbm'] = Pipeline(steps=[('norm', MinMaxScaler()),('s',rfe),('m',model)])\n",
    "    # xgb\n",
    "    rfe = RFECV(estimator=XGBClassifier())\n",
    "    models['xgb'] = Pipeline(steps=[('norm', MinMaxScaler()),('s',rfe),('m',model)])\n",
    "    # etc\n",
    "    rfe = RFECV(estimator=ExtraTreesClassifier())\n",
    "    models['etc'] = Pipeline(steps=[('norm', MinMaxScaler()),('s',rfe),('m',model)])\n",
    "    # ldr\n",
    "    rfe = RFECV(estimator=LinearDiscriminantAnalysis())\n",
    "    models['ldr'] = Pipeline(steps=[('norm', MinMaxScaler()),('s',rfe),('m',model)])\n",
    "    # cart\n",
    "    rfe = RFECV(estimator=DecisionTreeClassifier())\n",
    "    #models['cart'] = Pipeline(steps=[('norm', MinMaxScaler()),('s',rfe),('m',model)])\n",
    "    return models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-09 18:47:06.944414\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "# get the models to evaluate\n",
    "models = get_models()\n",
    "# evaluate the models and store results\n",
    "results, names = list(), list()\n",
    "for name, model in models.items():\n",
    "    scores = evaluate_model(X_train, y_train, model)\n",
    "    results.append(scores)\n",
    "    names.append(name)\n",
    "    print('>%s %.3f (%.3f)' % (name, np.mean(scores), np.std(scores)))\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-08 11:54:44.168640\n",
      "[12:39:40] WARNING: C:\\Users\\Administrator\\workspace\\xgboost-win64_release_1.1.0\\src\\learner.cc:480: \n",
      "Parameters: { scale_pos_weight } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.75      0.58      0.65       587\n",
      "           1       0.76      0.84      0.80      4800\n",
      "           2       0.62      0.56      0.59      2361\n",
      "           3       0.77      0.57      0.65       787\n",
      "           4       0.97      0.98      0.97       845\n",
      "           5       0.95      0.95      0.95      4290\n",
      "           6       0.77      0.69      0.73       846\n",
      "           7       0.90      0.94      0.92      2490\n",
      "           8       0.88      0.88      0.88      1558\n",
      "\n",
      "    accuracy                           0.83     18564\n",
      "   macro avg       0.82      0.78      0.79     18564\n",
      "weighted avg       0.82      0.83      0.82     18564\n",
      "\n",
      "Execution time :  0:57:59.920079\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "# create pipeline\n",
    "estimators = []\n",
    "estimators.append(('norm', MinMaxScaler()))\n",
    "estimators.append(('rfe', RFECV(estimator=RandomForestClassifier())))\n",
    "estimators.append(('xgb', model))\n",
    "pipeline = Pipeline(estimators)\n",
    "\n",
    "pipeline.fit(X_train, y_train)\n",
    "predictions = pipeline.predict(X_test)\n",
    "print(classification_report(y_test, predictions))\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Evaluate RFE (Recursive Feature Elimination) Feature Selection + Outlier + Normalization + XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Evaluate RFE (Recursive Feature Elimination) Feature Selection + Outlier + Normalization + PowerTransformer + XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate Feature Selection with Importance + Normalization + XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# feature selection\n",
    "def select_features(X_train, y_train, X_test, max_features):\n",
    "    # configure to select a subset of features\n",
    "    # Based on RFE Value\n",
    "    fs = SelectFromModel(GradientBoostingClassifier(n_estimators=1000), max_features=max_features)\n",
    "    # learn relationship from training data\n",
    "    fs.fit(X_train, y_train)\n",
    "    # transform train input data\n",
    "    X_train_fs = fs.transform(X_train)\n",
    "    # transform test input data\n",
    "    X_test_fs = fs.transform(X_test)\n",
    "    return X_train_fs, X_test_fs, fs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-12 10:58:49.773429\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "# create pipeline\n",
    "estimators = []\n",
    "estimators.append(('norm', MinMaxScaler()))\n",
    "estimators.append(('xgb', model))\n",
    "pipeline = Pipeline(estimators)\n",
    "\n",
    "# feature selection\n",
    "X_train_fs, X_test_fs, fs = select_features(X_train, y_train, X_test, 20)\n",
    "\n",
    "pipeline.fit(X_train_fs, y_train)\n",
    "predictions = pipeline.predict(X_test_fs)\n",
    "print(classification_report(y_test, predictions))\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Evaluate Feature Selection with Importance + Outlier + Normalization + XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Evaluate Feature Selection with Importance + Outlier + Normalization + PowerTransformer + XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#trans = QuantileTransformer(n_quantiles=100, output_distribution='normal')\n",
    "#output_distribution='uniform', 'normal'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Find Best LDA Component"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "LDA is a technique for multiclass classification that can be used to automatically perform dimensionality reduction. It is good practice to perhaps standardize the data prior to fitting an LDA model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "estimators = []\n",
    "estimators.append(('stnd', StandardScaler()))\n",
    "estimators.append(('lda', LinearDiscriminantAnalysis()))\n",
    "estimators.append(('xgb', model))\n",
    "pipeline = Pipeline(estimators)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# define the grid\n",
    "grid_param = dict()\n",
    "grid_param['lda__n_components'] = [i+1 for i in range(X.shape[1])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-09 01:37:25.298276\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "kfold = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "# RandomizedSearchCV/GridSearchCV\n",
    "grid_search = RandomizedSearchCV(pipeline,\n",
    "                        grid_param,\n",
    "                        cv = kfold,\n",
    "                        n_jobs = -1,\n",
    "                        verbose=False)\n",
    "grid_search.fit(X, y)\n",
    "# summarize best\n",
    "print('Best Mean Accuracy: %.3f' % grid_search.best_score_)\n",
    "print('Best Config: %s' % grid_search.best_params_)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda_n_components = grid_search.best_params_['lda__n_components']\n",
    "lda_n_components"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Evaluate Standarization + Linear Discriminant Analysis (LDA) Dimensionality Reduction + XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-12 08:34:06.090035\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "n_components cannot be larger than min(n_features, n_classes - 1).",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-24-305de58dd3f0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mpipeline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPipeline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimators\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0mpipeline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0mpredictions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpipeline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclassification_report\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredictions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[1;32m    328\u001b[0m         \"\"\"\n\u001b[1;32m    329\u001b[0m         \u001b[0mfit_params_steps\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_fit_params\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 330\u001b[0;31m         \u001b[0mXt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params_steps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    331\u001b[0m         with _print_elapsed_time('Pipeline',\n\u001b[1;32m    332\u001b[0m                                  self._log_message(len(self.steps) - 1)):\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36m_fit\u001b[0;34m(self, X, y, **fit_params_steps)\u001b[0m\n\u001b[1;32m    290\u001b[0m                 \u001b[0mcloned_transformer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    291\u001b[0m             \u001b[0;31m# Fit or load from cache the current transformer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 292\u001b[0;31m             X, fitted_transformer = fit_transform_one_cached(\n\u001b[0m\u001b[1;32m    293\u001b[0m                 \u001b[0mcloned_transformer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    294\u001b[0m                 \u001b[0mmessage_clsname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Pipeline'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/joblib/memory.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    350\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    351\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 352\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    353\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    354\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcall_and_shelve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36m_fit_transform_one\u001b[0;34m(transformer, X, y, weight, message_clsname, message, **fit_params)\u001b[0m\n\u001b[1;32m    738\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0m_print_elapsed_time\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage_clsname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    739\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'fit_transform'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 740\u001b[0;31m             \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    741\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    742\u001b[0m             \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/sklearn/base.py\u001b[0m in \u001b[0;36mfit_transform\u001b[0;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[1;32m    691\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    692\u001b[0m             \u001b[0;31m# fit method of arity 2 (supervised transformation)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 693\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    694\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    695\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/sklearn/discriminant_analysis.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y)\u001b[0m\n\u001b[1;32m    453\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    454\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_components\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mmax_components\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 455\u001b[0;31m                 raise ValueError(\n\u001b[0m\u001b[1;32m    456\u001b[0m                     \u001b[0;34m\"n_components cannot be larger than min(n_features, \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    457\u001b[0m                     \u001b[0;34m\"n_classes - 1).\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: n_components cannot be larger than min(n_features, n_classes - 1)."
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "estimators = []\n",
    "estimators.append(('stnd', StandardScaler()))\n",
    "estimators.append(('lda', LinearDiscriminantAnalysis(n_components=lda_n_components)))\n",
    "estimators.append(('xgb', model))\n",
    "pipeline = Pipeline(estimators)\n",
    "\n",
    "pipeline.fit(X_train, y_train)\n",
    "predictions = pipeline.predict(X_test)\n",
    "print(classification_report(y_test, predictions))\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Evaluate Normalization + PowerTransformer  + XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "#trans = QuantileTransformer(n_quantiles=100, output_distribution='normal')\n",
    "#output_distribution='uniform', 'normal'\n",
    "estimators = []\n",
    "estimators.append(('norm', MinMaxScaler()))\n",
    "estimators.append(('power', PowerTransformer()))\n",
    "estimators.append(('xgb', model))\n",
    "pipeline = Pipeline(estimators)\n",
    "\n",
    "pipeline.fit(X_train, y_train)\n",
    "predictions = pipeline.predict(X_test)\n",
    "print(classification_report(y_test, predictions))\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Evaluate Outlier + Normalization + XGB"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "identify outliers in the training dataset\n",
    "* iso = IsolationForest(contamination=0.1)\n",
    "* yhat = iso.fit_predict(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "select all rows that are not outliers\n",
    "* mask = yhat != -1\n",
    "* X_train, y_train = X_train[mask, :], y_train[mask]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "summarize the shape of the updated training dataset\n",
    "* print(X_train.shape, y_train.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "* ee = EllipticEnvelope(contamination=0.01)\n",
    "* lof = LocalOutlierFactor()\n",
    "* ee = OneClassSVM(nu=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "estimators = []\n",
    "estimators.append(('out', LocalOutlierFactor()))\n",
    "estimators.append(('norm', MinMaxScaler()))\n",
    "estimators.append(('xgb', model))\n",
    "pipeline = Pipeline(estimators)\n",
    "\n",
    "pipeline.fit(X_train, y_train)\n",
    "predictions = pipeline.predict(X_test)\n",
    "print(classification_report(y_test, predictions))\n",
    "\n",
    "# identify outliers in the training dataset\n",
    "#lof = LocalOutlierFactor()\n",
    "#yhat = lof.fit_predict(transformed)\n",
    "# select all rows that are not outliers\n",
    "#mask = yhat != -1\n",
    "#X_transformed, y_transformed = X_transformed[mask, :], y[mask]\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Find Best SVD Component"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Singular Value Decomposition, or SVD, might be the most popular technique for dimensionality reduction when data is sparse. It can also be a good idea to normalize data prior to performing the SVD transform if the input variables have differing units or scales."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "estimators = []\n",
    "estimators.append(('norm', MinMaxScaler()))\n",
    "estimators.append(('svd', TruncatedSVD()))\n",
    "estimators.append(('xgb', model))\n",
    "pipeline = Pipeline(estimators)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# define the grid\n",
    "grid_param = dict()\n",
    "grid_param['svd__n_components'] = [i+1 for i in range(X.shape[1])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-06 16:28:11.969119\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "kfold = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "# RandomizedSearchCV/GridSearchCV\n",
    "grid_search = RandomizedSearchCV(pipeline,\n",
    "                        grid_param,\n",
    "                        cv = kfold,\n",
    "                        n_jobs = -1,\n",
    "                        verbose=False)\n",
    "grid_search.fit(X, y)\n",
    "# summarize best\n",
    "print('Best Mean Accuracy: %.3f' % grid_search.best_score_)\n",
    "print('Best Config: %s' % grid_search.best_params_)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "svd_n_components = grid_search.best_params_['n_components']\n",
    "print(svd_n_components)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Evaluate Normalization + SVD + XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-12 08:23:20.991809\n",
      "[08:23:21] WARNING: /workspace/src/learner.cc:480: \n",
      "Parameters: { scale_pos_weight } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.68      0.49      0.57       587\n",
      "           1       0.70      0.85      0.77      4800\n",
      "           2       0.57      0.45      0.50      2361\n",
      "           3       0.74      0.40      0.52       787\n",
      "           4       0.96      0.95      0.96       845\n",
      "           5       0.93      0.94      0.94      4290\n",
      "           6       0.76      0.60      0.67       846\n",
      "           7       0.86      0.92      0.89      2490\n",
      "           8       0.86      0.86      0.86      1558\n",
      "\n",
      "    accuracy                           0.79     18564\n",
      "   macro avg       0.78      0.72      0.74     18564\n",
      "weighted avg       0.79      0.79      0.78     18564\n",
      "\n",
      "Execution time :  0:10:45.082706\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "svd_n_components = 46\n",
    "estimators = []\n",
    "estimators.append(('norm', MinMaxScaler()))\n",
    "estimators.append(('svd', TruncatedSVD(n_components=svd_n_components)))\n",
    "estimators.append(('xgb', model))\n",
    "pipeline = Pipeline(estimators)\n",
    "\n",
    "pipeline.fit(X_train, y_train)\n",
    "predictions = pipeline.predict(X_test)\n",
    "print(classification_report(y_test, predictions))\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Find Best PCA Component"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "It can also be a good idea to normalize data prior to performing the PCA transform if the input variables have differing units or scales."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "estimators = []\n",
    "estimators.append(('norm', MinMaxScaler()))\n",
    "estimators.append(('pca', PCA()))\n",
    "estimators.append(('xgb', model))\n",
    "pipeline = Pipeline(estimators)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# define the grid\n",
    "grid_param = dict()\n",
    "grid_param['pca__n_components'] = [i+1 for i in range(X.shape[1])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-10 00:05:08.419224\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "kfold = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "# RandomizedSearchCV/GridSearchCV\n",
    "grid_search = RandomizedSearchCV(pipeline,\n",
    "                        grid_param,\n",
    "                        cv = kfold,\n",
    "                        n_jobs = -1,\n",
    "                        verbose=False)\n",
    "grid_search.fit(X, y)\n",
    "# summarize best\n",
    "print('Best Mean Accuracy: %.3f' % grid_search.best_score_)\n",
    "print('Best Config: %s' % grid_search.best_params_)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "pca_n_components = grid_search.best_params_['n_components']\n",
    "pca_n_componentsconda "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Evaluate Normalization + PCA + XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "pca_n_components = 46\n",
    "estimators = []\n",
    "estimators.append(('norm', MinMaxScaler()))\n",
    "estimators.append(('pca', PCA(n_components=pca_n_components)))\n",
    "estimators.append(('xgb', model))\n",
    "pipeline = Pipeline(estimators)\n",
    "\n",
    "pipeline.fit(X_train, y_train)\n",
    "predictions = pipeline.predict(X_test)\n",
    "print(classification_report(y_test, predictions))\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Evaluate Normalization + PCA + Outlier + XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "estimators = []\n",
    "estimators.append(('norm', MinMaxScaler()))\n",
    "estimators.append(('pca', PCA(n_components=best_n_components)))\n",
    "estimators.append(('out', LocalOutlierFactor()))\n",
    "estimators.append(('xgb', model))\n",
    "pipeline = Pipeline(estimators)\n",
    "\n",
    "pipeline.fit(X_train, y_train)\n",
    "predictions = pipeline.predict(X_test)\n",
    "print(classification_report(y_test, predictions))\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Evaluate Normalization + PowerTransformer + PCA + XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-12 03:09:19.675804\n",
      "[03:09:24] WARNING: /workspace/src/learner.cc:480: \n",
      "Parameters: { scale_pos_weight } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.69      0.35      0.46       587\n",
      "           1       0.71      0.84      0.77      4800\n",
      "           2       0.56      0.48      0.51      2361\n",
      "           3       0.69      0.36      0.47       787\n",
      "           4       0.94      0.95      0.95       845\n",
      "           5       0.91      0.93      0.92      4290\n",
      "           6       0.71      0.56      0.62       846\n",
      "           7       0.84      0.89      0.86      2490\n",
      "           8       0.81      0.82      0.82      1558\n",
      "\n",
      "    accuracy                           0.78     18564\n",
      "   macro avg       0.76      0.69      0.71     18564\n",
      "weighted avg       0.77      0.78      0.77     18564\n",
      "\n",
      "Execution time :  0:35:12.715099\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "#trans = QuantileTransformer(n_quantiles=100, output_distribution='normal')\n",
    "#output_distribution='uniform', 'normal'\n",
    "best_n_components = 46\n",
    "estimators = []\n",
    "estimators.append(('norm', MinMaxScaler()))\n",
    "estimators.append(('power', PowerTransformer()))\n",
    "estimators.append(('pca', PCA(n_components=best_n_components)))\n",
    "estimators.append(('xgb', model))\n",
    "pipeline = Pipeline(estimators)\n",
    "\n",
    "pipeline.fit(X_train, y_train)\n",
    "predictions = pipeline.predict(X_test)\n",
    "print(classification_report(y_test, predictions))\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Evaluate Normalization + PowerTransformer + PCA + Outlier  + XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "norm_scaler = MinMaxScaler()\n",
    "#scaler = StandardScaler()\n",
    "scaler.fit(X_train)\n",
    "X_train_norm = scaler.transform(X_train)\n",
    "X_test_norm = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#output_distribution='uniform', 'normal'\n",
    "transformer = PowerTransformer()\n",
    "#transformer = QuantileTransformer(n_quantiles=100, output_distribution='normal')\n",
    "transformer.fit(X_train)\n",
    "X_train_transformed = transformer.transform(X_train_norm)\n",
    "X_test_transformed = transformer.transform(X_test_norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# define transform https://machinelearningmastery.com/dimensionality-reduction-algorithms-with-python/\n",
    "best_n_components = 46\n",
    "# dimention_reduction = TruncatedSVD (n_components=best_n_components)\n",
    "dimention_reduction = PCA(n_components=best_n_components)\n",
    "# prepare transform on dataset\n",
    "dimention_reduction.fit(X_train_transformed)\n",
    "# apply transform to dataset\n",
    "X_train_transformed = dimention_reduction.transform(X_train_transformed)\n",
    "X_test_transformed = dimention_reduction.transform(X_test_transformed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# identify outliers in the training dataset\n",
    "#outlier = IsolationForest(contamination=0.1)\n",
    "#outlier = EllipticEnvelope(contamination=0.01)\n",
    "#outlier = OneClassSVM(nu=0.01)\n",
    "outlier = LocalOutlierFactor()\n",
    "yhat = outlier.fit_predict(X_train_transformed)\n",
    "# select all rows that are not outliers\n",
    "mask = yhat != -1\n",
    "X_train_transformed, y_train_transformed = X_train_transformed[mask, :], y_train[mask]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-13 17:33:06.843826\n",
      "[17:33:06] WARNING: /workspace/src/learner.cc:480: \n",
      "Parameters: { scale_pos_weight } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.62      0.36      0.46       587\n",
      "           1       0.68      0.82      0.74      4800\n",
      "           2       0.53      0.42      0.47      2361\n",
      "           3       0.67      0.32      0.44       787\n",
      "           4       0.94      0.90      0.92       845\n",
      "           5       0.88      0.92      0.90      4290\n",
      "           6       0.72      0.57      0.63       846\n",
      "           7       0.83      0.88      0.85      2490\n",
      "           8       0.78      0.80      0.79      1558\n",
      "\n",
      "    accuracy                           0.76     18564\n",
      "   macro avg       0.74      0.67      0.69     18564\n",
      "weighted avg       0.75      0.76      0.75     18564\n",
      "\n",
      "Execution time :  0:09:25.003869\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "model.fit(X_train_transformed, y_train_transformed)\n",
    "predictions = model.predict(X_test_transformed)\n",
    "print(classification_report(y_test, predictions))\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Norm -> Power -> PCA -> Local Outlier\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.62      0.36      0.46       587\n",
      "           1       0.68      0.82      0.74      4800\n",
      "           2       0.53      0.42      0.47      2361\n",
      "           3       0.67      0.32      0.44       787\n",
      "           4       0.94      0.90      0.92       845\n",
      "           5       0.88      0.92      0.90      4290\n",
      "           6       0.72      0.57      0.63       846\n",
      "           7       0.83      0.88      0.85      2490\n",
      "           8       0.78      0.80      0.79      1558\n",
      "\n",
      "    accuracy                           0.76     18564\n",
      "   macro avg       0.74      0.67      0.69     18564\n",
      "weighted avg       0.75      0.76      0.75     18564\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print ('Norm -> Power -> PCA -> Local Outlier')\n",
    "print(classification_report(y_test, predictions))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Normalize Pipelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# get modeling pipelines to evaluate\n",
    "def get_norm_pipelines(model, best_n_components):\n",
    "    pipelines = list()\n",
    "    # normalize\n",
    "    p = Pipeline([('n',MinMaxScaler()), ('m',model)])\n",
    "    pipelines.append(('norm', p))\n",
    "    # normalize and RFE\n",
    "    p = Pipeline([('n',MinMaxScaler()), ('rfe', RFECV(estimator=RandomForestClassifier())), ('m',model)])\n",
    "    pipelines.append(('norm_rfe', p))\n",
    "    # normalize and power\n",
    "    p = Pipeline([('n',MinMaxScaler()), ('p', PowerTransformer()), ('m',model)])\n",
    "    pipelines.append(('norm_power', p))\n",
    "    # normalize and power and RFE\n",
    "    p = Pipeline([('n',MinMaxScaler()), ('p', PowerTransformer()), ('rfe', RFECV(estimator=RandomForestClassifier())),\n",
    "                  ('m',model)])\n",
    "    pipelines.append(('norm_power_RFE', p))\n",
    "    # normalize and power and PCA\n",
    "    p = Pipeline([('s',MinMaxScaler()), ('p', PowerTransformer()), \n",
    "                  ('pca', PCA(n_components=best_n_components)),('m',model)])\n",
    "    pipelines.append(('norm_power_pca', p))\n",
    "    # normalize and power and SVD\n",
    "    p = Pipeline([('n',MinMaxScaler()), ('p', PowerTransformer()), \n",
    "                  ('svd', TruncatedSVD(n_components=best_n_components)),('m',model)])\n",
    "    pipelines.append(('norm_power_svd', p))\n",
    "    # normalize and power and LDA\n",
    "    p = Pipeline([('n',MinMaxScaler()), ('p', PowerTransformer()), \n",
    "                  ('lda', LinearDiscriminantAnalysis(n_components=best_n_components)),('m',model)])\n",
    "    pipelines.append(('norm_power_lda', p))\n",
    "    # normalize and Quantile\n",
    "    p = Pipeline([('n',MinMaxScaler()), \n",
    "                  ('q', QuantileTransformer(n_quantiles=100, output_distribution='normal')), ('m',model)])\n",
    "    pipelines.append(('norm_quantile', p))\n",
    "    # normalize and Quantile and RFE\n",
    "    p = Pipeline([('n',MinMaxScaler()), \n",
    "                  ('q', QuantileTransformer(n_quantiles=100, output_distribution='normal')), \n",
    "                  ('rfe', RFECV(estimator=RandomForestClassifier())),('m',model)])\n",
    "    pipelines.append(('norm_quantile_rfe', p))\n",
    "    # normalize and Quantile and PCA\n",
    "    p = Pipeline([('n',MinMaxScaler()), \n",
    "                  ('q', QuantileTransformer(n_quantiles=100, output_distribution='normal')), \n",
    "                  ('pca', PCA(n_components=best_n_components)),('m',model)])\n",
    "    pipelines.append(('norm_quantile_pca', p))\n",
    "    # normalize and Quantile and SVD\n",
    "    p = Pipeline([('n',MinMaxScaler()), \n",
    "                  ('q', QuantileTransformer(n_quantiles=100, output_distribution='normal')), \n",
    "                  ('svd', TruncatedSVD(n_components=best_n_components)),('m',model)])\n",
    "    pipelines.append(('norm_quantile_svd', p))\n",
    "    # normalize and Quantile and LDA\n",
    "    p = Pipeline([('n',MinMaxScaler()), \n",
    "                  ('q', QuantileTransformer(n_quantiles=100, output_distribution='normal')), \n",
    "                  ('lda', LinearDiscriminantAnalysis(n_components=best_n_components)),('m',model)])\n",
    "    pipelines.append(('norm_quantile_svd', p))\n",
    "    return pipelines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Standardize Pipelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# get modeling pipelines to evaluate\n",
    "def get_standardize_pipelines(model, best_n_components):\n",
    "    pipelines = list()\n",
    "    # standardize\n",
    "    p = Pipeline([('s',StandardScaler()), ('m',model)])\n",
    "    pipelines.append(('std', p))\n",
    "    # standardize and RFE\n",
    "    p = Pipeline([('s',StandardScaler()), ('rfe', RFECV(estimator=RandomForestClassifier())), ('m',model)])\n",
    "    pipelines.append(('std_rfe', p))\n",
    "    # standardize and power\n",
    "    p = Pipeline([('s',StandardScaler()), ('p', PowerTransformer()), ('m',model)])\n",
    "    pipelines.append(('std_power', p))\n",
    "    # standardize and power and RFE\n",
    "    p = Pipeline([('s',StandardScaler()), ('p', PowerTransformer()), \n",
    "                  ('rfe', RFECV(estimator=RandomForestClassifier())),('m',model)])\n",
    "    pipelines.append(('std_power_rfe', p))\n",
    "    # standardize and power and PCA\n",
    "    p = Pipeline([('s',StandardScaler()), ('p', PowerTransformer()), \n",
    "                  ('pca', PCA(n_components=best_n_components)),('m',model)])\n",
    "    pipelines.append(('std_power_pca', p))\n",
    "    # standardize and power and SVD\n",
    "    p = Pipeline([('s',StandardScaler()), ('p', PowerTransformer()), \n",
    "                  ('svd', TruncatedSVD(n_components=best_n_components)),('m',model)])\n",
    "    pipelines.append(('std_power_svd', p))\n",
    "    # standardize and power and LDA\n",
    "    p = Pipeline([('s',StandardScaler()), ('p', PowerTransformer()), \n",
    "                  ('lda', LinearDiscriminantAnalysis(n_components=best_n_components)),('m',model)])\n",
    "    pipelines.append(('std_power_lda', p))\n",
    "    # standardize and Quantile\n",
    "    p = Pipeline([('s',StandardScaler()), \n",
    "                  ('q', QuantileTransformer(n_quantiles=100, output_distribution='normal')), ('m',model)])\n",
    "    pipelines.append(('std_quantile', p))\n",
    "    # standardize and Quantile and RFE\n",
    "    p = Pipeline([('s',StandardScaler()), \n",
    "                  ('q', QuantileTransformer(n_quantiles=100, output_distribution='normal')), \n",
    "                  ('rfe', RFECV(estimator=RandomForestClassifier())),('m',model)])\n",
    "    pipelines.append(('std_quantile_rfe', p))\n",
    "    # standardize and Quantile and PCA\n",
    "    p = Pipeline([('s',StandardScaler()), \n",
    "                  ('q', QuantileTransformer(n_quantiles=100, output_distribution='normal')), \n",
    "                  ('pca', PCA(n_components=best_n_components)),('m',model)])\n",
    "    pipelines.append(('std_quantile_pca', p))\n",
    "    # standardize and Quantile and SVD\n",
    "    p = Pipeline([('s',StandardScaler()), \n",
    "                  ('q', QuantileTransformer(n_quantiles=100, output_distribution='normal')), \n",
    "                  ('svd', TruncatedSVD(n_components=best_n_components)),('m',model)])\n",
    "    pipelines.append(('std_quantile_svd', p))\n",
    "    # standardize and Quantile and LDA\n",
    "    p = Pipeline([('s',StandardScaler()), \n",
    "                  ('q', QuantileTransformer(n_quantiles=100, output_distribution='normal')), \n",
    "                  ('lda', LinearDiscriminantAnalysis(n_components=best_n_components)),('m',model)])\n",
    "    pipelines.append(('std_quantile_lda', p))\n",
    "    return pipelines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Robust Pipelines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "https://machinelearningmastery.com/robust-scaler-transforms-for-machine-learning/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#Robust scaling techniques that use percentiles can be used to scale numerical input variables that contain outliers.\n",
    "#trans = RobustScaler(with_centering=False, with_scaling=True)\n",
    "# get modeling pipelines to evaluate\n",
    "def get_robust_pipelines(model, best_n_components):\n",
    "    pipelines = list()\n",
    "    # Robust\n",
    "    p = Pipeline([('r',RobustScaler(with_centering=False, with_scaling=True)), ('m',model)])\n",
    "    pipelines.append(('robust', p))\n",
    "    # Robust and RFE\n",
    "    p = Pipeline([('r',RobustScaler(with_centering=False, with_scaling=True)), \n",
    "                  ('rfe', RFECV(estimator=RandomForestClassifier())), ('m',model)])\n",
    "    pipelines.append(('robust_rfe', p))\n",
    "    # Robust and power\n",
    "    p = Pipeline([('r',RobustScaler(with_centering=False, with_scaling=True)), \n",
    "                  ('p', PowerTransformer()), ('m',model)])\n",
    "    pipelines.append(('robust_power', p))\n",
    "    # Robust and power and RFE\n",
    "    p = Pipeline([('r',RobustScaler(with_centering=False, with_scaling=True)), \n",
    "                  ('p', PowerTransformer()), ('rfe', RFECV(estimator=RandomForestClassifier())),\n",
    "                  ('m',model)])\n",
    "    pipelines.append(('robust_power_rfe', p))\n",
    "    # Robust and power and PCA\n",
    "    p = Pipeline([('r',RobustScaler(with_centering=False, with_scaling=True)), \n",
    "                  ('p', PowerTransformer()), \n",
    "                  ('pca', PCA(n_components=best_n_components)),('m',model)])\n",
    "    pipelines.append(('robust_power_pca', p))\n",
    "    # Robust and power and SVD\n",
    "    p = Pipeline([('r', RobustScaler(with_centering=False, with_scaling=True)), ('p', PowerTransformer()), \n",
    "                  ('svd', TruncatedSVD(n_components=best_n_components)),('m',model)])\n",
    "    pipelines.append(('robust_power_svd', p))\n",
    "    # Robust and power and LDA\n",
    "    p = Pipeline([('r', RobustScaler(with_centering=False, with_scaling=True)), ('p', PowerTransformer()), \n",
    "                  ('lda', LinearDiscriminantAnalysis(n_components=best_n_components)),('m',model)])\n",
    "    pipelines.append(('robust_power_lda', p))\n",
    "    # Robust and Quantile\n",
    "    p = Pipeline([('r', RobustScaler(with_centering=False, with_scaling=True)), \n",
    "                  ('q', QuantileTransformer(n_quantiles=100, output_distribution='normal')), ('m',model)])\n",
    "    pipelines.append(('robust_quantile', p))\n",
    "    # Robust and Quantile and RFE\n",
    "    p = Pipeline([('r', RobustScaler(with_centering=False, with_scaling=True)), \n",
    "                  ('q', QuantileTransformer(n_quantiles=100, output_distribution='normal')), \n",
    "                  ('rfe', RFECV(estimator=RandomForestClassifier())),('m',model)])\n",
    "    pipelines.append(('robust_quantile_rfe', p))\n",
    "    # Robust and Quantile and PCA\n",
    "    p = Pipeline([('r', RobustScaler(with_centering=False, with_scaling=True)), \n",
    "                  ('q', QuantileTransformer(n_quantiles=100, output_distribution='normal')), \n",
    "                  ('pca', PCA(n_components=best_n_components)),('m',model)])\n",
    "    pipelines.append(('robust_quantile_pca', p))\n",
    "    # Robust and Quantile and SVD\n",
    "    p = Pipeline([('r', RobustScaler(with_centering=False, with_scaling=True)), \n",
    "                  ('q', QuantileTransformer(n_quantiles=100, output_distribution='normal')), \n",
    "                  ('svd', TruncatedSVD(n_components=best_n_components)),('m',model)])\n",
    "    pipelines.append(('robust_quantile_svd', p))\n",
    "    # Robust and Quantile and LDA\n",
    "    p = Pipeline([('r', RobustScaler(with_centering=False, with_scaling=True)), \n",
    "                  ('q', QuantileTransformer(n_quantiles=100, output_distribution='normal')), \n",
    "                  ('lda', LinearDiscriminantAnalysis(n_components=best_n_components)),('m',model)])\n",
    "    pipelines.append(('robust_quantile_lda', p))\n",
    "    return pipelines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Robust Pipeline Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Started at: 2020-11-13 19:24:27.227942\n",
      "[19:24:27] WARNING: /workspace/src/learner.cc:480: \n",
      "Parameters: { scale_pos_weight } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      ">robust: 0.828\n",
      "[20:21:38] WARNING: /workspace/src/learner.cc:480: \n",
      "Parameters: { scale_pos_weight } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      ">robust_rfe: 0.826\n",
      "[20:32:01] WARNING: /workspace/src/learner.cc:480: \n",
      "Parameters: { scale_pos_weight } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      ">robust_power: 0.829\n",
      "[20:44:47] WARNING: /workspace/src/learner.cc:480: \n",
      "Parameters: { scale_pos_weight } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      ">robust_power_pca: 0.769\n",
      "[20:59:31] WARNING: /workspace/src/learner.cc:480: \n",
      "Parameters: { scale_pos_weight } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      ">robust_power_svd: 0.768\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "n_components cannot be larger than min(n_features, n_classes - 1).",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-23-56bce0e1f16a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;31m# evaluate each pipeline\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpipeline\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpipelines\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     \u001b[0mpipeline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m     \u001b[0mpredictions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpipeline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'>%s: %.3f'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccuracy_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredictions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[1;32m    328\u001b[0m         \"\"\"\n\u001b[1;32m    329\u001b[0m         \u001b[0mfit_params_steps\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_fit_params\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 330\u001b[0;31m         \u001b[0mXt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params_steps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    331\u001b[0m         with _print_elapsed_time('Pipeline',\n\u001b[1;32m    332\u001b[0m                                  self._log_message(len(self.steps) - 1)):\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36m_fit\u001b[0;34m(self, X, y, **fit_params_steps)\u001b[0m\n\u001b[1;32m    290\u001b[0m                 \u001b[0mcloned_transformer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    291\u001b[0m             \u001b[0;31m# Fit or load from cache the current transformer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 292\u001b[0;31m             X, fitted_transformer = fit_transform_one_cached(\n\u001b[0m\u001b[1;32m    293\u001b[0m                 \u001b[0mcloned_transformer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    294\u001b[0m                 \u001b[0mmessage_clsname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Pipeline'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/joblib/memory.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    350\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    351\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 352\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    353\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    354\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcall_and_shelve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36m_fit_transform_one\u001b[0;34m(transformer, X, y, weight, message_clsname, message, **fit_params)\u001b[0m\n\u001b[1;32m    738\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0m_print_elapsed_time\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage_clsname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    739\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'fit_transform'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 740\u001b[0;31m             \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    741\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    742\u001b[0m             \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtransformer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/sklearn/base.py\u001b[0m in \u001b[0;36mfit_transform\u001b[0;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[1;32m    691\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    692\u001b[0m             \u001b[0;31m# fit method of arity 2 (supervised transformation)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 693\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    694\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    695\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.8/site-packages/sklearn/discriminant_analysis.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y)\u001b[0m\n\u001b[1;32m    453\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    454\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_components\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mmax_components\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 455\u001b[0;31m                 raise ValueError(\n\u001b[0m\u001b[1;32m    456\u001b[0m                     \u001b[0;34m\"n_components cannot be larger than min(n_features, \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    457\u001b[0m                     \u001b[0;34m\"n_classes - 1).\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: n_components cannot be larger than min(n_features, n_classes - 1)."
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "best_n_components = 46\n",
    "# get the modeling pipelines\n",
    "pipelines = get_robust_pipelines(model, best_n_components)\n",
    "# evaluate each pipeline\n",
    "for name, pipeline in pipelines:\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    predictions = pipeline.predict(X_test)\n",
    "    print('> %s: %.3f' % (name, accuracy_score(y_test, predictions)))\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "pipeline = Pipeline([('r',RobustScaler(with_centering=False, with_scaling=True)), \n",
    "                  ('p', PowerTransformer()), ('rfe', RFECV(estimator=RandomForestClassifier())),\n",
    "                  ('m',model)])\n",
    "pipeline.fit(X_train, y_train)\n",
    "predictions = pipeline.predict(X_test)\n",
    "print('> %s: %.3f' % (name, accuracy_score(y_test, predictions)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "pipeline = Pipeline([('r', RobustScaler(with_centering=False, with_scaling=True)), \n",
    "                  ('q', QuantileTransformer(n_quantiles=100, output_distribution='normal')), \n",
    "                  ('rfe', RFECV(estimator=RandomForestClassifier())),('m',model)])\n",
    "pipeline.fit(X_train, y_train)\n",
    "predictions = pipeline.predict(X_test)\n",
    "print('> %s: %.3f' % (name, accuracy_score(y_test, predictions)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Standardize Pipeline Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "best_n_components = 46\n",
    "# get the modeling pipelines\n",
    "pipelines = get_standardize_pipelines(model, best_n_components)\n",
    "# evaluate each pipeline\n",
    "for name, pipeline in pipelines:\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    predictions = pipeline.predict(X_test)\n",
    "    print('>%s: %.3f' % (name, accuracy_score(y_test, predictions)))\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Normalize Pipeline Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "print (f\"Job Started at: {init_time}\")\n",
    "best_n_components = 46\n",
    "# get the modeling pipelines\n",
    "pipelines = get_norm_pipelines(model, best_n_components)\n",
    "# evaluate each pipeline\n",
    "for name, pipeline in pipelines:\n",
    "    pipeline.fit(X_train, y_train)\n",
    "    predictions = pipeline.predict(X_test)\n",
    "    print('>%s: %.3f' % (name, accuracy_score(y_test, predictions)))\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}