03_Loan_Defaulter_Detection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "from numpy import mean\n",
    "from numpy import std\n",
    "import numpy as np\n",
    "import pandas  as pd\n",
    "from matplotlib import pyplot\n",
    "import ppscore as pps\n",
    "import operator\n",
    "from scipy.stats import shapiro, normaltest\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.preprocessing import OrdinalEncoder\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold, StratifiedKFold, RepeatedStratifiedKFold\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.metrics import fbeta_score\n",
    "from sklearn.metrics import f1_score, accuracy_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.kernel_approximation import RBFSampler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report,confusion_matrix,plot_confusion_matrix\n",
    "#from sklearn.ensemble import BalancedBaggingClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from imblearn.combine import SMOTEENN\n",
    "from imblearn.combine import SMOTETomek\n",
    "from imblearn.under_sampling import TomekLinks\n",
    "from imblearn.under_sampling import EditedNearestNeighbours\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from imblearn.over_sampling import ADASYN  #Its a improved version of Smote\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "from sklearn.preprocessing import StandardScaler, RobustScaler\n",
    "from sklearn.preprocessing import PowerTransformer\n",
    "from sklearn.preprocessing import QuantileTransformer\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.decomposition import PCA, TruncatedSVD\n",
    "from sklearn.neighbors import LocalOutlierFactor\n",
    "from sklearn.feature_selection import RFE, SelectFromModel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Load the Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#print(\"F1 micro: %1.4f\\n\" % f1_score(y_test, y_predicted, average='micro'))\n",
    "#print(\"F1 macro: %1.4f\\n\" % f1_score(y_test, y_predicted, average='macro'))\n",
    "#print(\"F1 weighted: %1.4f\\n\" % f1_score(y_test, y_predicted, average='weighted'))\n",
    "#print(\"Accuracy: %1.4f\" % (accuracy_score(y_test, y_predicted)))\n",
    "#scores = cross_val_score(clf, X, y, cv=5, scoring='f1_macro')\n",
    "#knn_scores = cross_val_score(knn, X, y, cv=10)\n",
    "#print(\"Accuracy: %0.2f (+/- %0.2f)\" % (knn_scores.mean(), knn_scores.std() * 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('Data/classification_testData_shape(395219,82).csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(395219, 83)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'loan_amnt', 'term', 'int_rate', 'installment',\n",
       "       'annual_inc', 'loan_status', 'dti', 'open_acc', 'pub_rec', 'revol_bal',\n",
       "       'revol_util', 'total_acc', 'mort_acc', 'pub_rec_bankruptcies', 'A2',\n",
       "       'A3', 'A4', 'A5', 'B1', 'B2', 'B3', 'B4', 'B5', 'C1', 'C2', 'C3', 'C4',\n",
       "       'C5', 'D1', 'D2', 'D3', 'D4', 'D5', 'E1', 'E2', 'E3', 'E4', 'E5', 'F1',\n",
       "       'F2', 'F3', 'F4', 'F5', 'G1', 'G2', 'G3', 'G4', 'G5', 'OTHER', 'OWN',\n",
       "       'RENT', 'verification_status_Source Verified',\n",
       "       'verification_status_Verified', 'application_type_INDIVIDUAL',\n",
       "       'application_type_JOINT', 'initial_list_status_w',\n",
       "       'purpose_credit_card', 'purpose_debt_consolidation',\n",
       "       'purpose_educational', 'purpose_home_improvement', 'purpose_house',\n",
       "       'purpose_major_purchase', 'purpose_medical', 'purpose_moving',\n",
       "       'purpose_other', 'purpose_renewable_energy', 'purpose_small_business',\n",
       "       'purpose_vacation', 'purpose_wedding', 'issue_month', 'issue_year',\n",
       "       'cr_month', 'cr_year', '05113', '11650', '22690', '29597', '30723',\n",
       "       '48052', '70466', '86630', '93700'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>term</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>loan_status</th>\n",
       "      <th>dti</th>\n",
       "      <th>open_acc</th>\n",
       "      <th>pub_rec</th>\n",
       "      <th>...</th>\n",
       "      <th>cr_year</th>\n",
       "      <th>05113</th>\n",
       "      <th>11650</th>\n",
       "      <th>22690</th>\n",
       "      <th>29597</th>\n",
       "      <th>30723</th>\n",
       "      <th>48052</th>\n",
       "      <th>70466</th>\n",
       "      <th>86630</th>\n",
       "      <th>93700</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>11.44</td>\n",
       "      <td>329.48</td>\n",
       "      <td>117000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>26.24</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1990</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>8000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>11.99</td>\n",
       "      <td>265.68</td>\n",
       "      <td>65000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>22.05</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2004</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>15600.0</td>\n",
       "      <td>0</td>\n",
       "      <td>10.49</td>\n",
       "      <td>506.97</td>\n",
       "      <td>43057.0</td>\n",
       "      <td>0</td>\n",
       "      <td>12.79</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2007</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>7200.0</td>\n",
       "      <td>0</td>\n",
       "      <td>6.49</td>\n",
       "      <td>220.65</td>\n",
       "      <td>54000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.60</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2006</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>24375.0</td>\n",
       "      <td>1</td>\n",
       "      <td>17.27</td>\n",
       "      <td>609.33</td>\n",
       "      <td>55000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>33.95</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1999</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 83 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  loan_amnt  term  int_rate  installment  annual_inc  \\\n",
       "0           0    10000.0     0     11.44       329.48    117000.0   \n",
       "1           1     8000.0     0     11.99       265.68     65000.0   \n",
       "2           2    15600.0     0     10.49       506.97     43057.0   \n",
       "3           3     7200.0     0      6.49       220.65     54000.0   \n",
       "4           4    24375.0     1     17.27       609.33     55000.0   \n",
       "\n",
       "   loan_status    dti  open_acc  pub_rec  ...  cr_year  05113  11650  22690  \\\n",
       "0            0  26.24      16.0      0.0  ...     1990      0      0      1   \n",
       "1            0  22.05      17.0      0.0  ...     2004      1      0      0   \n",
       "2            0  12.79      13.0      0.0  ...     2007      1      0      0   \n",
       "3            0   2.60       6.0      0.0  ...     2006      0      0      0   \n",
       "4            1  33.95      13.0      0.0  ...     1999      0      1      0   \n",
       "\n",
       "   29597  30723  48052  70466  86630  93700  \n",
       "0      0      0      0      0      0      0  \n",
       "1      0      0      0      0      0      0  \n",
       "2      0      0      0      0      0      0  \n",
       "3      0      0      0      0      0      0  \n",
       "4      0      0      0      0      0      0  \n",
       "\n",
       "[5 rows x 83 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"loan_status\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    317696\n",
       "1     77523\n",
       "Name: loan_status, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"loan_status\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>ppscore</th>\n",
       "      <th>case</th>\n",
       "      <th>is_valid_score</th>\n",
       "      <th>metric</th>\n",
       "      <th>baseline_score</th>\n",
       "      <th>model_score</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>predict_itself</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>loan_amnt</td>\n",
       "      <td>0.0</td>\n",
       "      <td>regression</td>\n",
       "      <td>True</td>\n",
       "      <td>mean absolute error</td>\n",
       "      <td>6631.725000</td>\n",
       "      <td>9180.195000</td>\n",
       "      <td>DecisionTreeRegressor()</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>term</td>\n",
       "      <td>0.0</td>\n",
       "      <td>regression</td>\n",
       "      <td>True</td>\n",
       "      <td>mean absolute error</td>\n",
       "      <td>0.237600</td>\n",
       "      <td>0.360800</td>\n",
       "      <td>DecisionTreeRegressor()</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>int_rate</td>\n",
       "      <td>0.0</td>\n",
       "      <td>regression</td>\n",
       "      <td>True</td>\n",
       "      <td>mean absolute error</td>\n",
       "      <td>3.568534</td>\n",
       "      <td>5.070024</td>\n",
       "      <td>DecisionTreeRegressor()</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Unnamed: 0</td>\n",
       "      <td>installment</td>\n",
       "      <td>0.0</td>\n",
       "      <td>regression</td>\n",
       "      <td>True</td>\n",
       "      <td>mean absolute error</td>\n",
       "      <td>193.233012</td>\n",
       "      <td>272.617046</td>\n",
       "      <td>DecisionTreeRegressor()</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6884</th>\n",
       "      <td>93700</td>\n",
       "      <td>30723</td>\n",
       "      <td>0.0</td>\n",
       "      <td>regression</td>\n",
       "      <td>True</td>\n",
       "      <td>mean absolute error</td>\n",
       "      <td>0.131200</td>\n",
       "      <td>0.226924</td>\n",
       "      <td>DecisionTreeRegressor()</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6885</th>\n",
       "      <td>93700</td>\n",
       "      <td>48052</td>\n",
       "      <td>0.0</td>\n",
       "      <td>regression</td>\n",
       "      <td>True</td>\n",
       "      <td>mean absolute error</td>\n",
       "      <td>0.144600</td>\n",
       "      <td>0.246078</td>\n",
       "      <td>DecisionTreeRegressor()</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6886</th>\n",
       "      <td>93700</td>\n",
       "      <td>70466</td>\n",
       "      <td>0.0</td>\n",
       "      <td>regression</td>\n",
       "      <td>True</td>\n",
       "      <td>mean absolute error</td>\n",
       "      <td>0.150400</td>\n",
       "      <td>0.254194</td>\n",
       "      <td>DecisionTreeRegressor()</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6887</th>\n",
       "      <td>93700</td>\n",
       "      <td>86630</td>\n",
       "      <td>0.0</td>\n",
       "      <td>regression</td>\n",
       "      <td>True</td>\n",
       "      <td>mean absolute error</td>\n",
       "      <td>0.027000</td>\n",
       "      <td>0.052505</td>\n",
       "      <td>DecisionTreeRegressor()</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6888</th>\n",
       "      <td>93700</td>\n",
       "      <td>93700</td>\n",
       "      <td>1.0</td>\n",
       "      <td>predict_itself</td>\n",
       "      <td>True</td>\n",
       "      <td>None</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6889 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               x            y  ppscore            case  is_valid_score  \\\n",
       "0     Unnamed: 0   Unnamed: 0      1.0  predict_itself            True   \n",
       "1     Unnamed: 0    loan_amnt      0.0      regression            True   \n",
       "2     Unnamed: 0         term      0.0      regression            True   \n",
       "3     Unnamed: 0     int_rate      0.0      regression            True   \n",
       "4     Unnamed: 0  installment      0.0      regression            True   \n",
       "...          ...          ...      ...             ...             ...   \n",
       "6884       93700        30723      0.0      regression            True   \n",
       "6885       93700        48052      0.0      regression            True   \n",
       "6886       93700        70466      0.0      regression            True   \n",
       "6887       93700        86630      0.0      regression            True   \n",
       "6888       93700        93700      1.0  predict_itself            True   \n",
       "\n",
       "                   metric  baseline_score  model_score  \\\n",
       "0                    None        0.000000     1.000000   \n",
       "1     mean absolute error     6631.725000  9180.195000   \n",
       "2     mean absolute error        0.237600     0.360800   \n",
       "3     mean absolute error        3.568534     5.070024   \n",
       "4     mean absolute error      193.233012   272.617046   \n",
       "...                   ...             ...          ...   \n",
       "6884  mean absolute error        0.131200     0.226924   \n",
       "6885  mean absolute error        0.144600     0.246078   \n",
       "6886  mean absolute error        0.150400     0.254194   \n",
       "6887  mean absolute error        0.027000     0.052505   \n",
       "6888                 None        0.000000     1.000000   \n",
       "\n",
       "                        model  \n",
       "0                        None  \n",
       "1     DecisionTreeRegressor()  \n",
       "2     DecisionTreeRegressor()  \n",
       "3     DecisionTreeRegressor()  \n",
       "4     DecisionTreeRegressor()  \n",
       "...                       ...  \n",
       "6884  DecisionTreeRegressor()  \n",
       "6885  DecisionTreeRegressor()  \n",
       "6886  DecisionTreeRegressor()  \n",
       "6887  DecisionTreeRegressor()  \n",
       "6888                     None  \n",
       "\n",
       "[6889 rows x 9 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pps.matrix(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "X = df.drop('loan_status',axis=1)\n",
    "y = df['loan_status']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# label encode the target variable to have the classes 0 and 1\n",
    "y = LabelEncoder().fit_transform(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(395219, 82) (395219,)\n"
     ]
    }
   ],
   "source": [
    "print(X.shape, y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "395219\n",
      "82\n"
     ]
    }
   ],
   "source": [
    "print(X.shape[0])\n",
    "print(X.shape[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Class Ratio Calculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.098087019336197\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "counter = Counter(y)\n",
    "weight = counter[0] / counter[1]\n",
    "print (weight)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Classical Model Generation Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# define models to test\n",
    "def get_models():\n",
    "    models = list()\n",
    "    # LR\n",
    "    models.append(('LR', LogisticRegression(solver='lbfgs')))\n",
    "    # LDA\n",
    "    models.append(('LDA', LinearDiscriminantAnalysis()))\n",
    "    #KNN\n",
    "    models.append(('KNN', KNeighborsClassifier()))\n",
    "    #NB\n",
    "    models.append(('NB', GaussianNB()))\n",
    "    #CART\n",
    "    models.append(('CART', DecisionTreeClassifier()))\n",
    "    if X.shape[0] < 100000:\n",
    "        #SVM\n",
    "        models.append(('SVM', SVC(gamma='auto')))\n",
    "\n",
    "    return models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Ensemble Model Generation Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# define models to test\n",
    "def get_ensemble_models():\n",
    "    models = list()\n",
    "    #XGB\n",
    "    models.append(('XGB', XGBClassifier()))\n",
    "    #SGD\n",
    "    models.append(('SGD', SGDClassifier()))\n",
    "    #ADA\n",
    "    models.append(('AD', AdaBoostClassifier())) \n",
    "    #ET \n",
    "    models.append(('ET', ExtraTreesClassifier()))\n",
    "    #RF\n",
    "    models.append(('RF', RandomForestClassifier(n_estimators=1000)))\n",
    "    #GBM\n",
    "    models.append(('GBM', GradientBoostingClassifier(n_estimators=1000)))\n",
    "    \n",
    "    return models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Balanced Model Generation Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# define models to test\n",
    "def get_balanced_models():\n",
    "    models = list()\n",
    "    #XGB\n",
    "    models.append(('XGB_W', XGBClassifier(scale_pos_weight=weight)))    \n",
    "    #LR\n",
    "    models.append(('LR_Bal', LogisticRegression(solver='lbfgs', class_weight='balanced'))) \n",
    "    if X.shape[0] < 100000:\n",
    "        #SVM Balanced\n",
    "        models.append(('SVM_Bal', SVC(gamma='scale', class_weight='balanced')))    \n",
    "        #SVM Weight\n",
    "        models.append(('SVM_W', SVC(gamma='scale', class_weight=weights)))    \n",
    "    #Balanced RF\n",
    "    models.append(('Bal_RF', BalancedRandomForestClassifier(n_estimators=1000)))      \n",
    "    #RF\n",
    "    models.append(('RF_Bal', RandomForestClassifier(n_estimators=1000, class_weight='balanced')))\n",
    "\n",
    "    return models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Normality Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def is_normal(data):\n",
    "    alpha = 0.05\n",
    "    stat, p = normaltest(data)\n",
    "    if p > alpha:\n",
    "        normalTest = True\n",
    "    else:\n",
    "        normalTest = False\n",
    "    \n",
    "    stat, p = shapiro(data)\n",
    "    if p > alpha:\n",
    "        shapiroTest = True\n",
    "    else:\n",
    "        shapiroTest = False\n",
    "        \n",
    "    return normalTest and shapiroTest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Z:\\ProgramData\\Anaconda3\\lib\\site-packages\\scipy\\stats\\morestats.py:1681: UserWarning: p-value may not be accurate for N > 5000.\n",
      "  warnings.warn(\"p-value may not be accurate for N > 5000.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gaussian Like columns: []\n",
      "Non-Gaussian Like columns: ['Unnamed: 0', 'loan_amnt', 'term', 'int_rate', 'installment', 'annual_inc', 'dti', 'open_acc', 'pub_rec', 'revol_bal', 'revol_util', 'total_acc', 'mort_acc', 'pub_rec_bankruptcies', 'A2', 'A3', 'A4', 'A5', 'B1', 'B2', 'B3', 'B4', 'B5', 'C1', 'C2', 'C3', 'C4', 'C5', 'D1', 'D2', 'D3', 'D4', 'D5', 'E1', 'E2', 'E3', 'E4', 'E5', 'F1', 'F2', 'F3', 'F4', 'F5', 'G1', 'G2', 'G3', 'G4', 'G5', 'OTHER', 'OWN', 'RENT', 'verification_status_Source Verified', 'verification_status_Verified', 'application_type_INDIVIDUAL', 'application_type_JOINT', 'initial_list_status_w', 'purpose_credit_card', 'purpose_debt_consolidation', 'purpose_educational', 'purpose_home_improvement', 'purpose_house', 'purpose_major_purchase', 'purpose_medical', 'purpose_moving', 'purpose_other', 'purpose_renewable_energy', 'purpose_small_business', 'purpose_vacation', 'purpose_wedding', 'issue_month', 'issue_year', 'cr_month', 'cr_year', '05113', '11650', '22690', '29597', '30723', '48052', '70466', '86630', '93700']\n"
     ]
    }
   ],
   "source": [
    "Gaussian_Like = []\n",
    "Non_Gaussian = []\n",
    "\n",
    "for i, name in enumerate (X.columns):\n",
    "    if is_normal(X[name]):\n",
    "        Gaussian_Like.append(name)\n",
    "    else:\n",
    "        Non_Gaussian.append(name)\n",
    "        \n",
    "print (f\"Gaussian Like columns: {Gaussian_Like}\")\n",
    "print (f\"Non-Gaussian Like columns: {Non_Gaussian}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Standardize Only Gaussian-Like Input Variables\n",
    "# Normalize Only Non-Gaussian Input Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "if len(Gaussian_Like)> 0 and len(Non_Gaussian) > 0:\n",
    "    print(\"Normality Function is working\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Sample Generation Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Advise not to use Under sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "sm = ADASYN()\n",
    "X, y = sm.fit_sample(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def get_sampling_models():\n",
    "    models = list()\n",
    "    # SMOTETomek\n",
    "    models.append((\"SMOTETomek\", SMOTETomek(tomek=TomekLinks(sampling_strategy='majority'))))\n",
    "    #SMOTEENN\n",
    "    models.append((\"SMOTEENN\", SMOTEENN()))\n",
    "    return models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# define models to test\n",
    "def get_xgb_sampling_models():\n",
    "    models = list()\n",
    "    #SMOTEENN\n",
    "    sampling = SMOTEENN(enn=EditedNearestNeighbours(sampling_strategy='majority'))\n",
    "    model = XGBClassifier()\n",
    "    steps = [('e', sampling), ('m', model)]\n",
    "    models.append(('XGB-Sampling',Pipeline(steps=steps)))\n",
    "    #SMOTEENN + Norm\n",
    "    sampling = SMOTEENN(enn=EditedNearestNeighbours(sampling_strategy='majority'))\n",
    "    model = XGBClassifier()\n",
    "    steps = [('t', MinMaxScaler()), ('e', sampling), ('m', model)]\n",
    "    models.append(('XGB-Norm-Sampling', Pipeline(steps=steps)))\n",
    "    #SMOTEENN + Std\n",
    "    sampling = SMOTEENN(enn=EditedNearestNeighbours(sampling_strategy='majority'))\n",
    "    model = XGBClassifier()\n",
    "    steps = [('t', StandardScaler()), ('e', sampling), ('m', model)]\n",
    "    models.append(('XGB-Std_Sampling', Pipeline(steps=steps)))\n",
    "    #SMOTEENN + Power\n",
    "    sampling = SMOTEENN(enn=EditedNearestNeighbours(sampling_strategy='majority'))\n",
    "    model = XGBClassifier()\n",
    "    steps = [('t1', MinMaxScaler()), ('t2', PowerTransformer()), ('e', sampling), ('m', model)]\n",
    "    models.append(('XGB-Norm-Power-Sampling', Pipeline(steps=steps)))\n",
    "    \n",
    "    return models, names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Model Evaluation Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# evaluate a model\n",
    "def evaluate_model(X, y, model):\n",
    "    # define evaluation procedure\n",
    "    cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "        # evaluate model\n",
    "    scores = cross_val_score(model, X, y, scoring='f1_micro', cv=cv, n_jobs=-1)\n",
    "    #print('F-measure: %.3f' % score)\n",
    "    return scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Spot Check Algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F-measure of LR: 0.804\n",
      "F-measure of LDA: 0.888\n",
      "F-measure of KNN: 0.771\n",
      "F-measure of NB: 0.804\n",
      "F-measure of CART: 0.831\n",
      "Execution time :  0:09:10.509524\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "models = get_models()\n",
    "for name, model in models:\n",
    "    scores = evaluate_model(X, y, model)\n",
    "    print(f\"F-measure of {name}: {mean(scores):.3f}\")\n",
    "    \n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Spot  Check Algorithms with Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F-measure of LR: 0.889\n",
      "F-measure of LDA: 0.888\n",
      "F-measure of KNN: 0.866\n",
      "F-measure of NB: 0.865\n",
      "F-measure of CART: 0.830\n",
      "Execution time :  6:32:28.956152\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "models = get_models()\n",
    "for name, model in models:\n",
    "    # define pipeline\n",
    "    pipeline = Pipeline(steps=[('norm', MinMaxScaler()), ('m', model)])\n",
    "    scores = evaluate_model(X, y, pipeline)\n",
    "    print(f\"F-measure of {name}: {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Spot  Check Algorithms with Standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "models = get_models()\n",
    "for name, model in models:\n",
    "    # define pipeline\n",
    "    pipeline = Pipeline(steps=[('Stnd', StandardScaler()), ('m', model)])\n",
    "    scores = evaluate_model(X, y, pipeline)\n",
    "    print(f\"F-measure of {name}: {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Spot Check Ensemble Algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F-measure of XGB: 0.890\n",
      "F-measure of SGD: 0.727\n",
      "F-measure of AD: 0.889\n",
      "F-measure of ET: 0.888\n",
      "F-measure of RF: 0.889\n",
      "F-measure of GBM: 0.890\n",
      "Execution time :  11:11:38.949747\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "models = get_ensemble_models()\n",
    "for name, model in models:\n",
    "    scores = evaluate_model(X, y, model)\n",
    "    print(f\"F-measure of {name}: {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Spot Check Ensemble Algorithms with Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "models = get_ensemble_models()\n",
    "for name, model in models:\n",
    "    # define pipeline\n",
    "    pipeline = Pipeline(steps=[('norm', MinMaxScaler()), ('m', model)])\n",
    "    scores = evaluate_model(X, y, pipeline)\n",
    "    print(f\"F-measure of {name}: {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Spot Check Ensemble Algorithms with Standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "models = get_ensemble_models()\n",
    "for name, model in models:\n",
    "    # define pipeline\n",
    "    pipeline = Pipeline(steps=[('Stnd', StandardScaler()), ('m', model)])\n",
    "    scores = evaluate_model(X, y, pipeline)\n",
    "    print(f\"F-measure of {name}: {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Spot Check Balanced Model Algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "models = get_balanced_models()\n",
    "for name, model in models:\n",
    "    scores = evaluate_model(X, y, model)\n",
    "    print(f\"F-measure of {name}: {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Spot Check Balanced Model Algorithms with Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "models = get_balanced_models()\n",
    "for name, model in models:\n",
    "    # define pipeline\n",
    "    pipeline = Pipeline(steps=[('norm', MinMaxScaler()), ('m', model)])\n",
    "    scores = evaluate_model(X, y, pipeline)\n",
    "    print(f\"F-measure of {name}: {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Spot Check Balanced Model Algorithms with Standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "models = get_balanced_models()\n",
    "for name, model in models:\n",
    "    # define pipeline\n",
    "    pipeline = Pipeline(steps=[('Stnd', StandardScaler()), ('m', model)])\n",
    "    scores = evaluate_model(X, y, pipeline)\n",
    "    print(f\"F-measure of {name}: {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Raw XGB Classifier Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F-measure of XGB: 0.889\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "model = XGBClassifier()\n",
    "scores = evaluate_model(X, y, model)\n",
    "print(f\"F-measure of XGB: {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Scale Weighted XGB Classifier Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F-measure of Weighted XGB: 0.810\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "model = XGBClassifier(scale_pos_weight=weight)\n",
    "scores = evaluate_model(X, y, model)\n",
    "print(f\"F-measure of Weighted XGB: {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F-measure of Weighted XGB: 0.782\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "model = XGBClassifier(scale_pos_weight=5)\n",
    "scores = evaluate_model(X, y, model)\n",
    "print(f\"F-measure of Weighted XGB: {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### XGB with Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F-measure of XGB + Norm: 0.889\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "# define model\n",
    "model = XGBClassifier()\n",
    "# define pipeline\n",
    "pipeline = Pipeline(steps=[('norm', MinMaxScaler()), ('m', model)])\n",
    "scores = evaluate_model(X, y, pipeline)\n",
    "print(f\"F-measure of XGB + Norm: {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### XGB with Standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F-measure of XGB + Standardization: 0.889\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "# define model\n",
    "model = XGBClassifier()\n",
    "# define pipeline\n",
    "pipeline = Pipeline(steps=[('Stnd', StandardScaler()), ('m', model)])\n",
    "scores = evaluate_model(X, y, pipeline)\n",
    "print(f\"F-measure of XGB + Standardization: {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### XGB with Normalization & Standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F-measure of XGB + Norm + sampling: 0.889\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "# define model\n",
    "model = XGBClassifier()\n",
    "# define pipeline\n",
    "pipeline = Pipeline(steps=[('Stnd', StandardScaler()), ('norm', MinMaxScaler()), ('m', model)])\n",
    "scores = evaluate_model(X, y, pipeline)\n",
    "print(f\"F-measure of XGB + Norm + Stand: {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Normalize Only Non-Gaussian Input Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "if len(Gaussian_Like)> 0 and len(Non_Gaussian) > 0:\n",
    "    init_time = datetime.now()\n",
    "    # define the selective transforms\n",
    "    t = [('e', MinMaxScaler(), Non_Gaussian)]\n",
    "    selective = ColumnTransformer(transformers=t, remainder='passthrough')\n",
    "    # define pipeline\n",
    "    model = XGBRegressor()\n",
    "    # define pipeline\n",
    "    pipeline = Pipeline(steps=[('s', selective), ('m', model)])\n",
    "    scores = evaluate_model(X, y, pipeline)\n",
    "    display_scores(scores)\n",
    "    fin_time = datetime.now()\n",
    "    print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Standardize Only Gaussian-Like Input Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "if len(Gaussian_Like)> 0 and len(Non_Gaussian) > 0:\n",
    "    init_time = datetime.now()\n",
    "    # define the selective transforms\n",
    "    t = [('n', StandardScaler(), Gaussian_Like)]\n",
    "    selective = ColumnTransformer(transformers=t, remainder='passthrough')\n",
    "    # define pipeline\n",
    "    model = XGBRegressor()\n",
    "    # define pipeline\n",
    "    pipeline = Pipeline(steps=[('s', selective), ('m', model)])\n",
    "    scores = evaluate_model(X, y, pipeline)\n",
    "    display_scores(scores)\n",
    "    fin_time = datetime.now()\n",
    "    print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### XGB with Selectively Normalize and Standardize Input Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "if len(Gaussian_Like)> 0 and len(Non_Gaussian) > 0:\n",
    "    init_time = datetime.now()\n",
    "    # define the selective transforms\n",
    "    t = [('e', MinMaxScaler(), Non_Gaussian), ('n', StandardScaler(), Gaussian_Like)]\n",
    "    selective = ColumnTransformer(transformers=t)\n",
    "    # define pipeline\n",
    "    model = XGBRegressor()\n",
    "    # define pipeline\n",
    "    pipeline = Pipeline(steps=[('s', selective), ('m', model)])\n",
    "    scores = evaluate_model(X, y, pipeline)\n",
    "    display_scores(scores)\n",
    "    fin_time = datetime.now()\n",
    "    print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### XGB with PowerTransformer/QuantileTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F-measure of XGB + PowerTransformer : 0.889\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "# define model\n",
    "model = XGBClassifier()\n",
    "# define pipeline\n",
    "#trans = QuantileTransformer(n_quantiles=100, output_distribution='normal')\n",
    "#output_distribution='uniform', 'normal'\n",
    "pipeline = Pipeline(steps=[('t', PowerTransformer()), ('m', model)])\n",
    "scores = evaluate_model(X, y, pipeline)\n",
    "print(f\"F-measure of XGB + PowerTransformer : {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### XGB with PCA & Outlier Removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "# get a list of models to evaluate\n",
    "def get_models():\n",
    "    models = dict()\n",
    "    for i in range(1,X.shape[1]):\n",
    "        steps = [('pca', PCA(n_components=i)), ('m', XGBRegressor())]\n",
    "        #steps = [('svd', TruncatedSVD(n_components=i)), ('m', XGBRegressor())]\n",
    "        models[str(i)] = Pipeline(steps=steps)\n",
    "    return models\n",
    "\n",
    "# get the models to evaluate\n",
    "models = get_models() \n",
    "# evaluate the models and store results\n",
    "results, names = list(), list()\n",
    "for name, model in models.items():\n",
    "    scores = evaluate_model(X, y, model)\n",
    "    scores = np.sqrt(-scores)\n",
    "    results.append(scores.mean())\n",
    "    names.append(name)\n",
    "\n",
    "maxIndex = results.index(max(results))\n",
    "print(f\"RMSE of PCA value {names[maxIndex]} is {results[maxIndex]}\")\n",
    "print(results, names)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))\n",
    "best_n_components = int(names[maxIndex])\n",
    "print(best_n_components)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F-measure of XGB + PCA: 0.888\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "# define transform\n",
    "pca = PCA(n_components=best_n_components)\n",
    "# prepare transform on dataset\n",
    "pca.fit(X)\n",
    "# apply transform to dataset\n",
    "X_transformed = pca.transform(X)\n",
    "# identify outliers in the training dataset\n",
    "lof = LocalOutlierFactor()\n",
    "yhat = lof.fit_predict(X_transformed)\n",
    "# select all rows that are not outliers\n",
    "mask = yhat != -1\n",
    "X_transformed, y_transformed = X_transformed[mask, :], y[mask]\n",
    "# define model\n",
    "model = XGBClassifier()\n",
    "# evaluate the model\n",
    "scores = evaluate_model(X_transformed, y_transformed, model)\n",
    "\n",
    "print(f\"F-measure of XGB + PCA: {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))\n",
    "# n_components=40 -> 0.888"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### XGB with Normalization and PowerTransformer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F-measure of XGB + Norm + PowerTransformer: 0.889\n",
      "Execution time :  0:43:55.649322\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "# define model\n",
    "model = XGBClassifier()\n",
    "# define pipeline\n",
    "steps = list()\n",
    "steps.append(('norm', MinMaxScaler()))\n",
    "steps.append(('power', PowerTransformer()))\n",
    "steps.append(('m', model))\n",
    "pipeline = Pipeline(steps=steps)\n",
    "scores = evaluate_model(X, y, pipeline)\n",
    "print(f\"F-measure of XGB + Norm + PowerTransformer: {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### XGB with Normalization & PCA "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F-measure of XGB + Norm + PCA : 0.889\n",
      "Execution time :  1:59:51.619810\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "# define model\n",
    "model = XGBClassifier()\n",
    "# define pipeline\n",
    "steps = list()\n",
    "steps.append(('norm', MinMaxScaler()))\n",
    "steps.append(('pca', PCA()))\n",
    "steps.append(('m', model))\n",
    "pipeline = Pipeline(steps=steps)\n",
    "scores = evaluate_model(X, y, pipeline)\n",
    "print(f\"F-measure of XGB + Norm + PCA : {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### XGB with Normalization, PCA and PowerTransformer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F-measure of XGB + Norm + Power + PCA: 0.887\n",
      "Execution time :  1:12:02.963438\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "# define model\n",
    "model = XGBClassifier()\n",
    "# define pipeline\n",
    "steps = list()\n",
    "steps.append(('norm', MinMaxScaler()))\n",
    "steps.append(('power', PowerTransformer()))\n",
    "steps.append(('pca', PCA(n_components=40)))\n",
    "steps.append(('m', model))\n",
    "pipeline = Pipeline(steps=steps)\n",
    "scores = evaluate_model(X, y, pipeline)\n",
    "print(f\"F-measure of XGB + Norm + Power + PCA: {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### XGB with Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F-measure of XGB + UnderSample : 0.868\n",
      "Execution time :  0:28:00.000010\n"
     ]
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "from numpy import mean\n",
    "import numpy as np\n",
    "import pandas  as pd\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import RepeatedStratifiedKFold\n",
    "from imblearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import PowerTransformer\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.decomposition import PCA\n",
    "from xgboost import XGBClassifier\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from imblearn.combine import SMOTETomek\n",
    "from imblearn.combine import SMOTEENN\n",
    "from imblearn.under_sampling import EditedNearestNeighbours\n",
    "from imblearn.under_sampling import TomekLinks\n",
    "\n",
    "def convert(value):\n",
    "    if value == 0:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "    \n",
    "def load_dataset():\n",
    "    df = pd.read_csv('../Data/classification_testData_shape(395219,79).csv')\n",
    "    df[\"loan_repaid\"] = df[\"loan_repaid\"].apply(lambda v: convert(v))\n",
    "    \n",
    "    X = df.drop('loan_repaid',axis=1).values\n",
    "    y = df['loan_repaid'].values\n",
    "    y = LabelEncoder().fit_transform(y)\n",
    "    \n",
    "    return X, y\n",
    "\n",
    "# evaluate a model\n",
    "def evaluate_model(X, y, model):\n",
    "    # define evaluation procedure\n",
    "    cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "        # evaluate model\n",
    "    scores = cross_val_score(model, X, y, scoring='f1_micro', cv=cv, n_jobs=-1)\n",
    "    #print('F-measure: %.3f' % score)\n",
    "    return scores\n",
    "\n",
    "init_time = datetime.now()\n",
    "# load the dataset\n",
    "X, y = load_dataset()\n",
    "# define model\n",
    "model = XGBClassifier()\n",
    "# Define Sampling\n",
    "#resample = SMOTEENN(enn=EditedNearestNeighbours(sampling_strategy='majority'))\n",
    "resample = SMOTETomek(tomek=TomekLinks(sampling_strategy='majority'))\n",
    "#over = RandomOverSampler()\n",
    "#under = RandomUnderSampler()\n",
    "#oversample = SMOTE()\n",
    "#undersample = TomekLinks()\n",
    "undersample = EditedNearestNeighbours(n_neighbors=3)\n",
    "# Define Pipeline\n",
    "steps = list()\n",
    "steps.append(('norm', MinMaxScaler()))\n",
    "steps.append(('u', resample))\n",
    "steps.append(('m', model))\n",
    "pipeline = Pipeline(steps=steps)\n",
    "# evaluate model\n",
    "scores = evaluate_model(X, y, pipeline)\n",
    "# summarize performance\n",
    "print(f\"F-measure of XGB + Norm + SMOTETomek : {mean(scores):.3f}\")\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))\n",
    "\n",
    "# F-measure of XGB + SMOTEENN : 0.854\n",
    "# F-measure of XGB + SMOTETomek : 0.889\n",
    "# F-measure of XGB + Random_Over_Under : 0.811\n",
    "# F-measure of XGB + Over_Sampling : 0.889\n",
    "# F-measure of XGB + UnderSample (Tomek) : 0.888\n",
    "# F-measure of XGB + UnderSample (ENN) : 0.868"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "### XGB with Grid Search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train, Test Validation Data Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, stratify=y, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.20, stratify=y_train, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sampling = SMOTEENN()\n",
    "#sampling = SMOTEENN (enn= EditedNearestNeighbours (sampling_strategy ='majority'))\n",
    "#X_resampled, y_resampled = sampling.fit_sample(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Base line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time :  0:00:26.730484\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "clf_xgb = XGBClassifier()\n",
    "#norm_scaler = MinMaxScaler()\n",
    "#norm_scaler.fit(X_train)\n",
    "#pipeline = Pipeline(steps=[('norm', ), ('m', model)])\n",
    "clf_xgb.fit(X_train, y_train, verbose=False, \n",
    "            early_stopping_rounds=10, eval_metric='aucpr',\n",
    "            eval_set=[(X_valid, y_valid)])\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.99      0.94     63539\n",
      "           1       0.93      0.47      0.63     15505\n",
      "\n",
      "    accuracy                           0.89     79044\n",
      "   macro avg       0.91      0.73      0.78     79044\n",
      "weighted avg       0.89      0.89      0.87     79044\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, clf_xgb.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWgAAAEGCAYAAABIGw//AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZgV1Z3/8feHBlkFZRUBI0bcJy6gokZDxAhmxiWJJiQmEpchLtGMiTo6+SWaZJzRMYYEjUSjRjRuaDSSRY2ixiUo4oqKKInKLgKigGzd/f39Uafx0unltnbTdW9/Xs9Tz606VefcU/fS33s4deqUIgIzM8ufdq1dATMzq5sDtJlZTjlAm5nllAO0mVlOOUCbmeVU+9auQF717lkR2w/q0NrVsCZ47cUurV0Fa4K1rGZ9rNPHKWPUZ7vGsuVVRR37zIvr7o+I0R/n/TY3B+h6bD+oA9PvH9Ta1bAmGLXtXq1dBWuCp2Lqxy5j2fIqpt+/XVHHVvR/vffHfsPNzAHazEpWANVUt3Y1WowDtJmVrCDYEMV1cZQiB2gzK2luQZuZ5VAQVJXxdBUO0GZW0qpxgDYzy50Aqso4QPtGFTMradVEUUtjJG0l6U5Jr0qaJekAST0lPSDp9fS6dcHxF0iaI2m2pFEF6UMlzUz7JkhSSu8o6faU/pSk7RurkwO0mZWsADZEFLUU4RfAfRGxC7AnMAs4H5gaEUOAqWkbSbsBY4DdgdHAVZIqUjkTgXHAkLTU3BxzMvBuROwIjAcubaxCDtBmVrKCoKrIpSGSugOHANcBRMT6iFgBHA1MSodNAo5J60cDt0XEuoh4A5gD7CepP9A9IqZFNtn+jbXy1JR1JzCypnVdHwdoMytdAVVFLkBvSTMKlnEFJe0AvAP8RtJzkq6V1BXoFxGLANJr33T8AGBeQf75KW1AWq+dvkmeiKgE3gN6NXR6vkhoZiUru5OwaEsjYlg9+9oD+wBnRsRTkn5B6s6oR10t32ggvaE89XIL2sxKmKgqcmnEfGB+RDyVtu8kC9hvp24L0uuSguMLJ+sZCCxM6QPrSN8kj6T2QA9geUOVcoA2s5KVXSRUUUuD5UQsBuZJ2jkljQReAaYAY1PaWOCetD4FGJNGZgwmuxg4PXWDrJQ0PPUvn1ArT01ZxwIPRSMPhXUXh5mVrGwc9MeasbTQmcDNkrYA/gGcSNaInSzpZGAucBxARLwsaTJZEK8EzojYOCnIacANQGfg3rRAdgHyJklzyFrOYxqrkAO0mZW06kZax8WKiOeBuvqoR9Zz/MXAxXWkzwD2qCN9LSnAF8sB2sxKVjO3oHPHAdrMSlYgqsr4UpoDtJmVtObq4sgjB2gzK1mBWB8VjR9YohygzaxkZTequIvDzCyXfJHQzCyHIkRVuAVtZpZL1W5Bm5nlT3aRsHzDWPmemZmVPV8kNDPLsSqPgzYzyx/fSWhmlmPVHsVhZpY/2WRJDtBmZrkTiA2+1dvMLH8i8I0qZmb5JN+oYmaWR4Fb0GZmueWLhGZmORTIE/abmeVRABs8F4eZWR7J80GbmeVR4DsJzcxyyy1oM7McilBZt6DL98zMrOxlFwkriloaI+lNSTMlPS9pRkrrKekBSa+n160Ljr9A0hxJsyWNKkgfmsqZI2mCJKX0jpJuT+lPSdq+sTo5QJtZCcueSVjMUqTPRsReETEsbZ8PTI2IIcDUtI2k3YAxwO7AaOAqSTW/AhOBccCQtIxO6ScD70bEjsB44NLGKuMAbWYlK7tIqKKWj+hoYFJanwQcU5B+W0Ssi4g3gDnAfpL6A90jYlpEBHBjrTw1Zd0JjKxpXdfHAdrMSloV7YpaihDAXyQ9I2lcSusXEYsA0mvflD4AmFeQd35KG5DWa6dvkiciKoH3gF4NVcgXCc2sZDXxTsLeNX3LyTURcU3B9kERsVBSX+ABSa82UFZdbxoNpDeUp14O0GZW0prw0NilBX3L/yQiFqbXJZLuBvYD3pbUPyIWpe6LJenw+cCgguwDgYUpfWAd6YV55ktqD/QAljdUYXdxmFnJioAN1e2KWhoiqaukLWvWgcOBl4ApwNh02FjgnrQ+BRiTRmYMJrsYOD11g6yUNDz1L59QK09NWccCD6V+6nq5BW1mJSvr4miWdmY/4O50za49cEtE3CfpaWCypJOBucBxABHxsqTJwCtAJXBGRFSlsk4DbgA6A/emBeA64CZJc8hazmMaq5QDtJmVtOa4kzAi/gHsWUf6MmBkPXkuBi6uI30GsEcd6WtJAb5YDtAlatV7FYw/ZxBvvtoJCb77s7k88eetePKB7nTYIuj/iXV8b/w8uvWoYsN68YvzBvL6i11QOzjtxwvY88BVADx891bcdkU/JOjZbwP/ecVb9OhVtfF9HvtjD/573GCuuHc2O+25prVOt6xNeuoV1qyqoLoaqirFmUfsxNe/t5gjvraM95Znf6K/+d/+PP1Qd/oNXM+v//oq8//REYBXn+nKhPMHNlR8WasZZleuSipAS6oCZpLVexYwNiI+qOfYo4DdIuKSOvatiohuLVrZFjbxhwMYNuJ9fvDrN9mwXqxb0441h6zkpP9aSEV7uPa/+3PbFX055f8t4t6bs5E8Vz80mxVL2/P943fgintfI6qzcn79yKv06FXFtT/pz5Tf9OEb5ywG4INV7fj9dX3YZZ/VrXmqbcJ5x32S95dv+ud496/7cOev+v7TsYve6sjpn9t5c1Ut53yrd56sSXf57AGsB06t78CImFJXcC4Hq1e2Y+aTXRn9tewCcIctgm49qhg6YiUV6W9816EfsHRRBwDmvtaRvQ/OWsxb9a6kW48qXnuhCxFAiLVr2hEBq1dV0GubDRvfZ9L/9ee405ewRccGr2OYtarq9FzCxpZSVGoButBjwI6Sjkz3tT8n6UFJ/QAkfVPSlWl9sKRpkp6W9JNWrXUzWPxWR3r0quTys7fj9M/txPjvDWLtB5t+lfff2pN9D10JwA67r2Xa/T2oqoTFc7fg9Re78M7CDrTvAGdeMo9TD92Fr+29O3Nf68Sory4DYM7MzryzsAPDP/f+Zj+/NifE/9z6D6687zWOOH7ZxuQjT1zKxAdn892fzaVbj8qN6dtst55f/mU2l/1uDnvst6o1apwb2SiOiqKWUlSSATqNITyCrLvjcWB4ROwN3AacV0eWXwATI2JfYHED5Y6TNEPSjHeWVdV3WKurqoI5M7vwbycs5aoHXqNTl2puv/LD/wrf8ot+VLQPDv3iuwCMGrOM3v3X8+3ROzPxhwPYbdhqKiqCyg3wxxt788u/zOaW515m8K5ruP2KflRXw9UXDWDchQvrq4I1o7OP3pFvj9qJ7x8/mKO+uZQ99l/FHyf14sQDduX0z+3E8rc7bPwuli9pz9f33ZUzDt+Zqy/alvOvmkuXbvn9t9rSam5UacFbvVtVqQXozpKeB2aQDXm5jmwg+P2SZgLnkk1eUttBwK1p/ab6Co+IayJiWEQM69Mrv7+4vftvoE//DeyyT9b9/ul/W8GcmZ0BeGDy1kx/sDv/eeVb1NzlX9EeTv3RQiY+OJsf3fAGq96rYMAO6/j7y1mebbdfjwSfOWoFr8zoyppV7Xjz1U6c96UdOWG/3Zj1bBcu/OYOvPZC51Y533K3/O2sK+q9ZR144r4e7LL3B6xY2oHqahEh7r25FzvvlV2g3bC+HSvfzfqx5szswsI3t2DADutare55UM5dHCV1kZDUB12YIOkK4GcRMUXSCOCievKWTUdqz76V9N52PfPmdGTQjut4/rEt2W7IOp5+eEsm/7Ifl931Op26fHi6az8QIDp1qeaZv3ajon3wiZ3WsWxxe+a+1okVyyrYqlcVzz66JYOGrKVr92ruePmljfnP/dKO/PsPF3gURwvo2LmKdu1gzeoKOnauYuhnVnLzz/rRs+8Gli/JAveBR7zHm7M7AdCjZyUrV1RQXS222W4dAwavY/HcLVrzFFqVR3HkXw9gQVofW88xT5ANCv8tcPzmqFRLO+O/F3Dptz9B5QaxzXbr+d74uZz5+Z3YsE5c8JUdAdhl6Gq+c+l8VizrwPe/ugNqB7222cB5V7wFQK9tKjn+u4s55wtDaN8h6DtgPef8fG5rnlabs3WfSi687k0AKtoHD9+9NTMe6c65E+byyd3XEAFvz9+CCedlQ+n+ZfgqTjh3MVWVoqpaTDh/ICtXlMOf8UdXzqM41MidhrlS1/A4SUeTza26AHgS2DciRkj6JjAsIr6dbsW8hewH6XfA/2tsmN2wPTvF9PsHNXSI5cyobfdq/CDLjadiKu/H8o/V/N16l75x6PXHFnXsXQdNfKahuTjyqKR+eusKqhFxDx/e616YfgPZ7Zak+VoPKNhdlsPvzNoid3GYmeWQ+6DNzHLMAdrMLIeaOGF/yXGANrOSVqpjnIvhAG1mJSsCKhuZjL+UOUCbWUlzF4eZWQ65D9rMLMfCAdrMLJ98kdDMLIci3AdtZpZTosqjOMzM8sl90GZmOeS5OMzM8iqyfuhy5QBtZiXNozjMzHIoyvwiYfmemZm1CRHFLcWQVCHpOUl/TNs9JT0g6fX0unXBsRdImiNptqRRBelDJc1M+yZI2eObJXWUdHtKf0rS9o3VxwHazEpahIpaivQdYFbB9vnA1IgYAkxN20jajew5p7sDo4GrJFWkPBOBccCQtIxO6ScD70bEjmSP6bu0sco4QJtZycpax80ToCUNBP4VuLYg+WhgUlqfBBxTkH5bRKxLj9SbA+wnqT/QPSKmRfbA1xtr5akp605gZE3ruj7ugzazktaEYXa9Jc0o2L4mIq4p2P45cB6wZUFav4hYBBARiyT1TekDyB5SXWN+StuQ1mun1+SZl8qqlPQe0AtYWl+FHaDNrKQ1YZjd0vqe6i3p34AlEfGMpBFFlFXXr0I0kN5Qnno5QJtZyQpEdfOM4jgIOErS54FOQHdJvwXeltQ/tZ77A0vS8fOBQQX5BwILU/rAOtIL88yX1B7oASxvqFLugzazkhZFLg2WEXFBRAyMiO3JLv49FBFfB6YAY9NhY4F70voUYEwamTGY7GLg9NQdslLS8NS/fEKtPDVlHZvewy1oMytT0eJzcVwCTJZ0MjAXOA4gIl6WNBl4BagEzoiIqpTnNOAGoDNwb1oArgNukjSHrOU8prE3d4A2s9LWzLd6R8QjwCNpfRkwsp7jLgYuriN9BrBHHelrSQG+WA7QZlbS2uRsdpKuoIHfpog4q0VqZGZWpACqq9tggAZmNLDPzKz1BdAWW9ARMalwW1LXiFjd8lUyMyteOU832ugwO0kHSHqFdH+6pD0lXdXiNTMzK0ZzjLPLqWLGQf8cGAUsA4iIF4BDWrJSZmbFKW4ejlK9kFjUKI6ImFdrTo+q+o41M9usSrR1XIxiAvQ8SQcCIWkL4Cw2nY7PzKx1BEQZj+IopovjVOAMspmYFgB7pW0zsxxQkUvpabQFHRFLgeM3Q13MzJqujLs4ihnFsYOkP0h6R9ISSfdI2mFzVM7MrFFtfBTHLcBkoD+wLXAHcGtLVsrMrCg1N6oUs5SgYgK0IuKmiKhMy28p2d8jMys3zfnQ2LxpaC6Onmn1YUnnA7eRBeavAH/aDHUzM2tcGY/iaOgi4TNs+giXbxXsC+AnLVUpM7NiqURbx8VoaC6OwZuzImZmTVbCFwCLUdSdhJL2AHYje1YXABFxY0tVysysOKV7AbAYjQZoSRcCI8gC9J+BI4DHAQdoM2t9ZdyCLmYUx7Fkj3xZHBEnAnsCHVu0VmZmxaoucilBxXRxrImIakmVkrqTPXbcN6qYWetrqxP2F5ghaSvg12QjO1YB01u0VmZmRWqTozhqRMTpafVXku4DukfEiy1bLTOzIrXFAC1pn4b2RcSzLVMlMzODhlvQlzewL4BDm7kuufL6a1vz+cO+3NrVsCaoGFLZ2lWwJtBbjzVPOW2xBR0Rn92cFTEza7Kgzd7qbWaWf2Xcgi5mHLSZWW4pilsaLEPqJGm6pBckvSzpRym9p6QHJL2eXrcuyHOBpDmSZksaVZA+VNLMtG+C0gNdJXWUdHtKf0rS9o2dmwO0mZW25pmwfx1waETsSfZYv9GShgPnA1MjYggwNW0jaTdgDLA7MBq4SlJFKmsiMA4YkpbRKf1k4N2I2BEYD1zaWKWKeaKKJH1d0g/T9naS9mv0dM3MNodmCNCRWZU2O6QlgKOBSSl9EnBMWj8auC0i1kXEG8AcYD9J/cmGIk+LiCCbEqMwT01ZdwIja1rX9SmmBX0VcADw1bS9EvhlEfnMzFpUsd0bqYujt6QZBcu4TcqSKiQ9T3a39AMR8RTQLyIWAaTXvunwAcC8guzzU9qAtF47fZM8EVEJvAf0auj8irlIuH9E7CPpuVTwu5K2KCKfmVnLK34Ux9KIGFbfzoioAvZKd07fnWbxrE9dbxoNpDeUp17FtKA3pL6VAJDUh5KdesTMyk1zXCQsFBErgEfI+o7fTt0WpNcl6bD5wKCCbAOBhSl9YB3pm+SR1B7oASxvqC7FBOgJwN1AX0kXk001+j9F5DMza3nN0ActqU9qOSOpM3AY8CowBRibDhsL3JPWpwBj0siMwWQXA6enbpCVkoan/uUTauWpKetY4KHUT12vYubiuFnSM2RTjgo4JiJmNZbPzKzFNbF13ID+wKTUW9AOmBwRf5Q0DZgs6WRgLnAcQES8LGky8ApQCZyRukgATgNuADoD96YF4DrgJklzyFrOYxqrVDET9m8HfAD8oTAtIuY2espmZi2tGQJ0mgBu7zrSl5E1TuvKczFwcR3pM4B/6r+OiLWkAF+sYi4S/okPO787AYOB2WTj/8zMWpXK+IpYMV0c/1K4nWa5+1Y9h5uZWTNp8lwcEfGspH1bojJmZk1WxnNxFNMH/d2CzXbAPsA7LVYjM7NiNd9FwlwqpgW9ZcF6JVmf9O9apjpmZk3UVgN0GnLSLSLO3Uz1MTNrmrYYoCW1j4jKhh59ZWbWmkTbHcUxnay/+XlJU4A7gNU1OyPirhaum5lZw9wHTU9gGdkzCGvGQwfgAG1mra+NBui+aQTHS/zzLE1l/JGYWUkp42jUUICuALrxEabIMzPbXNpqF8eiiPjxZquJmdlH0UYDdPk+y9zMykO03VEcdc7gZGaWK22xBR0RDc70b2aWB221D9rMLP8coM3McqiIx1mVMgdoMytZwl0cZma55QBtZpZXDtBmZjnlAG1mlkOezc7MLMccoM3M8qmt3uptZpZ77uIwM8ujMr9RpV1rV8DM7GOJIpcGSBok6WFJsyS9LOk7Kb2npAckvZ5ety7Ic4GkOZJmSxpVkD5U0sy0b4IkpfSOkm5P6U9J2r6xU3OANrOSVXMnYTFLIyqB70XErsBw4AxJuwHnA1MjYggwNW2T9o0BdgdGA1dJqkhlTQTGAUPSMjqlnwy8GxE7AuOBSxurlAO0mZU0VUdRS0MiYlFEPJvWVwKzgAHA0cCkdNgk4Ji0fjRwW0Ssi4g3gDnAfpL6A90jYlpEBHBjrTw1Zd0JjKxpXdfHAdrMSlex3RtZfO4taUbBMq6uIlPXw97AU0C/iFgEWRAH+qbDBgDzCrLNT2kD0nrt9E3yREQl8B7Qq6HT80VCMytpTRjFsTQihjVYltQN+B3wHxHxfgMN3Pqe1drQM1yb/HxXt6DNrLQ1w0VCAEkdyILzzRFxV0p+O3VbkF6XpPT5wKCC7AOBhSl9YB3pm+SR1B7oATT4YBQHaDMrac1xkTD1BV8HzIqInxXsmgKMTetjgXsK0sekkRmDyS4GTk/dICslDU9lnlArT01ZxwIPpX7qermLw8xKW/OMgz4I+AYwU9LzKe2/gEuAyZJOBuYCxwFExMuSJgOvkI0AOSMiqlK+04AbgM7AvWmB7AfgJklzyFrOYxqrlAO0mZWuZnqqd0Q8Tt19xFDPA7Qj4mLg4jrSZwB71JG+lhTgi+UAbWYly09UMTPLs4a7cUuaA7SZlTS3oC3XjvnSa4w64g0i4M03ejD+sn35xokvsf/wRVRWtmPRwq6Mv2xfVq/eAoAvf3UWh49+g+pq8atf7s2zM7YB4IQTZzLyc2/Rbcv1fOnIL7bmKZW1AYNWcv6FT2/c7r/tam66fle6d1/P8E8vorpavLeiIz/7331YvqwzAF8+fjaHf/6t7Dub8CmefbrfJmX+8H+msU3/1Zx+4mGb9VxanSdL+mgkhaTLC7bPkXRRI3mOSfe417XvIkkLJD0v6SVJRzVS1p8lbVVPOecUeRq516vXGo465nW+c/phnP7vo6ioCD7z2Xk890w/TjvlcM4YdzgL5m/Jl7/6KgCDtnufQ0bM49RTRvGDCw7hjLOepV277F/4U09uy398u87rIdaMFszbkjNPOZQzTzmU74z7LGvXVjDtsW2587YhnHHSSM485VCmT9uGr41N39kn3ueQQ+dz6jdH8oNzD+SMs1/Y+J0BHHjwAtauabttLVUXt5SilhwHvQ74oqTeTchzDFBngE7GR8ReZFdCr5dUb/0j4vMRsaIJ712yKiqCLTpW0a5dNR07VrFsWSeee2Ybqquzj+fVWb3o3WcNAAcctIBHHxlE5YYK3l7clYULu7HTztlY+dmzevHu8s6tdh5t0Z77LGHxwq4sebsLaz7osDG9U6dKIg0qOODTi3j0oYEffmcLurLTrtl31qlzJV/48hxuvXHnVql/HjhAfzSVwDXA2bV3SPqEpKmSXkyv20k6EDgKuCy1kj9ZX8ERMSuV31vS7yU9k6YIHFfwHm/W/DhI+n6aEvBBoKz+JS9b1pm77tiZSbf8kZsn/4HVqzvw3DPbbHLM4aPfYMb0LK1XrzW8s6TLxn1L3+lMr95rNmud7UOfGTmfR6Z+eOPZCae8zKQ77mPEYfO46bpdAejVey3vLPnwhzP7ztYC8I2TXuGuyUNYt66CNinILhIWs5Sglr6T8JfA8ZJ61Eq/ErgxIj4F3AxMiIi/kd1pc25E7BURf6+vUEn7A9XAO8BJETEUGAacJalXrWOHkg0I3xv4IrBvA+WOq5lIZX3VB00911bRrdt6hh+4gBO//q98/StH0qlTJZ8d+dbG/V/52iyqqsTDU7cDoOG5s2xzat++mv0PXMzjjwzYmHbjtbsz9rjRPPLgII784j8AUF1XwQJ22HEF2w5czbTHtt1cVc6lZppuNJdaNEBHxPtk0+2dVWvXAcAtaf0m4NNFFnl2usvnp8BX0m2SZ0l6AXiS7D73IbXyHAzcHREfpPpMaaC+10TEsIgYtkVFl/oOy5W99nmbxYu78v57HamqascTjw9g192XATDyc2+y3/CFXPa/+1MzBn/p0s706fvhj0/vPmtYttTdGq1h2P6L+fvrW7Hi3U7/tO+RBwdx0CELgKzF3Kfvh//L6d1nDcuWdWKX3Zez404r+M1t9/PTKx5lwKBVXPLzxzZb/XOjmebiyKPNMRfHz8kmqu7awDHFfnzjU+v64Ih4TNII4DDggIjYE3gO+Od/7SX79TTunSVd2GXX5XTsWAkEe+29hHlzt2Tovos5bsyr/OgHn2bdug8vID35t205ZMQ82neoot82q9l2wCpem92z9U6gDfvMyPn8taB7Y9sBqzau73/QIubP3RKAJ5/ozyGHzv/wOxu4itdm9eTP9+zAN750BCeOGcU5Zx7CgnndOP8/Dt7s59GamnHC/lxq8Uu/EbE83bN+MnB9Sv4bWbfDTcDxwOMpfSWwZROK70H2hIIPJO1C9iSE2h4FbpB0Cdn5Hglc3eQTyanZr/bi8UcHMmHig1RViX/M2Yp7/7QDv7r2fjp0qObiS/+aHTerF1f+Yihz3+rBY38dxNXX3U9VlZg4YW+qq7PW9Un//iIjDp1Lx45V3HjrH7n/3sHcfOPurXl6Zatjx0r2HraEKy7fe2Paid96mQGDVhIhlrzdhSsv3wuAuW9257GHB3L1pKnZd/bzPTd+Z21eND4ZfylTI5MpffSCpVUR0S2t9wPeAP4vIi5KE2JfD/Qm60c+MSLmSjoI+DXZCJBjC/uh0xC9VRHx04K0jsDvySbCng30AS6KiEckvQkMi4ilkr5PNqvUW2RT/r1SWE5denTuHwd88qSP/0HY5rOhsrVrYE0w7a1JvLd28cf6pdlyq4Gx9yHfKerYx/5w3jONzQedNy3Wgq4Jzmn9baBLwfabwKF15HmCeobZRcRFdaStA46o5/jtC9brnNTEzEpfqXZfFKPtjm43s9IXQBl3cThAm1lpK9/47ABtZqXNXRxmZjlVzqM4HKDNrHSV8E0oxXCANrOSld2oUr4R2gHazEpbic5UVwwHaDMraW5Bm5nlkfugzczyqrzn4nCANrPS5i4OM7McitJ9nFUxHKDNrLSVcQt6c0zYb2bWcprpiSqSrpe0RNJLBWk9JT0g6fX0unXBvgskzUnPOx1VkD5U0sy0b4KUPWhOUkdJt6f0p9K0yw1ygDazkqbq6qKWItwAjK6Vdj4wNSKGAFPTNpJ2I3voyO4pz1WSap7cOxEYR/b4vSEFZZ5M9oCRHYHxwKWNVcgB2sxKV5DdqFLM0lhREY8Cy2slHw1MSuuTgGMK0m+LiHUR8QYwB9hPUn+ge0RMS89MvbFWnpqy7gRG1rSu6+M+aDMrWSKacqNKb0kzCraviYhrGsnTLyIWAUTEIkl9U/oAsgdV15if0jak9drpNXnmpbIqJb0H9AKW1vfmDtBmVtqKD9BLm/GRV3W1fKOB9Iby1MtdHGZW2iKKWz6at1O3Bel1SUqfDwwqOG4gsDClD6wjfZM8ktqTPfS6dpfKJhygzax0NWMfdD2mAGPT+ljgnoL0MWlkxmCyi4HTU3fISknDU//yCbXy1JR1LPBQNPLUbndxmFlJK3KERuPlSLcCI8j6qucDFwKXAJMlnQzMBY4DiIiXJU0GXgEqgTMioioVdRrZiJDOwL1pAbgOuEnSHLKW85jG6uQAbWYl7GN1X2xaUsRX69k1sp7jLwYuriN9BrBHHelrSQG+WA7QZla6grK+k9AB2sxKm+fiMDPLJ0/Yb2aWVw7QZmY5FAFV5dvH4QBtZqXNLWgzs5xygDYzy6EA/ExCM7M8Cgj3QZuZ5dlSt5YAAAZnSURBVE/gi4RmZrnlPmgzs5xygDYzy6Pmmywpjxygzax0BdBM043mkQO0mZU2t6DNzPLIt3qbmeVTQHgctJlZTvlOQjOznHIftJlZDkV4FIeZWW65BW1mlkdBVFW1diVajAO0mZUuTzdqZpZjHmZnZpY/AYRb0GZmORSesN/MLLfK+SKhooyHqHwckt4B3mrterSA3sDS1q6ENUm5fmefiIg+H6cASfeRfT7FWBoRoz/O+21uDtBtjKQZETGstethxfN31na1a+0KmJlZ3RygzcxyygG67bmmtStgTebvrI1yH7SZWU65BW1mllMO0GZmOeUAXSYkVUl6XtJLku6Q1KWBY4+SdH49+1a1XC3bBkkh6fKC7XMkXdRInmMk7VbPvoskLSj4fo9qpKw/S9qqnnLOKfI0LAccoMvHmojYKyL2ANYDp9Z3YERMiYhLNl/V2px1wBclFXsDBcAxQJ0BOhkfEXsBxwHXS6r3bzciPh8RK5rw3pZTDtDl6TFgR0lHSnpK0nOSHpTUD0DSNyVdmdYHS5om6WlJP2nVWpePSrKRF2fX3iHpE5KmSnoxvW4n6UDgKOCy1Er+ZH0FR8SsVH5vSb+X9IyklyWNK3iPN2t+HCR9X9JsSQ8COzfzeVoLc4AuM5LaA0cAM4HHgeERsTdwG3BeHVl+AUyMiH2BxZutouXvl8DxknrUSr8SuDEiPgXcDEyIiL8BU4Bz0/+C/l5foZL2B6qBd4CTImIoMAw4S1KvWscOBcYAewNfBPZtnlOzzcWTJZWPzpKeT+uPAdeRtZhul9Qf2AJ4o458BwFfSus3AZe2dEXbgoh4X9KNwFnAmoJdB5AFS8g+7/8rssizJX0dWAl8JSJC0lmSvpD2DwKGAMsK8hwM3B0RHwBImvLRzsZaiwN0+ViT+ig3knQF8LOImCJpBHBRPXk9GL5l/Bx4FvhNA8cU+9mPj4if1myk7/Mw4ICI+EDSI0Cnj1G+5ZC7OMpbD2BBWh9bzzFPkP03GOD4Fq9RGxIRy4HJwMkFyX9j08/78bS+EtiyCcX3AN5NwXkXYHgdxzwKfEFSZ0lbAkc2pf7W+hygy9tFwB2SHqP+6Sq/A5wh6WmyP3prXpez6XSYZwEnSnoR+AbZ5w/ZNYJz0wXdei8SFrgPaJ/K+QnwZO0DIuJZ4HbgeeB3ZF1fVkJ8q7eZWU65BW1mllMO0GZmOeUAbWaWUw7QZmY55QBtZpZTDtD2kTRl9rwiyrpB0rFp/dr6ZnVL+0ekuSua+h4b56coJr3WMU2a4c+zxllzcYC2j6rB2fMkVXyUQiPilIh4pYFDRgBNDtBmpcgB2ppDzex5IyQ9LOkWYKakCkmXpZnyXpT0LQBlrpT0iqQ/AX1rCpL0iKRhaX20pGclvZBmftue7Ifg7NR6P1hSH0m/S+/xtKSDUt5ekv6Sbvy4GlBjJ1Hf7HBp3+WpLlMl9Ulpn5R0X8rzWLqjz6zZeC4O+1gKZs+7LyXtB+wREW+kIPdeROwrqSPwhKS/kM2utjPwL0A/4BXg+lrl9gF+DRySyuoZEcsl/QpYVTMvRfoxGB8Rj0vaDrgf2BW4EHg8In4s6V+BTQJuPU5K79EZeFrS7yJiGdAVeDYivifph6nsb5NNKXpqRLyeZpm7Cjj0I3yMZnVygLaPqq7Z8w4EpkdEzax5hwOfqulfJruVfAhwCHBrRFQBCyU9VEf5w4FHa8pK81rU5TBgN2ljA7l7mnfiENKscRHxJ0nvFnFO9c0OV012yzTAb4G7JHVL53tHwXt3LOI9zIrmAG0fVV2z5wGsLkwCzoyI+2sd93kan2VNRRwDWTfdARFROKVnTV2KnsegCbPD1ZTbDlhR+zMwa07ug7aWdD9wmqQOAJJ2ktSVbJa1MamPuj/w2TryTgM+I2lwytszpdee9e0vZN0NpONqAuajpNn5JB0BbN1IXRuaHa4dUPO/gK+RdZ28D7wh6bj0HpK0ZyPvYdYkDtDWkq4l619+VtJLwNVk/2u7G3id7KkvE4G/1s4YEe+Q9RvfJekFPuxi+APZFJrPSzqYbHa4Yeki5Ct8OJrkR8Ahkp4l62qZ20hdG5odbjWwu6RnyPqYf5zSjwdOTvV7GTi6iM/ErGiezc7MLKfcgjYzyykHaDOznHKANjPLKQdoM7OccoA2M8spB2gzs5xygDYzy6n/D23daWiX1wARAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_confusion_matrix(clf_xgb, X_test, y_test, values_format='d', display_labels=[\"Paid\",\"Not Paid\"]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Randomized Search Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_RandomizedSearch_model(parameters):\n",
    "    model = XGBClassifier(objective='binary:logistic')\n",
    "    xgb_grid = RandomizedSearchCV(model,\n",
    "                        parameters,\n",
    "                        scoring='f1_micro',\n",
    "                        cv = 5,\n",
    "                        n_jobs = 10,\n",
    "                        verbose=False)\n",
    "    xgb_grid.fit(X_train, y_train)\n",
    "    return xgb_grid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### First Grid Search Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = {\n",
    "                'n_estimators': [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000],\n",
    "                'max_depth': [3, 4, 5, 6, 7, 8, 9, 10],\n",
    "                'learning_rate': [0.0001, 0.001, 0.01, 0.02, 0.1, 0.2, 0.3], \n",
    "                'subsample' : [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 1.0],\n",
    "                'colsample_bytree' : [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 1.0],\n",
    "                'colsample_bylevel' : [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 1.0],\n",
    "                'gamma': [0],\n",
    "                'reg_lambda': [0, 1.0, 3.0, 5.0, 7.0, 10.0, 12.0],\n",
    "                'scale_pos_weight': [1, 2, 3, 4, 5, 6]\n",
    "              }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time :  0:23:57.528534\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "grid_model = get_RandomizedSearch_model(parameters)\n",
    "grid_predictions = grid_model.predict(X_test)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5943520239336385\n",
      "0.8867845756793684\n"
     ]
    }
   ],
   "source": [
    "#f1_score, accuracy_score\n",
    "\n",
    "print(f1_score(y_test, grid_predictions))\n",
    "print(accuracy_score(y_test, grid_predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'subsample': 0.5,\n",
       " 'scale_pos_weight': 3,\n",
       " 'reg_lambda': 12.0,\n",
       " 'n_estimators': 800,\n",
       " 'max_depth': 3,\n",
       " 'learning_rate': 0.0001,\n",
       " 'gamma': 0,\n",
       " 'colsample_bytree': 0.8,\n",
       " 'colsample_bylevel': 0.8}"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_model.best_params_\n",
    "\n",
    "#f1_score = 0.594, accuracy_score = 0.886"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWgAAAEGCAYAAABIGw//AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZgdVbnv8e8vnXmETGQEgkQwIAQIGEQRRCEqCHrgGEWJGg0g01GRC8dzPRy9KDhFAUERkIAgIGNUBjHAkSEkJCESCIZEhpCRDJCRDN393j9qddhpunfvDt3p2t2/z/PUs2uvXWvtVd3w9sqqVW8pIjAzs/xp19IdMDOzujlAm5nllAO0mVlOOUCbmeWUA7SZWU61b+kO5FXf3hWx59AOLd0Na4QXn+3a0l2wRtjEBrbEZr2bNo47ulusWl1V0rEzn938YESMeTfft7M5QNdjz6EdmP7g0JbuhjXCcYNGtnQXrBGmxZR33caq1VVMf3D3ko6tGDi/77v+wp3MAdrMylYA1VS3dDeajQO0mZWtINgapU1xlCMHaDMrax5Bm5nlUBBUteJ0FQ7QZlbWqnGANjPLnQCqHKDNzPKpNY+gfSehmZWtALZGlLQ1RNIuku6Q9E9JL0g6XFJvSQ9Jmp9edy04/iJJCyTNk3RcQfkhkuakzy6XpFTeSdJtqXyapD0b6pMDtJmVrSCoKnErwS+BByJiX+BA4AXgQmBKRAwHpqT3SBoBjAX2A8YAV0mqSO1cDUwAhqet5u7F8cAbEbE3MBG4rKEOOUCbWfkKqCpxK0ZST+BI4DqAiNgSEW8CJwKT0mGTgJPS/onArRGxOSJeBhYAh0kaCPSMiKmRPQ3lxlp1atq6AzimZnRdHwdoMytb2Z2EpW1AX0kzCrYJBU3tBawAfifpGUnXSuoG7BYRSwHSa/90/GDgtYL6i1LZ4LRfu3y7OhFRCawB+hQ7P18kNLMyJqooOd/SyogYVc9n7YGDgXMiYpqkX5KmM+r94neKIuXF6tTLI2gzK1vZRUKVtDVgEbAoIqal93eQBezladqC9Pp6wfGF2dSGAEtS+ZA6yrerI6k90AtYXaxTDtBmVrayddAqaSvaTsQy4DVJ+6SiY4C5wGRgXCobB9yb9icDY9PKjGFkFwOnp2mQdZJGp/nl02rVqWnrZODhaOCp3Z7iMLOyVt3w6LhU5wA3S+oIvAR8hWwQe7uk8cBC4BSAiHhe0u1kQbwSOCtiW9amM4EbgC7A/WmD7ALkTZIWkI2cxzbUIQdoMytbNSPoJmkrYjZQ1xz1MfUcfwlwSR3lM4D96yjfRArwpXKANrOyFYiqVjxT6wBtZmWtCac4cscB2szKViC2REXDB5YpB2gzK1vZjSqe4jAzy6WmukiYRw7QZla2IkRVeARtZpZL1R5Bm5nlT3aRsPWGsdZ7ZmbW6vkioZlZjlV5HbSZWf74TkIzsxyr9ioOM7P8yZIlOUCbmeVOILb6Vm8zs/yJwDeqmJnlk3yjiplZHgUeQZuZ5ZYvEpqZ5VAgJ+w3M8ujALY6F4eZWR7J+aDNzPIo8J2EZma55RG0mVkORcgjaDOzPMouEvpWbzOzHGrdzyRsvWdmZq1edpFQJW0NkfSKpDmSZkuakcp6S3pI0vz0umvB8RdJWiBpnqTjCsoPSe0skHS5JKXyTpJuS+XTJO3ZUJ8coM2srFXRrqStREdHxMiIGJXeXwhMiYjhwJT0HkkjgLHAfsAY4CpJNXMtVwMTgOFpG5PKxwNvRMTewETgsoY64wBtZmWr5k7CphhB1+NEYFLanwScVFB+a0RsjoiXgQXAYZIGAj0jYmpEBHBjrTo1bd0BHFMzuq6PA7SZlbVq2pW0AX0lzSjYJtRqKoC/SppZ8NluEbEUIL32T+WDgdcK6i5KZYPTfu3y7epERCWwBuhT7Nx8kdDMylYEbK0ueZy5smDqoi5HRMQSSf2BhyT9s8ixdY18o0h5sTr18gjazMpWNsXRrqStwbYilqTX14G7gcOA5WnagvT6ejp8ETC0oPoQYEkqH1JH+XZ1JLUHegGri/XJAdrMylpVysfR0FaMpG6SetTsA8cCzwGTgXHpsHHAvWl/MjA2rcwYRnYxcHqaBlknaXSaXz6tVp2atk4GHk7z1PXyFEeZWr+mgonnD+WVf3ZGgm/9fCFPP9yTqQ/2QoJd+m7l/F8spM+ASpa91pGvf2Rfhuy1GYB9D9nAeZdl02T/+YW9WP16B6oqYf8PbODsHy6iogKWL+rAz7+1O2tWtafHLlVccMWr9Bu0tSVPuc0ZddRazvjBEiraBff/oTe3X7lbS3cpd2qW2TWB3YC70zW79sAtEfGApKeB2yWNBxYCpwBExPOSbgfmApXAWRFRldo6E7gB6ALcnzaA64CbJC0gGzmPbahTZRWgJVUBc8j6/QIwLiI21nPsp4EREXFpHZ+tj4juzdrZZnb19wYz6qi1/N/fvsLWLWLzW+3YY5/XGXfBMgDuubYvv584YFsgHrjHZq7+27x3tPPd37xCtx7VRMAPvr4nj/1pF4466U1++/3BfOzk1Xz8399g9uPd+d2PBnLBFQt36jm2Ze3aBWf9cDEXjd2LlUs7cMV983nqwV4snN+5pbuWM01zq3dEvAQcWEf5KuCYeupcAlxSR/kMYP86yjeRAnypym2K4620RnF/YAtwRn0HRsTkuoJza7BhXTvmPNWNMV/Ipq86dAy696qiW4/qbcdseqsdxRfwZGrqVFVC5RZtu4zx6oudGPmh9QAceMR6pj7Yq2lPwora56CNLHmlI8sWdqJyazsevXcXDj9uTUt3K5eq03MJG9rKUbkF6EKPAXtLOiHdlfOMpL9J2g1A0pclXZn2h0maKulpST9o0V43gWWvdqJXn0p+9s3d+cbH38vEbw9l08bsV/m7Swdw6iEjePiuXTntO0vfrrOwI9/4+Hs5/7N7M2dat+3a+8/P78XnDtifLt2r+fDxbwKw14hNPH5fFpSfuL8XG9dXsHZ16815kDd9BmxlxZKO296vXNqBvgM9xVRbtoqjoqStHJVlgE5XQD9BNt3xODA6Ig4CbgUuqKPKL4GrI+JQYFmRdifUrJFcsaqqvsNaXFUVLJjTleNPW8lVD71I567V3HZltjzzKxcu4+aZc/noZ99g8vX9AOjdfyu/f3ouVz30IqdfvJhLv7EHG9a9/av/4R9e4g/PPM/WLWL249nMz4TvLWbO1O584+PvZc7U7vQduIWK9kWvZ1gTqutfP8UvJ7VNO+FGlRZVbgG6i6TZwAyyCfvryJaxPChpDvAdslsvazsC+EPav6m+xiPimogYFRGj+vXJ71/cvgO30m/gVvY9OJt+/9Dxb7JgTpftjjn6M29sGwF37BT07J39wRl+wFsM2nMLi1/qtN3xHTsHhx+7ZttURp8BlXzvule46qEX+fKF2Ui8W89qbOdYubQD/QZt2fa+78CtrFrWoQV7lF+e4siPmjnokRFxTkRsAa4AroyI9wOnA/VdRWk144/e/SvpO2gLry3Iguzsx3qw+/DNLH7p7X8SP/VgL4buna3aeHNVBVXpHwRLX+3I4pc7MmD3Lby1oR2rlmfXiasqYfqUntvqrFlVQXWKx7de0Z9jP1d0uaY1sXmzuzJ42BZ2G7qZ9h2qOerEN3nqr74OUFtTJkvKo7JaxVGPXsDitD+unmOeIFvS8nvg1J3RqeZ21v9bzGVn70HlVjFg9y18e+JCJp4/lEX/6kS7dtB/8BbOTSs45jzVnRt/MoCK9lDRLjj30kX03LWKN1a05+Iv78XWLaKqCkYesZ7jT1sJwLNTu3P9jwYhBe//wAbO+uGiYt2xJlZdJX713cH88JaXaFcBf721N6++6BUcdWnNCfvVwDrpXKlreZykE8kyQy0GngIOjYijJH0ZGBURZ6eF5LeQ/UG6E/ivhpbZjTqwc0x/cGixQyxnjhs0sqW7YI0wLaawNla/q6Htrvv2j49ef3JJx951xNUzG7jVO3fKagRdV1CNiHt5+06dwvIbyBaLk7JNHV7wcatcfmfWFpXr9EUpyipAm5kVasI7CXPJAdrMypoDtJlZDtWsg26tHKDNrKyV6xrnUjhAm1nZioDK0hP2lx0HaDMra57iMDPLIc9Bm5nlWDhAm5nlky8SmpnlUITnoM3MckpUeRWHmVk+eQ7azCyHnIvDzCyvonU/CswB2szKmldxmJnlUPgioZlZfrXmKY7W+6fHzNqECJW0lUJShaRnJP05ve8t6SFJ89PrrgXHXiRpgaR5ko4rKD9E0pz02eWSlMo7SbotlU+TtGdD/XGANrOyFdG0ARo4D3ih4P2FwJSIGA5MSe+RNILsQdT7AWOAqyRVpDpXAxOA4Wkbk8rHA29ExN5kz1G9rKHOOECbWVmrDpW0NUTSEOBTwLUFxScCk9L+JOCkgvJbI2JzeubpAuAwSQOBnhExNbInct9Yq05NW3cAx9SMruvjAG1mZS2itA3oK2lGwTahVlO/AC4AqgvKdouIpdn3xFKgfyofDLxWcNyiVDY47dcu365ORFQCa4A+xc7NFwnNrGwForr0VRwrI2JUXR9IOh54PSJmSjqqhLbqGvlGkfJiderlAG1mZa2JFnEcAXxa0ieBzkBPSb8HlksaGBFL0/TF6+n4RcDQgvpDgCWpfEgd5YV1FklqD/QCVhfrlKc4zKx8NdFFwoi4KCKGRMSeZBf/Ho6ILwKTgXHpsHHAvWl/MjA2rcwYRnYxcHqaBlknaXSaXz6tVp2atk5O3+ERtJm1Ys27DvpS4HZJ44GFwCkAEfG8pNuBuUAlcFZEVKU6ZwI3AF2A+9MGcB1wk6QFZCPnsQ19uQO0mZW1ps5mFxGPAo+m/VXAMfUcdwlwSR3lM4D96yjfRArwpao3QEu6giJ/myLi3MZ8kZlZUwugurpt5uKYsdN6YWa2IwJoi+lGI2JS4XtJ3SJiQ/N3ycysdG06F4ekwyXNJd3+KOlASVc1e8/MzEoRJW5lqJRldr8AjgNWAUTEP4Ajm7NTZmalKW2JXbk+FqukVRwR8VqtW8ar6jvWzGynKtPRcSlKCdCvSfogEJI6AueyfbYnM7OWERCteBVHKVMcZwBnkSX6WAyMTO/NzHJAJW7lp8ERdESsBE7dCX0xM2u8VjzFUcoqjr0k/UnSCkmvS7pX0l47o3NmZg1q46s4bgFuBwYCg4A/An9ozk6ZmZWk5kaVUrYyVEqAVkTcFBGVafs9Zfv3yMxam0Yk7C87xXJx9E67j0i6ELiVLDB/DvjLTuibmVnDWvEqjmIXCWey/RMCTi/4LIAfNFenzMxKpTIdHZeiWC6OYTuzI2ZmjVbGFwBLUdKdhJL2B0aQPQoGgIi4sbk6ZWZWmvK9AFiKBgO0pP8GjiIL0PcBnwAeJ3ucuJlZy2rFI+hSVnGcTPZEgWUR8RXgQKBTs/bKzKxU1SVuZaiUKY63IqJaUqWknmRPtfWNKmbW8tpqwv4CMyTtAvyWbGXHemB6s/bKzKxEbXIVR42I+Eba/bWkB4CeEfFs83bLzKxEbTFASzq42GcRMat5umRmZlB8BP2zIp8F8NEm7kuuzJ/fm0988gst3Q1rhIq932rpLlgjaGHHpmmnLY6gI+LondkRM7NGC9rsrd5mZvnXikfQpayDNjPLLUVpW9E2pM6Spkv6h6TnJf1PKu8t6SFJ89PrrgV1LpK0QNI8SccVlB8iaU767HKlB7pK6iTptlQ+TdKeDZ2bA7SZlbemSdi/GfhoRBxI9li/MZJGAxcCUyJiODAlvUfSCGAssB8wBrhKUkVq62pgAjA8bWNS+XjgjYjYG5gIXNZQp0p5oookfVHS99L73SUd1uDpmpntDE0QoCOzPr3tkLYATgQmpfJJwElp/0Tg1ojYHBEvAwuAwyQNJFuKPDUigiwlRmGdmrbuAI6pGV3Xp5QR9FXA4cDn0/t1wK9KqGdm1qxKnd5IUxx9Jc0o2CZs15ZUIWk22d3SD0XENGC3iFgKkF77p8MHA68VVF+Uygan/drl29WJiEpgDdCn2PmVcpHwAxFxsKRnUsNvSGqa9TFmZu9W6as4VkbEqPo+jIgqYGS6c/rulMWzPnV9aRQpL1anXqWMoLemuZUAkNSPsk09YmatTVNcJCwUEW8Cj5LNHS9P0xak19fTYYuAoQXVhgBLUvmQOsq3qyOpPdALWF2sL6UE6MuBu4H+ki4hSzX6wxLqmZk1vyaYg5bUL42ckdQF+BjwT2AyMC4dNg64N+1PBsamlRnDyC4GTk/TIOskjU7zy6fVqlPT1snAw2meul6l5OK4WdJMspSjAk6KiBcaqmdm1uwaOTouYiAwKc0WtANuj4g/S5oK3C5pPLAQOAUgIp6XdDswF6gEzkpTJABnAjcAXYD70wZwHXCTpAVkI+exDXWqlIT9uwMbgT8VlkXEwgZP2cysuTVBgE4J4A6qo3wV2eC0rjqXAJfUUT4DeMf8dURsIgX4UpVykfAvvD353RkYBswjW/9nZtai1IqviJUyxfH+wvcpy93p9RxuZmZNpNG5OCJilqRDm6MzZmaN1opzcZQyB/2tgrftgIOBFc3WIzOzUjXdRcJcKmUE3aNgv5JsTvrO5umOmVkjtdUAnZacdI+I7+yk/piZNU5bDNCS2kdEZbFHX5mZtSTRdldxTCebb54taTLwR2BDzYcRcVcz983MrDjPQdMbWEX2DMKa9dABOECbWctrowG6f1rB8RzvzNLUin8kZlZWWnE0KhagK4Du7ECKPDOznaWtTnEsjYjv77SemJntiDYaoFvvs8zNrHWItruKo84MTmZmudIWR9ARUTTTv5lZHrTVOWgzs/xzgDYzy6ESHmdVzhygzaxsCU9xmJnllgO0mVleOUCbmeWUA7SZWQ45m52ZWY45QJuZ5VNbvdXbzCz3WvMUR7uW7oCZ2Q6LRmxFSBoq6RFJL0h6XtJ5qby3pIckzU+vuxbUuUjSAknzJB1XUH6IpDnps8slKZV3knRbKp8mac+GTs8B2szKWxMEaKAS+HZEvA8YDZwlaQRwITAlIoYDU9J70mdjgf2AMcBV6SHbAFcDE4DhaRuTyscDb0TE3sBE4LKGOuUAbWZlq+ZOwlK2YiJiaUTMSvvrgBeAwcCJwKR02CTgpLR/InBrRGyOiJeBBcBhkgYCPSNiakQEcGOtOjVt3QEcUzO6ro/noM2srKm65EnovpJmFLy/JiKueUd72dTDQcA0YLeIWApZEJfUPx02GHiqoNqiVLY17dcur6nzWmqrUtIaoA+wsr4OO0CbWflqXLKklRExqtgBkroDdwL/ERFriwxw63sUYLFHBDb68YGe4jCzstYUUxwAkjqQBeebI+KuVLw8TVuQXl9P5YuAoQXVhwBLUvmQOsq3qyOpPdALKJp33wHazMpb06ziEHAd8EJE/Lzgo8nAuLQ/Dri3oHxsWpkxjOxi4PQ0HbJO0ujU5mm16tS0dTLwcJqnrpenOMysrDXROugjgC8BcyTNTmX/CVwK3C5pPLAQOAUgIp6XdDswl2wFyFkRUZXqnQncAHQB7k8bZH8AbpK0gGzkPLahTjlAm1l5a4IAHRGPU/+Dsut8PmtEXAJcUkf5DGD/Oso3kQJ8qRygzax8teGnepuZ5ZqfqGJmlmfFr7OVNQdoMytrHkFbrp100j8Zc9y/iBCvvNKLn08czZAhaznn7Kfp3KWS15d348c//iAb3+qwrU6/fhv4za/v4+ab9+fOu94HwJFHvsrYzz1Pu3bB9KcHcf31B7XUKbV63bpv4bwLZrPHsLUE8ItLD+aQw5Zz3PGvsubNjgBM+u0IZjw1gP4DNvCbm6awaGF3AObN7c2VPxsJQPv21Zz5H//ggINWUl0tbrz2fTzxv4Pr+9rWx0/13jGSAvh5RHw7vT8f6B4RFxepcxLwYkTMreOzi4GvAyvI+v2fETG5SFv3AV+IiDfraGd9RPy0seeUR336bOTET7/I6Wd8ki1b2nPRRY/zkY+8ygnHz+faaw9iznP9Ofbj/+LfTn6Bm246YFu9CRNmMWPGwG3ve/TYzPivzubcc49jzdrOfPtbUxl54DJm/2NAS5xWq3f6uXOYOa0/P/zeYbRvX02nzpUccthy7vnje7jr1uHvOH7p4m6cM/6j7yj/3JfmsebNTnz91I8jBT16btkZ3c+V1nyRsDlvVNkMfFZS30bUOQkYUeTziRExkmypyvWS6u1/RHyydnBurSoqgo4dq2jXrppOnapYvaoLQ4asZc5z/QCY9cwAPnTEa9uOP/zwRSxb2p1XF/baVjZwwHoWL+7BmrWdAXhm9gCOKKhjTadL163sf+AqHvzLHgBUVrZjw/qOO9TWsZ9ayG2/fy8AEWLtmk5N1s9yoerStnLUnAG6ErgG+GbtDyTtIWmKpGfT6+6SPgh8GviJpNmS3lNfwxHxQmq/r6R7JM1MOVwnFHzHKzV/HCR9N+Vs/RuwTxOfZ4tataord961LzdOmswtN9/Dxg0dmPXMQF55ZRdGj14MwIc//Bp9+24EoFOnSk45eS4337L9Ms0lS3swdOha+vdfT7t21Rx++CL69du408+nLRg4aANr3uzINy+axRXXPsJ5FzxDp86VAJzwmZf41e8e5j/+zyy6d397NDxg4EauuPYRLrv8MfY7IMut0y19ftr4F7j82ke46H+ms8uum3b+CbWkILtIWMpWhpr7Vu9fAadK6lWr/Ergxog4ALgZuDwiniS7FfI7ETEyIv5VX6OSPgBUk013fDUiDgFGAedK6lPr2EPI7tg5CPgscGiRdidImiFpxpbK8ghO3btvYfToRXzlKydw6hdPolPnSo4++mUm/uIDnHD8fC7/5QN06bKVysrsV/2lL87h7nv2ZdOmDtu1s359R6688lAuuuhJfvqTv7F8eTeqqpwJoDlUVAR7D1/DffcM45yvHc2mTRX8+6kv8pd7hjH+88dy9lePZvWqznztrOcAWL2qM+NOOY5zvnY0v73y/VzwvZl06bqVioqgX/+3mPtcb8792tH88/nefO0bz7Xw2e18TZWLI4+a9SJhygZ1I3Au8FbBR4eTBUuAm4Afl9jkNyV9EVgHfC4iQtK5kj6TPh9Kdk/8qoI6HwbujoiNAJLqnbdOqQevAejVdVBZ/EpHjlzG8mXdt01NPPnEUEa8byWPPDKM7/7X0QAMHryWww7N8rXss88qPvSh1xj/1dl067aFCLFlSwV/+vN7mTZ9MNOmZxeYPjFmAdXVRVPV2g5auaILK1d0Zt4LvQF4/NFBnHLqfN58o/O2Yx748x5cfGmWzbJyawXrtma54Be8uAtLF3dlyND1zJ+3C5vequDJvw8C4LFHB3Hsp17dyWeTA2Xxf+qO2RmrOH4BzAJ+V+SYUn/EEwsv7kk6CvgYcHhEbJT0KNC5jnqt9le4YkVX9t13JZ06VbJ5cwUjRy5j/vw+9Oq1iTVrOiMFY8c+z3337Q3Ady742La6p546h01vtedPf87mMGvqdO++hU99aj4/uvSIFjmn1u6N1Z1Z8XpXBg9dx+LXejDykBUsfKUHu/bZxBursv98P/jhpbz6ck8AevbazPp1HamuFgMGbmDQkA0sXdINENOeHMABB63kH7P6MfLgrJ22xDeqvEsRsTolFRkPXJ+KnySbdrgJOBV4PJWvAxrzX1gvskfIbJS0L9mjamr7O3CDpEvJzvcE4DeNPpGcmjevL48/vjtXXP4AVVXt+NdLu3L//e/hk59awPHHzwfgySeG8NeH9mqwrTNOn8lee2XXVW+5ZX8WL+7ZrH1vy379y/dzwf+dSfsO1Sxb0pWJPzqYM857lr2GryUCli/ryhU/zZbSvX/kSr741X9SVSWqq8WVPzuQ9euyi4q/+/V+nP9fM5lwzhzWvNmRiT86uCVPa+eLaEzC/rKjBrLd7XjD0vqI6J72dwNeBn4cERenJxZcD/Qlm0f+SkQslHQE8FuyFSAnF85D17U8TlIn4B6yJxXMA/oBF0fEo5JeAUZFxEpJ3yVL+/cqWU7WuQ0ts+vVdVCM3vfr7/4HYTuN1r/V8EGWG1MX3siaTcve1Txaj12GxEFHnlfSsY/96YKZDSXsz5tmG0HXBOe0vxzoWvD+FeAdizoj4gnqWWZX1/rpiNgMfKKe4/cs2K8z65SZlT9PcZiZ5VEArXiKwwHazMpb643PDtBmVt48xWFmllOteRWHA7SZlS9nszMzy6fsRpXWG6EdoM2svJVpprpSOECbWVnzCNrMLI88B21mlletOxeHA7SZlbdWPMXhjOxmVr6i6R55Jel6Sa9Leq6grLekhyTNT6+7Fnx2kaQF6WlNxxWUHyJpTvrscklK5Z0k3ZbKp6WkcUU5QJtZeWu6R17dAIypVXYhMCUihgNT0nskjSBLmbxfqnOVpIpU52pgAtnDQ4YXtDmeLD3y3sBE4LKGOuQAbWblLUrcGmom4u/A6lrFJwKT0v4ksgdb15TfGhGbI+JlYAFwmKSBQM+ImBpZLucba9WpaesO4Jia0XV9PAdtZmVN1c26EHq3iFgKEBFLJfVP5YOBpwqOW5TKtqb92uU1dV5LbVVKWgP0AVbW9+UO0GZWvoLG3KjSV9KMgvfXpOeQ7oi6Rr5RpLxYnXo5QJtZ2RLRmBtVVu7AE1WWSxqYRs8DgddT+SKyh1TXGAIsSeVD6igvrLNIUnuyR/bVnlLZjuegzay8Nd1FwrpMBsal/XHAvQXlY9PKjGFkFwOnp+mQdZJGp/nl02rVqWnrZODhaOCZgx5Bm1l5a6J10JL+ABxFNhWyCPhv4FLgdknjgYXAKdlXxvPpYdhzgUrgrIioSk2dSbYipAtwf9oArgNukrSAbOQ8tqE+OUCbWflq3Bx08aYiPl/PR8fUc3ydzzqNiBnA/nWUbyIF+FI5QJtZWWvmVRwtygHazMrYu5pfzj0HaDMrX4EDtJlZbrXeGQ4HaDMrb07Yb2aWVw7QZmY5FAFVrXeOwwHazMqbR9BmZjnlAG1mlkMB+JmEZmZ5FBCegzYzy5/AFwnNzHLLc9BmZjnlAG1mlkdOlmRmlk8BON2omVlOeQRtZpZHvtXbzCyfAsLroM3Mcsp3EpqZ5bJFEK0AAAa0SURBVJTnoM3McijCqzjMzHLLI2gzszwKoqqqpTvRbBygzax8Od2omVmOteJldu1augNmZjsqgKiOkraGSBojaZ6kBZIubP7eN8wB2szKV6SE/aVsRUiqAH4FfAIYAXxe0oidcAZFeYrDzMpaE10kPAxYEBEvAUi6FTgRmNsUje8oRSteovJuSFoBvNrS/WgGfYGVLd0Ja5TW+jvbIyL6vZsGJD1A9vMpRWdgU8H7ayLimtTOycCYiPhaev8l4AMRcfa76d+75RF0Pd7tfzh5JWlGRIxq6X5Y6fw7q19EjGmiplRX803U9g7zHLSZGSwChha8HwIsaaG+bOMAbWYGTwPDJQ2T1BEYC0xu4T55iqMNuqalO2CN5t9ZM4uISklnAw8CFcD1EfF8C3fLFwnNzPLKUxxmZjnlAG1mllMO0K2EpCpJsyU9J+mPkroWOfbT9d3KKml98/WybZAUkn5W8P58SRc3UOek+u5ck3SxpMUFv99PN9DWfZJ2qaed80s8DcsBB+jW462IGBkR+wNbgDPqOzAiJkfEpTuva23OZuCzkkq9gQLgJLJbjOszMSJGAqcA10uq9//diPhkRLzZiO+2nHKAbp0eA/aWdIKkaZKekfQ3SbsBSPqypCvT/jBJUyU9LekHLdrr1qOSbOXFN2t/IGkPSVMkPZted5f0QeDTwE/SKPk99TUcES+k9vtKukfSTEnPS5pQ8B2v1PxxkPTdlADob8A+TXye1swcoFsZSe3JEr7MAR4HRkfEQcCtwAV1VPklcHVEHAos22kdbf1+BZwqqVet8iuBGyPiAOBm4PKIeJJsze130r+C/lVfo5I+AFQDK4CvRsQhwCjgXEl9ah17CNl63oOAzwKHNs2p2c7iddCtRxdJs9P+Y8B1ZCOm2yQNBDoCL9dR7wjg39L+TcBlzd3RtiAi1kq6ETgXeKvgo8PJgiVkP+8fl9jkNyV9EVgHfC4iQtK5kj6TPh8KDAdWFdT5MHB3RGwEkNTiN15Y4zhAtx5vpTnKbSRdAfw8IiZLOgq4uJ66XgzfPH4BzAJ+V+SYUn/2EyPipzVv0u/zY8DhEbFR0qNkyYB2tH3LIU9xtG69gMVpf1w9xzxB9s9ggFObvUdtSESsBm4HxhcUP8n2P+/H0/46oEcjmu8FvJGC877A6DqO+TvwGUldJPUATmhM/63lOUC3bhcDf5T0GPWnqzwPOEvS02T/01vT+hnbp8M8F/iKpGeBL5H9/CG7RvCddEG33ouEBR4A2qd2fgA8VfuAiJgF3AbMBu4km/qyMuJbvc3McsojaDOznHKANjPLKQdoM7OccoA2M8spB2gzs5xygLYd0pjseSW0dUN6qjKSrq0vq1v6/KiUu6Kx37EtP0Up5bWOaVSGP2eNs6biAG07qmj2PEkVO9JoRHwtIuYWOeQooNEB2qwcOUBbU6jJnneUpEck3QLMkVQh6ScpU96zkk4HUOZKSXMl/QXoX9OQpEcljUr7YyTNkvSPlPltT7I/BN9Mo/cPS+on6c70HU9LOiLV7SPpr+nGj98Aaugk6ssOlz77WerLFEn9Utl7JD2Q6jyW7ugzazLOxWHvSkH2vAdS0WHA/hHxcgpyayLiUEmdgCck/ZUsu9o+wPuB3YC5wPW12u0H/BY4MrXVOyJWS/o1sL4mL0X6YzAxIh6XtDvZQz/fB/w38HhEfF/Sp4DtAm49vpq+owvwtKQ7I2IV0A2YFRHflvS91PbZZClFz4iI+SnL3FXAR3fgx2hWJwdo21F1Zc/7IDA9Imqy5h0LHFAzv0x2K/lw4EjgDxFRBSyR9HAd7Y8G/l7TVsprUZePASOkbQPkninvxJGkrHER8RdJb5RwTvVlh6smu2Ua4PfAXZK6p/P9Y8F3dyrhO8xK5gBtO6qu7HkAGwqLgHMi4sFax32ShrOsqYRjIJumOzwiClN61vSl5DwGjcgOV9NuO+DN2j8Ds6bkOWhrTg8CZ0rqACDpvZK6kWVZG5vmqAcCR9dRdyrwEUnDUt3eqbx21re/kk03kI6rCZh/J2Xnk/QJYNcG+losO1w7oOZfAV8gmzpZC7ws6ZT0HZJ0YAPfYdYoDtDWnK4lm1+eJek54Ddk/2q7G5hP9tSXq4H/rV0xIlaQzRvfJekfvD3F8CeyFJqzJX2YLDvcqHQRci5vryb5H+BISbPIploWNtDXYtnhNgD7SZpJNsf8/VR+KjA+9e954MQSfiZmJXM2OzOznPII2swspxygzcxyygHazCynHKDNzHLKAdrMLKccoM3McsoB2swsp/4/IVSoGBxnbuAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_confusion_matrix(grid_model, X_test, y_test, values_format='d', display_labels=[\"Paid\",\"Not Paid\"]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'subsample': 0.5,\n",
       " 'scale_pos_weight': 3,\n",
       " 'reg_lambda': 12.0,\n",
       " 'n_estimators': 800,\n",
       " 'max_depth': 3,\n",
       " 'learning_rate': 0.0001,\n",
       " 'gamma': 0,\n",
       " 'colsample_bytree': 0.8,\n",
       " 'colsample_bylevel': 0.8}"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_model.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_predictions = opyimal_params.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      1.00      0.93     47655\n",
      "           1       0.98      0.44      0.60     11628\n",
      "\n",
      "    accuracy                           0.89     59283\n",
      "   macro avg       0.93      0.72      0.77     59283\n",
      "weighted avg       0.90      0.89      0.87     59283\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, grid_predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWgAAAEGCAYAAABIGw//AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5xVVd3H8c+XAQbkJjcVuYgKaGp5Q0NNw0uJPY+XfDQpTUzKa2mWlmY9Wj6Ul5TyMS1LE++XvGFPYoqaogIC4g1TLBVRkPv9OjO/54+9Bw/jzJkzMsPsc+b7fr32a/ZZe6911j4Dv7Nm7bXXUkRgZmbZ06q5K2BmZrVzgDYzyygHaDOzjHKANjPLKAdoM7OMat3cFciqHt3Kon/fNs1dDWuAt17ZormrYA2whpWsi7XalDIOP7hDLFxUWdC5U19Z+1hEDNuU99vcHKDr0L9vGyY/1re5q2ENcPi2ezR3FawBJsX4TS5j4aJKJj/Wr6Bzy3rN7LHJb7iZOUCbWdEKoIqq5q5Gk3GANrOiFQTro7AujmLkAG1mRc0taDOzDAqCyhKersIB2syKWhUO0GZmmRNApQO0mVk2uQVtZpZBAax3H7SZWfYE4S4OM7NMCqgs3fjsAG1mxSt5krB0OUCbWRETlWzSfEuZ5gBtZkUruUnoAG1mljnJOGgHaDOzTKpyC9rMLHvcgjYzy6hAVJbwyn0O0GZW1NzFYWaWQYFYF2XNXY0m4wBtZkUreVDFXRxmZpnkm4RmZhkUISrDLWgzs0yqcgvazCx7kpuEpRvGSvfKzKzk+SahmVmGVXoctJlZ9vhJQjOzDKvyKA4zs+xJJktygDYzy5xArC/hR71L96vHzEpeBFRGq4K2Qkgqk/SSpL+mr7tJelzSzPRn15xzL5L0tqQ3JR2ek763pFfTY9dKUppeLumeNH2SpP711ccB2syKmKgqcCvQucAbOa8vBMZHxEBgfPoaSbsAw4FdgWHA9ZKqm/I3AKcBA9NtWJo+ElgcEQOA0cAV9VXGAdrMilbQeC1oSX2A/wD+lJN8NDAm3R8DHJOTfndErI2Id4C3gX0l9QI6R8QLERHArTXyVJf1F+DQ6tZ1XdwHbWZFrQE3CXtImpLz+saIuDHn9W+AHwGdctK2jog5ABExR9JWaXpvYGLOebPTtPXpfs306jzvp2VVSFoKdAcW1FVhB2gzK1qBGjJh/4KIGFzbAUn/CcyLiKmShhZQVm1vGnnS8+WpkwO0mRWtANY3zlwcBwBHSfoK0A7oLOl24CNJvdLWcy9gXnr+bKBvTv4+wIdpep9a0nPzzJbUGugCLMpXKfdBm1kRE5UFbvlExEUR0Sci+pPc/HsyIk4CxgIj0tNGAA+n+2OB4enIjO1JbgZOTrtDlksakvYvn1wjT3VZx6Xv4Ra0mZWmoMmfJLwcuFfSSGAWcDxARLwu6V5gBlABnB0RlWmeM4FbgPbAo+kGcBNwm6S3SVrOw+t7cwdoMytqjb2iSkQ8DTyd7i8EDq3jvFHAqFrSpwC71ZK+hjTAF8oB2syKVoQ8F4eZWRYlNwlL91FvB2gzK2Jek9DMLJOSm4SesN/MLJM83aiZWQY18EnCouMAbWZFzYvGmpllUASsr3KANjPLnKSLwwHazCyTGvtJwixxgC5SlZXwvWGD6N5rPZfd+g6jTt+O2f9qB8DKZWV06FzJDU+8ydz32/KdL+5Mnx3WArDz3is594rZrFklRp3enw/fLadVWTDkS8sYefEcAObNbsNV3+/HyqVlVFWJU3/yIfseurzZrrWU/eCaWXz+sOUsWdCa0w/ZCYBv/+xDhnxpGevXiTnvteXq8/qxclkZB391McefNW9D3u0/s4azDx/Ev19v31zVb3YeZpchkiqBV0nq/QYwIiJW1XHuUcAuEXF5LcdWRETHJq1sE3voTz3pO3Atq1Ykf95d/If3Nhz7w8+3pUOnyg2ve223lhueePMTZfzXGfPZ44AVrF8nfvy1HXnxyU7sc8hy7vzt1hx05BKOHLGQ994q52cn7citk2c0/UW1QH+/pxtj/9yDC377/oa0ac904uZf9qKqUoy8+EOGf+8jbhq1LU892JWnHkyWxOu/82ou/fO7LTo4J0q7i6PYrmx1ROwREbsB64Az6joxIsbWFpxLwfwP2zB5fGeO+MbCTxyLgGfGbsnBxyzOW0a7LYI9DlgBQJu2wcDPrmb+nDYASLBqefL47MplZXTben0jX4FVe21SR5Yv3ridNO0fnaiqTFqFb0ztQI9en/z8Dz5mCU8/tOVmqWPWNfKahJlSbAE617PAAElHpivkviTpCUlbA0g6RdJ16f72kl6Q9KKky5q11o3g95f05ts//RDV8tt7bVIHuvasoPcO6zakzZ3VlrO+NIjzjx3Aq5M6fCLPiqVlTHy8M3t+IQnYJ/1wLk8+0JUT996Fn31zB84eNfsTeWzzOPzri3jxyc6fSD/oqCU85QCdjuIoK2grRkUZoNPVCI4g6e6YAAyJiD2Bu0nWFKvpt8ANEbEPMDdPuadJmiJpyvyFlXWd1qwmPt6ZLXtUMPBzq2s9/tRDXRma03ruttV6bn9xBtc//hanX/oBl5+1HSuXf/xrr6yAX521HUePXECv7ZKg/vRDXfnS1xZxx9QZXHbbv7nye9tRVdW012Wf9PVzPqKyAp58YONAvNOeK1m7uhXvvdnSuzc+flClkK0YFVuAbi9pOjCFZPLsm0iWlHlM0qvABSTLoNd0AHBXun9bXYVHxI0RMTgiBvfsns1v3BkvdmDi3ztz8r678Kszt+PlCZ244rv9gCTYPve3LnzxqCUbzm9bHnTulnzZDPzcarbtv44P/l2+4fhvLuhL7+3Xcux35m9IG3dXNw46Miljl8GrWLdWLFtUVLcrit5hxy9i38OWccV3t6PmUnZDj3b3Rq5S7uIotv91qyNij9wESf8LXBMRY9PFHi+tI2/epWWKxak/mcOpP0lGW7z8fEf+8vue/Pi6WQBMe7YTfQespee2H/dZLllYRqctKykrgznvteWDd9qyTb+kpXzLFduwcnkZ5139/kbvsVXv9Uyf0Ikvn7CIWTPLWbe2FV26V2ymK7TBQ5fxtbPnccGxA1i7euM2lBQc+J9LOf/YHZupdtniURzZ1wX4IN0fUcc5z5EsL3M7cOLmqFRz+MfDG3dvALw6sSO3XrUNZa2hrFVwzuWz6dy1kvkftuGu325D3wFrOPvLyfCuo741nyNOXMRpl3zAb87vywN/7ImA80fPQqX7f6BZXXj9e3xuvxV06VbB7VNmcNvVWzP8u/NoUx786p5/AfDPqR249sJkHdLPDlnJgjltmDurPF+xLUopj+JQPWsWZkptw+MkHQ2MJgnSE4F9ImKopFOAwRHx3XRRxztJvpDuB35a3zC7wbu3i8mP9c13imXM4dvuUf9JlhmTYjzLYtEmffV33XmrOOTm4wo694EDbpgaEYM35f02t6JqQdcWVCPiYT5eNTc3/RaShRuJiHeA/XIOl+TwO7OWyF0cZmYZ5D5oM7MMc4A2M8sgT9hvZpZhxTrGuRAO0GZWtCKgwhP2m5llk7s4zMwyyH3QZmYZFg7QZmbZ5JuEZmYZFOE+aDOzjBKVHsVhZpZN7oM2M8sgz8VhZpZVkfRDlyoHaDMrah7FYWaWQeGbhGZm2VXKXRyl+9VjZi1ChAra8pHUTtJkSS9Lel3Sz9P0bpIelzQz/dk1J89Fkt6W9Kakw3PS95b0anrsWilZ0VNSuaR70vRJkvrXd20O0GZWtCIaJ0ADa4FDImJ3YA9gmKQhwIXA+IgYCIxPXyNpF5KFqHcFhgHXSypLy7oBOA0YmG7D0vSRwOKIGECyjuoV9VXKAdrMilpVqKAtn0isSF+2SbcAjgbGpOljgGPS/aOBuyNibbrm6dvAvpJ6AZ0j4oVIVuS+tUae6rL+Ahxa3bquiwO0mRW1iMI2oIekKTnbabnlSCqTNB2YBzweEZOArSNiTvI+MQfYKj29N/B+TvbZaVrvdL9m+kZ5IqICWAp0z3dtvkloZkUrEFWFj+JYEBGD6ywrohLYQ9KWwIOSdstTVm0t38iTni9PndyCNrOiFgVuBZcXsQR4mqTv+KO024L057z0tNlA35xsfYAP0/Q+taRvlEdSa6ALsChfXRygzax4NdJNQkk905YzktoDhwH/BMYCI9LTRgAPp/tjgeHpyIztSW4GTk67QZZLGpL2L59cI091WccBT6b91HVyF4eZFbfGGQfdCxiTjsRoBdwbEX+V9AJwr6SRwCzgeICIeF3SvcAMoAI4O+0iATgTuAVoDzyabgA3AbdJepuk5Ty8vko5QJtZUWuM2ewi4hVgz1rSFwKH1pFnFDCqlvQpwCf6ryNiDWmAL1SdAVrS/5LnuykizmnIG5mZNbYAqqpa5lwcUzZbLczMPo0AWuJ0oxExJve1pA4RsbLpq2RmVrgWPReHpP0kzQDeSF/vLun6Jq+ZmVkhGnucXYYUMszuN8DhwEKAiHgZOKgpK2VmVpjChtgV67JYBY3iiIj3azwyXlnXuWZmm1WRto4LUUiAfl/S/kBIagucQ9rdYWbWrAKihEdxFNLFcQZwNslEHx+QTMV3dlNWysyscCpwKz71tqAjYgFw4maoi5lZw5VwF0chozh2kPSIpPmS5kl6WNIOm6NyZmb1auGjOO4E7iV5Vn1b4D7grqaslJlZQaofVClkK0KFBGhFxG0RUZFut1O030dmVmoaMGF/0ck3F0e3dPcpSRcCd5ME5hOA/9sMdTMzq18Jj+LId5NwKhuvEHB6zrEALmuqSpmZFUpF2jouRL65OLbfnBUxM2uwIr4BWIiCniRM1+baBWhXnRYRtzZVpczMClO8NwALUW+AlnQJMJQkQP8NOAKYQLKcuJlZ8yrhFnQhoziOI1lRYG5EfAvYHShv0lqZmRWqqsCtCBXSxbE6IqokVUjqTLKqrR9UMbPm11In7M8xJV3t9o8kIztWAJObtFZmZgVqkaM4qkXEWenu7yWNAzqnCyyamTW/lhigJe2V71hETGuaKpmZGeRvQV+d51gAhzRyXTJl5sxuHDFseHNXwxqgbMDa5q6CNYBmtW2cclpiCzoiDt6cFTEza7CgxT7qbWaWfS2xBW1mVgxaZBeHmVlRKOEAXciKKpJ0kqT/Tl/3k7Rv01fNzKwALXxFleuB/YCvp6+XA79rshqZmRVIUfhWjArp4vh8ROwl6SWAiFgsqXHGx5iZbaoWPopjvaQy0j8SJPWkaKceMbNSU6yt40IU0sVxLfAgsJWkUSRTjf6ySWtlZlaoEu6DLmQujjskTSWZclTAMRHxRpPXzMysPkXcv1yIQibs7wesAh7JTYuIWU1ZMTOzgrTkAE2ygnf14rHtgO2BN4Fdm7BeZmYFUQnfESuki+Ozua/TWe5Or+N0MzNrJIXcJNxIOs3oPk1QFzOzhmuEm4SS+kp6StIbkl6XdG6a3k3S45Jmpj+75uS5SNLbkt6UdHhO+t6SXk2PXStJaXq5pHvS9EmS+td3aYX0Qf8g52UrYC9gfn35zMyaXOPdJKwAfhgR0yR1AqZKehw4BRgfEZdLuhC4EPixpF2A4SRdvdsCT0gaFBGVwA3AacBEkoW2hwGPAiOBxRExQNJw4ArghHyVKqQF3SlnKyfpkz66QZduZtZUGqEFHRFzqhchiYjlwBtAb5JYNyY9bQxwTLp/NHB3RKyNiHeAt4F9JfUiWXXqhYgI4NYaearL+gtwaHXrui55W9DpAyodI+KC/JdnZtZMGnkUR9r1sCcwCdg6IuZAEsQlbZWe1pukhVxtdpq2Pt2vmV6d5/20rApJS4HuwIK66pJvyavWaSF1Ln1lZtacRINGcfSQNCXn9Y0RceNG5UkdgfuB70fEsjwN3NoORJ70fHnqlK8FPZmkv3m6pLHAfcDKDaVGPJCvYDOzJtewPugFETG4roOS2pAE5zty4ttHknqlredewLw0fTbQNyd7H+DDNL1PLem5eWZLag10ARblq3AhfdDdgIUkaxD+J3Bk+tPMrPk1zigOATcBb0TENTmHxgIj0v0RwMM56cPTkRnbAwOByWl3yHJJQ9IyT66Rp7qs44An037qOuVrQW+VjuB4jU823Uv42R0zKyqNE40OAL4JvCppepr2E+By4F5JI4FZwPEAEfG6pHuBGSQjQM5OR3AAnAncArQnGb3xaJp+E3CbpLdJWs71rkqdL0CXAR35FP0mZmabS2MMs4uICdQe6yCZh6i2PKOAUbWkTwF2qyV9DWmAL1S+AD0nIn7RkMLMzDa7Em4u5gvQpTsLtpmVhmi5c3HU2qw3M8uUltiCjoi8wz/MzLKgRc8HbWaWaQ7QZmYZVMTLWRXCAdrMipZwF4eZWWY5QJuZZZUDtJlZRjlAm5llUOOtqJJJDtBmVtwcoM3MsqmlPuptZpZ57uIwM8siP6hiZpZhDtBmZtnjJwnNzDJMVaUboR2gzax4uQ/azCy73MVhZpZVDtBmZtnkFrSZWVY5QJuZZVALXtXbzCzTPA7azCzLonQjtAO0mRU1t6At0zp0WMf3v/8i2/VfSgSMHr0ve+89l2HD/s3SpeUAjLnls7z44rYb8vTsuZI/3DiOO27flfvv3xmAK658km7d1rB2bRkAF//kiyxd2m7zX1AL8Od7HmP16jZUVkJVZSvOPW0oHTut46JLX2SrXquYN2cLfnXJPqxY0ZaysirO/fFLDBi0lFZlVTw5rh/33jEIgF9c9Tzduq+hrCx4/ZXuXD96d6qq1MxXtxn5QZVPR1IA10TED9PX5wMdI+LSPHmOAd6KiBm1HLsU+A4wn6TeP4mIsXnK+hvwjYhYUks5KyLi1w29pqw644yXmDJ1G0aNOoDWrSspL69k773n8tCDgzYE35pOO306U6Zs84n0K68YwsyZ3Zq6ygZceO4BLEu/QAG+duJbTJ/Wk/vuGMTxJ77F8SfN5M+/35UDD/6ANm2qOOuUQygvr+D3t47n6fG9mTe3A7+6ZB9Wr2oDBBdfNpkvDP2AZ57s03wX1QxK+SZhqyYsey1wrKQeDchzDLBLnuOjI2IP4HjgZkl11j8ivlIzOJeiLbZYz26fnc9j43YAoKKijJUr2+bNs99+s5k7twPvvddlc1TRCjTkC3N5Ylw/AJ4Y14/9vjAHgAjRrl0FrcqqaFteRUVFK1atbAOQBmcoKwtaty7hSJWHqgrbilFTBugK4EbgvJoHJG0nabykV9Kf/STtDxwFXCVpuqQd6yo4It5Iy+8h6SFJUyW9Lum0nPd4t/rLQdLFkt6U9ASwUyNfZ7PaZpsVLF1azg9+OJnrrnuMc78/mfLyCgCOPGom198wjvPOm0zHjusAKC+v4Piv/ZM7bt+11vLO+8FkrvvdY3z9G69T0n87NrNA/M/Vz/PbPz7FsCPfBWDLrmtYvDDpUlq8sB1duq4FYMLT27JmTWvueHAcY+57jPvvHsiK5R9/CV/26+e5c+yjrF7VhglP997s19KsguQmYSFbEWrqPujfAa9IurJG+nXArRExRtKpwLURcYykscBfI+Iv+QqV9HmgiqS749SIWCSpPfCipPsjYmHOuXsDw4E9Sa53GjC1jnJPA04DaNem86e43M2vrCwYMGAxN1y/F2++2Z3Tz5jG1054g0fGDuSuO3chQpx88qt85zvTGT16X775zdd48IFBrFnT5hNlXXnFEBYu3IL27dfz058+x6GHvsv48ds3w1WVvvPPOpBFC9vTZcu1jLrmOWbP6ljnuTt9ZjFVVeKkrw6jY6f1XHXds0yf0pO5czoA8LPz96dN20p+9LMp7L7XfF6astXmuoxMKOWbhE3ZgiYilgG3AufUOLQfcGe6fxvwhQKLPE/SdODXwAkREcA5kl4GJgJ9gYE18hwIPBgRq9L61NlvHRE3RsTgiBjctnWHAqvUvBYsaM+CBe15883uAEx4ti8DBixmyZJ2VFW1IkI8Om5HBu2UfGfttPNCRn77ZW4Z8wjHHPMWJwx/gyOPnAnAwoVbALB6dRueeno7Bu20qHkuqgVYtLA9AEuXlPPCs70Y9JnFLFncjq7d1wDQtfsali5O+qeHfmk2UydtRWVlK5YuKWfGq90YuPPGvXfr15Ux8bleDEm7RVqUKHArQk0aoFO/AUYC+SJeoR/f6IjYIyIOjIhnJQ0FDgP2i4jdgZeA2oYdFOmvp36LF7dn/vwt6N1nGQB77PkRs2Z1pmu31RvO2X//2bz3btLffMH5h3LKiCM5ZcSRPPTQIO65+zM88shAWrWqonPn5E/qsrIqPr/vhxvyWOMqb1dB+/brN+zvuc983vt3ZyY+tw2HDZsFwGHDZjFxQnITd95H7dl9rwVAUN6ugp13Xcz773WkXfuKDQG9VVkV+wz5iPdndWqWa2ou1Q+qFLIVoyYfZpd2P9xLEqRvTpOfJ+l2uA04EZiQpi8HGvIvrAuwOCJWSdoZGFLLOc8At0i6nOR6jwT+0OALybAbrt+LH/1oIm3aVDFnTkdGX7MvZ5w5jR12SFpZH33UgWuvHZy3jDZtqvifUf+gdesqWrUKXnppa8alNx6tcXXtupafjpoEJF1UTz/Rh6mTt+atf3blop9P5sv/8R7zP2rPL/97XwD++uAOnHfhNG4Y8yQSPP63frz77y5s2XUNl/xyIm3aJr+zl6f14G8P92/GK2sGESU9Yb+iiTrPJa2IiI7p/tbAO8CVEXGppP4kwboHST/ytyJilqQDgD+SjAA5LiL+lVPepdQYHiepHHgI6A28CfQELo2IpyW9CwyOiAWSLgZOBt4DZgMz6htm12WLbWPIoJGb/kHYZqNVa5u7CtYAL8y6laVr5m7SoO1OW/aJPQ86t6Bzn33kR1MjIn9LJWOarAVdHZzT/Y+ALXJevwscUkue56hjmF1t46cjYi1wRB3n98/ZHwWMKrTuZlY8irX7ohCbow/azKxpBFAVhW31kHSzpHmSXstJ6ybpcUkz059dc45dJOntdAjv4Tnpe0t6NT12rSSl6eWS7knTJ6U9CXk5QJtZcWu8URy3AMNqpF0IjI+IgcD49DWSdiG5j7Zrmud6SWVpnhtIhusOTLfqMkeS3DMbAIwGrqivQg7QZlbUGmsUR0Q8A9QcW3o0MCbdH0PytHN1+t0RsTYi3gHeBvaV1AvoHBEvpMOAb62Rp7qsvwCHVreu6+LJksysqDXxKI6tI2IOQETMkVT9FFBvkmcvqs1O09an+zXTq/O8n5ZVIWkp0B1YUNebO0CbWfFq2EMoPSRNyXl9Y0Tc+CnfubaWb+RJz5enTg7QZla0kgdVCo7QCz7FMLuPJPVKW8+9gHlp+mySJ5er9QE+TNP71JKem2e2pNYkz3HkfVzXfdBmVtyqCtw+nbHAiHR/BPBwTvrwdGTG9iQ3Ayen3SHLJQ1J+5dPrpGnuqzjgCejngdR3II2s6LWgBZ0/nKku4ChJF0hs4FLgMuBeyWNBGaRTHVMRLyePiE9g2RmzbMjojIt6kySESHtgUfTDeAm4DZJb5O0nIfXVycHaDMrXo04EVJEfL2OQ4fWcX6tD8BFxBRgt1rS15AG+EI5QJtZESvtuTgcoM2suBXpZPyFcIA2s+IVxbucVSEcoM2suLkFbWaWUaUbnx2gzay4qap0+zgcoM2seAWb8hBK5jlAm1nREtFoD6pkkQO0mRU3B2gzs4xygDYzyyD3QZuZZZdHcZiZZVK4i8PMLJMCB2gzs8wq3R4OB2gzK24eB21mllUO0GZmGRQBlaXbx+EAbWbFzS1oM7OMcoA2M8ugALwmoZlZFgWE+6DNzLIn8E1CM7PMch+0mVlGOUCbmWWRJ0syM8umADzdqJlZRrkFbWaWRX7U28wsmwLC46DNzDLKTxKamWWU+6DNzDIowqM4zMwyyy1oM7MsCqKysrkr0WQcoM2seHm6UTOzDPMwOzOz7Akg3II2M8ug8IT9ZmaZVco3CRUlPERlU0iaD7zX3PVoAj2ABc1dCWuQUv2dbRcRPTelAEnjSD6fQiyIiGGb8n6bmwN0CyNpSkQMbu56WOH8O2u5WjV3BczMrHYO0GZmGeUA3fLc2NwVsAbz76yFch+0mVlGuQVtZpZRDtBmZhnlAF0iJFVKmi7pNUn3Sdoiz7lHSbqwjmMrmq6WLYOkkHR1zuvzJV1aT55jJO1Sx7FLJX2Q8/s9qp6y/iZpyzrKOb/Ay7AMcIAuHasjYo+I2A1YB5xR14kRMTYiLt98VWtx1gLHSir0AQqAY4BaA3RqdETsARwP3Cypzv+7EfGViFjSgPe2jHKALk3PAgMkHSlpkqSXJD0haWsASadIui7d317SC5JelHRZs9a6dFSQjLw4r+YBSdtJGi/plfRnP0n7A0cBV6Wt5B3rKjgi3kjL7yHpIUlTJb0u6bSc93i3+stB0sWS3pT0BLBTI1+nNTEH6BIjqTVwBPAqMAEYEhF7AncDP6oly2+BGyJiH2DuZqto6fsdcKKkLjXSrwNujYjPAXcA10bE88BY4IL0r6B/1VWopM8DVcB84NSI2BsYDJwjqXuNc/cGhgN7AscC+zTOpdnm4smSSkd7SdPT/WeBm0haTPdI6gW0Bd6pJd8BwH+l+7cBVzR1RVuCiFgm6VbgHGB1zqH9SIIlJJ/3lQUWeZ6kk4DlwAkREZLOkfTV9HhfYCCwMCfPgcCDEbEKQNLYT3c11lwcoEvH6rSPcgNJ/wtcExFjJQ0FLq0jrwfDN43fANOAP+c5p9DPfnRE/Lr6Rfr7PAzYLyJWSXoaaLcJ5VsGuYujtHUBPkj3R9RxznMkfwYDnNjkNWpBImIRcC8wMif5eTb+vCek+8uBTg0ovguwOA3OOwNDajnnGeCrktpL6gQc2ZD6W/NzgC5tlwL3SXqWuqerPBc4W9KLJP/prXFdzcbTYZ4DfEvSK8A3ST5/SO4RXJDe0K3zJmGOcUDrtJzLgIk1T4iIacA9wHTgfpKuLysiftTbzCyj3II2M8soB2gzs4xygDYzyygHaDOzjHKANjPLKAdo+1QaMnteAWXdIum4dP9Pdc3qlh4fms5d0dD32DA/RSHpNc5p0Ax/njXOGosDtH1aecElfJAAAAKgSURBVGfPk1T2aQqNiG9HxIw8pwwFGhygzYqRA7Q1hurZ84ZKekrSncCrksokXZXOlPeKpNMBlLhO0gxJ/wdsVV2QpKclDU73h0maJunldOa3/iRfBOelrfcDJfWUdH/6Hi9KOiDN213S39MHP/4AqL6LqGt2uPTY1WldxkvqmabtKGlcmufZ9Ik+s0bjuThsk+TMnjcuTdoX2C0i3kmD3NKI2EdSOfCcpL+TzK62E/BZYGtgBnBzjXJ7An8EDkrL6hYRiyT9HlhRPS9F+mUwOiImSOoHPAZ8BrgEmBARv5D0H8BGAbcOp6bv0R54UdL9EbEQ6ABMi4gfSvrvtOzvkkwpekZEzExnmbseOORTfIxmtXKAtk+rttnz9gcmR0T1rHlfBj5X3b9M8ij5QOAg4K6IqAQ+lPRkLeUPAZ6pLiud16I2hwG7SBsayJ3TeScOIp01LiL+T9LiAq6prtnhqkgemQa4HXhAUsf0eu/Lee/yAt7DrGAO0PZp1TZ7HsDK3CTgexHxWI3zvkL9s6ypgHMg6abbLyJyp/SsrkvB8xg0YHa46nJbAUtqfgZmjcl90NaUHgPOlNQGQNIgSR1IZlkbnvZR9wIOriXvC8AXJW2f5u2Wptec9e3vJN0NpOdVB8xnSGfnk3QE0LWeuuabHa4VUP1XwDdIuk6WAe9IOj59D0navZ73MGsQB2hrSn8i6V+eJuk14A8kf7U9CMwkWfXlBuAfNTNGxHySfuMHJL3Mx10Mj5BMoTld0oEks8MNTm9CzuDj0SQ/Bw6SNI2kq2VWPXXNNzvcSmBXSVNJ+ph/kaafCIxM6/c6cHQBn4lZwTybnZlZRrkFbWaWUQ7QZmYZ5QBtZpZRDtBmZhnlAG1mllEO0GZmGeUAbWaWUf8PUvhO8X93Ap0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_confusion_matrix(opyimal_params, X_test, y_test, values_format='d', display_labels=[\"Paid\",\"Not Paid\"]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'gamma': 0,\n",
       " 'learning_rate': 0.1,\n",
       " 'max_depth': 5,\n",
       " 'reg_lambda': 1.0,\n",
       " 'scale_pos_weight': 1}"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "opyimal_params.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### Second GridSearch Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "param_grid = {\n",
    "    'max_depth': [5, 6, 7],\n",
    "    'learning_rate': [0.1, 0.25, 0.5],\n",
    "    'booster':['gbtree','gblinear', 'dart'],\n",
    "    'gamma': [0, 0.25, 1.0],\n",
    "    'reg_lambda': [0, 1.0, 10.0],\n",
    "    'scale_pos_weight': [1, 5, 10]\n",
    "}\n",
    "opyimal_params = GridSearchCV (\n",
    "    estimator=XGBClassifier(objective='binary:logistic', \n",
    "                            seed=42),\n",
    "    param_grid=param_grid,\n",
    "    scoring='roc_auc',\n",
    "    verbose=0,\n",
    "    n_jobs=10,\n",
    "    cv=3\n",
    ")\n",
    "opyimal_params.fit(X_train, y_train)\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "grid_predictions = opyimal_params.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "print(classification_report(y_test, grid_predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "plot_confusion_matrix(opyimal_params, X_test, y_test, values_format='d', display_labels=[\"Paid\",\"Not Paid\"]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "opyimal_params.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### Final Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "init_time = datetime.now()\n",
    "\n",
    "clf_xgb=XGBClassifier(seed=42,\n",
    "                      gamma=0,\n",
    "                      learning_rate=0.1,\n",
    "                      max_depth=5,\n",
    "                      reg_lambda=1.0,\n",
    "                      scale_pos_weight=1)\n",
    "\n",
    "clf_xgb.fit(X_train, y_train,verbose=False,\n",
    "            early_stopping_rounds=10,\n",
    "            eval_metric='aucpr',\n",
    "            eval_set=[(X_valid,y_valid)])\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "print(classification_report(y_test, clf_xgb.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "plot_confusion_matrix(clf_xgb, X_test, y_test, values_format='d', display_labels=[\"Paid\",\"Not Paid\"]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.regularizers import l1, l2, L1L2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "L1L2(l1=0.0, l2=0.0)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}