test5.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import StratifiedKFold,cross_val_score\n",
    "from lightgbm import LGBMClassifier\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "color = sns.color_palette()\n",
    "sns.set_style('darkgrid')\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "from bayes_opt import BayesianOptimization\n",
    "from sklearn.ensemble import (RandomForestClassifier, AdaBoostClassifier, \n",
    "                              GradientBoostingClassifier, ExtraTreesClassifier)\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "import gc\n",
    "gc.enable()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read data and test\n",
      "('Shapes : ', (307511, 122), (48744, 121))\n",
      "['NAME_CONTRACT_TYPE', 'NAME_TYPE_SUITE', 'NAME_INCOME_TYPE', 'NAME_EDUCATION_TYPE', 'NAME_FAMILY_STATUS', 'NAME_HOUSING_TYPE', 'OCCUPATION_TYPE', 'WEEKDAY_APPR_PROCESS_START', 'ORGANIZATION_TYPE', 'FONDKAPREMONT_MODE', 'HOUSETYPE_MODE', 'WALLSMATERIAL_MODE', 'EMERGENCYSTATE_MODE']\n",
      "(356255, 255)\n",
      "(356255, 260)\n",
      "['STATUS']\n",
      "['CREDIT_ACTIVE', 'CREDIT_CURRENCY', 'CREDIT_TYPE']\n",
      "(817395, 13)\n",
      "(305811, 67)\n",
      "(305811, 100)\n",
      "(305811, 135)\n",
      "(356255, 394)\n",
      "['NAME_CONTRACT_TYPE', 'WEEKDAY_APPR_PROCESS_START', 'FLAG_LAST_APPL_PER_CONTRACT', 'NAME_CASH_LOAN_PURPOSE', 'NAME_CONTRACT_STATUS', 'NAME_PAYMENT_TYPE', 'CODE_REJECT_REASON', 'NAME_TYPE_SUITE', 'NAME_CLIENT_TYPE', 'NAME_GOODS_CATEGORY', 'NAME_PORTFOLIO', 'NAME_PRODUCT_TYPE', 'CHANNEL_TYPE', 'NAME_SELLER_INDUSTRY', 'NAME_YIELD_GROUP', 'PRODUCT_COMBINATION']\n",
      "(338857, 208)\n",
      "(338857, 258)\n",
      "(338857, 307)\n",
      "(356255, 700)\n",
      "['NAME_CONTRACT_STATUS']\n",
      "(337252, 24)\n",
      "(356255, 724)\n",
      "[]\n",
      "(339587, 29)\n",
      "(356255, 753)\n",
      "['NAME_CONTRACT_STATUS']\n",
      "(103558, 81)\n",
      "(356255, 834)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def one_hot_encoder(df, nan_as_category = True):\n",
    "    original_columns = list(df.columns)\n",
    "    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n",
    "    print (categorical_columns)\n",
    "    df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category)\n",
    "    new_columns = [c for c in df.columns if c not in original_columns]\n",
    "    return df, new_columns\n",
    "\n",
    "print('Read data and test')\n",
    "data = pd.read_csv('./input/application_train.csv')\n",
    "test = pd.read_csv('./input/application_test.csv')\n",
    "print('Shapes : ', data.shape, test.shape)\n",
    "\n",
    "y = data['TARGET']\n",
    "del data['TARGET']\n",
    "gc.collect()\n",
    "all_data = pd.concat((data, test)).reset_index(drop=True)\n",
    "\n",
    "inc_by_org = all_data[['AMT_INCOME_TOTAL', 'ORGANIZATION_TYPE']].groupby('ORGANIZATION_TYPE').median()['AMT_INCOME_TOTAL']\n",
    "all_data['NEW_INC_BY_ORG'] = all_data['ORGANIZATION_TYPE'].map(inc_by_org)\n",
    "\n",
    "for bin_feature in ['CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY']:\n",
    "    all_data[bin_feature], uniques = pd.factorize(all_data[bin_feature])\n",
    "all_data, cat_cols = one_hot_encoder(all_data)\n",
    "print all_data.shape\n",
    "# Some simple new features (percentages)\n",
    "#all_data['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True)\n",
    "n_data = data.shape[0]\n",
    "\n",
    "all_data['DAYS_EMPLOYED_PERC'] = all_data['DAYS_EMPLOYED'] / all_data['DAYS_BIRTH']\n",
    "all_data['INCOME_CREDIT_PERC'] = all_data['AMT_INCOME_TOTAL'] / all_data['AMT_CREDIT']\n",
    "all_data['INCOME_PER_PERSON'] = all_data['AMT_INCOME_TOTAL'] / all_data['CNT_FAM_MEMBERS']\n",
    "all_data['ANNUITY_INCOME_PERC'] = all_data['AMT_ANNUITY'] / all_data['AMT_INCOME_TOTAL']\n",
    "all_data['PAYMENT_RATE'] = all_data['AMT_CREDIT'] / all_data['AMT_ANNUITY']\n",
    "\n",
    "print all_data.shape\n",
    "del data,test\n",
    "gc.collect()\n",
    "\n",
    "bureau = pd.read_csv('./input/bureau.csv')\n",
    "buro_bal = pd.read_csv('./input/bureau_balance.csv')\n",
    "buro_bal, bb_cat = one_hot_encoder(buro_bal)\n",
    "bureau, bureau_cat = one_hot_encoder(bureau)\n",
    "bb_aggregations = {'MONTHS_BALANCE': ['min', 'max','mean']}\n",
    "for col in bb_cat:\n",
    "    bb_aggregations[col] = ['mean']\n",
    "    bb_agg = buro_bal.groupby('SK_ID_BUREAU').agg(bb_aggregations)\n",
    "bb_agg.columns = pd.Index([e[0] + \"_\" + e[1].upper() for e in bb_agg.columns.tolist()])\n",
    "bb_agg['BB_COUNT'] = buro_bal.groupby('SK_ID_BUREAU').size()\n",
    "print bb_agg.shape\n",
    "bureau = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')\n",
    "bureau.drop(columns= 'SK_ID_BUREAU', inplace= True)\n",
    "del buro_bal,bb_agg\n",
    "gc.collect()\n",
    "\n",
    "num_aggregations = {\n",
    "    'DAYS_CREDIT': ['min', 'max', 'mean', 'var'],\n",
    "    'CREDIT_DAY_OVERDUE': ['max', 'mean'],\n",
    "    'DAYS_CREDIT_ENDDATE': ['min', 'max', 'mean'],\n",
    "    'AMT_CREDIT_MAX_OVERDUE': ['mean'],\n",
    "    'CNT_CREDIT_PROLONG': ['sum','mean'],\n",
    "    'AMT_CREDIT_SUM': ['max','mean', 'sum'],\n",
    "    'AMT_CREDIT_SUM_DEBT': ['max','mean', 'sum'],\n",
    "    'AMT_CREDIT_SUM_OVERDUE': ['mean'],\n",
    "    'AMT_CREDIT_SUM_LIMIT': ['mean', 'sum'],\n",
    "    'DAYS_CREDIT_UPDATE': ['min','max','mean'],\n",
    "    'AMT_ANNUITY': ['max', 'mean'],\n",
    "    'MONTHS_BALANCE_MIN': ['min'],\n",
    "    'MONTHS_BALANCE_MAX': ['max'],\n",
    "    'MONTHS_BALANCE_MEAN': ['mean'],\n",
    "    'BB_COUNT': ['mean', 'sum']\n",
    "}\n",
    "cat_aggregations = {}\n",
    "for cat in bureau_cat: cat_aggregations[cat] = ['mean']\n",
    "for cat in bb_cat: cat_aggregations[cat + \"_MEAN\"] = ['mean']\n",
    "bureau_agg = bureau.groupby('SK_ID_CURR').agg(dict(num_aggregations, **cat_aggregations))\n",
    "bureau_agg.columns = pd.Index(['BURO_' + e[0] + \"_\" + e[1].upper() for e in bureau_agg.columns.tolist()])\n",
    "bureau_agg['BURO_COUNT'] = bureau.groupby('SK_ID_CURR').size()\n",
    "print bureau_agg.shape\n",
    "\n",
    "active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]\n",
    "active_agg = active.groupby('SK_ID_CURR').agg(num_aggregations)\n",
    "active_agg.columns = pd.Index(['ACT_' + e[0] + \"_\" + e[1].upper() for e in active_agg.columns.tolist()])\n",
    "active_agg['ACT_COUNT'] = active.groupby('SK_ID_CURR').size()\n",
    "bureau_agg = bureau_agg.reset_index().join(active_agg, how='left', on='SK_ID_CURR')\n",
    "print bureau_agg.shape\n",
    "del active, active_agg\n",
    "gc.collect()\n",
    "\n",
    "closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]\n",
    "closed_agg = closed.groupby('SK_ID_CURR').agg(dict({'DAYS_ENDDATE_FACT': ['min','max','mean']},**num_aggregations))\n",
    "closed_agg.columns = pd.Index(['CLS_' + e[0] + \"_\" + e[1].upper() for e in closed_agg.columns.tolist()])\n",
    "closed_agg['CLS_COUNT'] = closed.groupby('SK_ID_CURR').size()\n",
    "bureau_agg = bureau_agg.join(closed_agg, how='left', on='SK_ID_CURR')\n",
    "print bureau_agg.shape\n",
    "del closed, closed_agg, bureau\n",
    "gc.collect()\n",
    "\n",
    "all_data = all_data.merge(bureau_agg, how='left', on='SK_ID_CURR')\n",
    "print all_data.shape\n",
    "del bureau_agg\n",
    "gc.collect()\n",
    "\n",
    "prev = pd.read_csv('./input/previous_application.csv')\n",
    "prev, cat_cols = one_hot_encoder(prev, nan_as_category= True)\n",
    "# Days 365.243 values -> nan\n",
    "prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)\n",
    "prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)\n",
    "prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)\n",
    "prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)\n",
    "prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True)\n",
    "# Add feature: value ask / value received percentage\n",
    "prev['APP_CREDIT_PERC'] = prev['AMT_APPLICATION'] / prev['AMT_CREDIT']\n",
    "# Previous applications numeric features\n",
    "num_aggregations = {\n",
    "    'AMT_ANNUITY': ['min','max', 'mean'],\n",
    "    'AMT_APPLICATION': ['min','max', 'mean'],\n",
    "    'AMT_CREDIT': ['min','max', 'mean'],\n",
    "    'APP_CREDIT_PERC': ['min', 'max', 'mean', 'var'],\n",
    "    'AMT_DOWN_PAYMENT': ['min', 'max', 'mean'],\n",
    "    'AMT_GOODS_PRICE': ['min', 'max', 'mean'],\n",
    "    'HOUR_APPR_PROCESS_START': ['min', 'max', 'mean'],\n",
    "    'RATE_DOWN_PAYMENT': ['min', 'max', 'mean'],\n",
    "    'DAYS_DECISION': ['min', 'max', 'mean'],\n",
    "    'SELLERPLACE_AREA': ['min', 'max', 'mean'],\n",
    "    'DAYS_FIRST_DRAWING': ['min', 'max', 'mean'],\n",
    "    'DAYS_FIRST_DUE': ['min', 'max', 'mean'],\n",
    "    'DAYS_LAST_DUE_1ST_VERSION': ['min', 'max', 'mean'],\n",
    "    'DAYS_LAST_DUE': ['min', 'max', 'mean'],\n",
    "    'DAYS_TERMINATION': ['min', 'max', 'mean'],\n",
    "    'CNT_PAYMENT': ['mean', 'sum']\n",
    "}\n",
    "# Previous applications categorical features\n",
    "cat_aggregations = {}\n",
    "for cat in cat_cols:\n",
    "    cat_aggregations[cat] = ['mean']\n",
    "\n",
    "prev_agg = prev.groupby('SK_ID_CURR').agg(dict(num_aggregations, **cat_aggregations))\n",
    "prev_agg.columns = pd.Index(['PREV_' + e[0] + \"_\" + e[1].upper() for e in prev_agg.columns.tolist()])\n",
    "prev_agg['PREV_COUNT'] = prev.groupby('SK_ID_CURR').size()\n",
    "print prev_agg.shape\n",
    "# Previous Applications: Approved Applications - only numerical features\n",
    "approved = prev[prev['NAME_CONTRACT_STATUS_Approved'] == 1]\n",
    "approved_agg = approved.groupby('SK_ID_CURR').agg(num_aggregations)\n",
    "approved_agg.columns = pd.Index(['APR_' + e[0] + \"_\" + e[1].upper() for e in approved_agg.columns.tolist()])\n",
    "approved_agg['APR_COUNT'] = approved.groupby('SK_ID_CURR').size()\n",
    "prev_agg = prev_agg.reset_index().join(approved_agg, how='left', on='SK_ID_CURR')\n",
    "print prev_agg.shape\n",
    "# Previous Applications: Refused Applications - only numerical features\n",
    "refused = prev[prev['NAME_CONTRACT_STATUS_Refused'] == 1]\n",
    "refused_agg = refused.groupby('SK_ID_CURR').agg(num_aggregations)\n",
    "refused_agg.columns = pd.Index(['REF_' + e[0] + \"_\" + e[1].upper() for e in refused_agg.columns.tolist()])\n",
    "refused_agg['REF_COUNT'] = refused.groupby('SK_ID_CURR').size()\n",
    "prev_agg = prev_agg.join(refused_agg, how='left', on='SK_ID_CURR')\n",
    "print prev_agg.shape\n",
    "del refused, refused_agg, approved, approved_agg, prev\n",
    "gc.collect()\n",
    "\n",
    "all_data = all_data.merge(prev_agg, how='left', on='SK_ID_CURR')\n",
    "print all_data.shape\n",
    "del prev_agg\n",
    "gc.collect()\n",
    "\n",
    "pos = pd.read_csv('./input/POS_CASH_balance.csv')\n",
    "pos, cat_cols = one_hot_encoder(pos, nan_as_category= True)\n",
    "# Features\n",
    "aggregations = {\n",
    "    'MONTHS_BALANCE': ['min','max', 'mean'],\n",
    "    'CNT_INSTALMENT': ['sum', 'mean'],\n",
    "    'SK_DPD': ['sum','max','min','mean'],\n",
    "    'SK_DPD_DEF': ['sum','max', 'min','mean']\n",
    "}\n",
    "for cat in cat_cols:\n",
    "    aggregations[cat] = ['mean']\n",
    "\n",
    "pos_agg = pos.groupby('SK_ID_CURR').agg(aggregations)\n",
    "pos_agg.columns = pd.Index(['POS_' + e[0] + \"_\" + e[1].upper() for e in pos_agg.columns.tolist()])\n",
    "# Count pos cash accounts\n",
    "pos_agg['POS_COUNT'] = pos.groupby('SK_ID_CURR').size()\n",
    "print pos_agg.shape\n",
    "del pos\n",
    "gc.collect()\n",
    "\n",
    "all_data = all_data.join(pos_agg, how='left', on='SK_ID_CURR')\n",
    "print all_data.shape\n",
    "del pos_agg\n",
    "gc.collect()\n",
    "\n",
    "ins = pd.read_csv('./input/installments_payments.csv')\n",
    "ins, cat_cols = one_hot_encoder(ins, nan_as_category= True)\n",
    "# Percentage and difference paid in each installment (amount paid and installment value)\n",
    "ins['PAYMENT_PERC'] = ins['AMT_PAYMENT'] / ins['AMT_INSTALMENT']\n",
    "ins['PAYMENT_DIFF'] = ins['AMT_INSTALMENT'] - ins['AMT_PAYMENT']\n",
    "# Days past due and days before due (no negative values)\n",
    "ins['DPD'] = ins['DAYS_ENTRY_PAYMENT'] - ins['DAYS_INSTALMENT']\n",
    "ins['DBD'] = ins['DAYS_INSTALMENT'] - ins['DAYS_ENTRY_PAYMENT']\n",
    "ins['DPD'] = ins['DPD'].apply(lambda x: x if x > 0 else 0)\n",
    "ins['DBD'] = ins['DBD'].apply(lambda x: x if x > 0 else 0)\n",
    "# Features: Perform aggregations\n",
    "aggregations = {\n",
    "    'NUM_INSTALMENT_VERSION': ['nunique'],\n",
    "    'DPD': ['max', 'mean', 'sum'],\n",
    "    'DBD': ['max', 'mean', 'sum'],\n",
    "    'PAYMENT_PERC': ['max', 'mean', 'sum', 'var'],\n",
    "    'PAYMENT_DIFF': ['max', 'mean', 'sum', 'var'],\n",
    "    'AMT_INSTALMENT': ['max', 'mean', 'sum'],\n",
    "    'AMT_PAYMENT': ['min', 'max', 'mean', 'sum'],\n",
    "    'DAYS_ENTRY_PAYMENT': ['max', 'mean', 'sum'],\n",
    "    'DAYS_INSTALMENT': ['max', 'mean', 'sum']\n",
    "}\n",
    "for cat in cat_cols:\n",
    "    aggregations[cat] = ['mean']\n",
    "ins_agg = ins.groupby('SK_ID_CURR').agg(aggregations)\n",
    "ins_agg.columns = pd.Index(['INS_' + e[0] + \"_\" + e[1].upper() for e in ins_agg.columns.tolist()])\n",
    "# Count installments accounts\n",
    "ins_agg['INS_COUNT'] = ins.groupby('SK_ID_CURR').size()\n",
    "print ins_agg.shape\n",
    "del ins\n",
    "gc.collect()\n",
    "\n",
    "all_data = all_data.join(ins_agg, how='left', on='SK_ID_CURR')\n",
    "print all_data.shape\n",
    "del ins_agg\n",
    "gc.collect()\n",
    "\n",
    "cc = pd.read_csv('./input/credit_card_balance.csv')\n",
    "cc, cat_cols = one_hot_encoder(cc, nan_as_category= True)\n",
    "# Features: Perform aggregations\n",
    "num_aggregations = {\n",
    "    'MONTHS_BALANCE': ['min','max', 'mean'],\n",
    "    'AMT_BALANCE': ['min','max', 'mean','sum'],\n",
    "    'AMT_CREDIT_LIMIT_ACTUAL': ['min','max', 'mean','sum','var'],\n",
    "    'AMT_DRAWINGS_ATM_CURRENT': ['min','max', 'mean','sum'],\n",
    "    'AMT_DRAWINGS_CURRENT': ['min','max', 'mean','sum'],\n",
    "    'AMT_DRAWINGS_OTHER_CURRENT': ['min','max', 'mean','sum'],\n",
    "    'AMT_DRAWINGS_POS_CURRENT': ['min','max', 'mean','sum'],\n",
    "    'AMT_INST_MIN_REGULARITY': ['min','max', 'mean','sum','var'],\n",
    "    'AMT_PAYMENT_CURRENT':['min','max', 'mean','sum'],\n",
    "    'AMT_PAYMENT_TOTAL_CURRENT':['min','max', 'mean','sum'],\n",
    "    'AMT_RECEIVABLE_PRINCIPAL':['min','max', 'mean','sum','var'],\n",
    "    'AMT_RECIVABLE':['min','max', 'mean','sum','var'],\n",
    "    'AMT_TOTAL_RECEIVABLE':['min','max', 'mean','sum','var'],\n",
    "    'CNT_DRAWINGS_ATM_CURRENT':['mean','sum'],\n",
    "    'CNT_DRAWINGS_CURRENT':['mean','sum'],\n",
    "    'CNT_DRAWINGS_OTHER_CURRENT':['mean','sum'],\n",
    "    'CNT_DRAWINGS_POS_CURRENT':['mean','sum'],\n",
    "    'CNT_INSTALMENT_MATURE_CUM':['mean','sum'],\n",
    "    'SK_DPD':['max', 'mean', 'sum'],\n",
    "    'SK_DPD_DEF':['max', 'mean', 'sum']\n",
    "}\n",
    "cat_aggregations = {}\n",
    "for cat in cat_cols:\n",
    "    cat_aggregations[cat] = ['mean']\n",
    "# General aggregations\n",
    "cc_agg = cc.groupby('SK_ID_CURR').agg(dict(num_aggregations,**cat_aggregations))\n",
    "cc_agg.columns = pd.Index(['CC_' + e[0] + \"_\" + e[1].upper() for e in cc_agg.columns.tolist()])\n",
    "# Count credit card lines\n",
    "cc_agg['CC_COUNT'] = cc.groupby('SK_ID_CURR').size()\n",
    "\n",
    "#cc_agg['INSTALLMENTS_PER_LOAN'] = cc_agg['CNT_INSTALMENT_MATURE_CUM_SUM'] / cc_agg['CC_COUNT']\n",
    "del cc\n",
    "print cc_agg.shape\n",
    "gc.collect()\n",
    "\n",
    "all_data = all_data.join(cc_agg, how='left', on='SK_ID_CURR')\n",
    "print all_data.shape\n",
    "del cc_agg\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data_na = (all_data.isnull().sum() / len(all_data)) * 100\n",
    "all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index).sort_values(ascending=False)\n",
    "missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})\n",
    "\n",
    "drop_col = missing_data[missing_data['Missing Ratio']==100]\n",
    "missing_data = missing_data[missing_data['Missing Ratio']<100]\n",
    "\n",
    "all_data.drop(drop_col.index.values,1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "all_data_na = (all_data.isnull().sum() / len(all_data)) * 100\n",
    "all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index).sort_values(ascending=False)\n",
    "missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})\n",
    "\n",
    "drop_col = missing_data[missing_data['Missing Ratio']==100]\n",
    "missing_data = missing_data[missing_data['Missing Ratio']<100]\n",
    "\n",
    "all_data.drop(drop_col.index.values,1,inplace=True)\n",
    "\n",
    "for i in missing_data.index.values:\n",
    "    all_data[i] = all_data[i].fillna(-999999999)\n",
    "\n",
    "all_data_na = (all_data.isnull().sum() / len(all_data)) * 100\n",
    "all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index).sort_values(ascending=False)\n",
    "missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})\n",
    "\n",
    "missing_data\n",
    "\n",
    "all_data[all_data==np.inf]=999999999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(307511, 819)\n",
      "(48744, 819)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['SK_ID_CURR']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = all_data[:n_data]\n",
    "test = all_data[n_data:]\n",
    "#del avg_buro, avg_prev,avg_pos,avg_cc_bal,avg_inst\n",
    "del all_data\n",
    "gc.collect()\n",
    "print data.shape\n",
    "print test.shape\n",
    "\n",
    "exclude_fea = []\n",
    "exclude_fea.append('SK_ID_CURR') \n",
    "exclude_fea"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "folds = KFold(n_splits=5, shuffle=True, random_state=1700)\n",
    "oof_preds = np.zeros(data.shape[0])\n",
    "sub_preds = np.zeros(test.shape[0])\n",
    "feature_importance_df = pd.DataFrame()\n",
    "feats = [f for f in data.columns if f not in exclude_fea]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max_iter reached after 1447 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed: 24.1min\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-d8b34d054e9b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mclf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLogisticRegression\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msolver\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'sag'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmax_iter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrandom_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1700\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mclf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrn_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrn_y\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0moof_preds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mval_idx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_x\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/sklearn/linear_model/logistic.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m   1289\u001b[0m                       \u001b[0mmax_squared_sum\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_squared_sum\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1290\u001b[0m                       sample_weight=sample_weight)\n\u001b[0;32m-> 1291\u001b[0;31m             for class_, warm_start_coef_ in zip(classes_, warm_start_coef))\n\u001b[0m\u001b[1;32m   1292\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1293\u001b[0m         \u001b[0mfold_coefs_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_iter_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mfold_coefs_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/sklearn/externals/joblib/parallel.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m    787\u001b[0m                 \u001b[0;31m# consumption.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    788\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 789\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    790\u001b[0m             \u001b[0;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    791\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/sklearn/externals/joblib/parallel.pyc\u001b[0m in \u001b[0;36mretrieve\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    697\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    698\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'supports_timeout'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 699\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    700\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    701\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python2.7/multiprocessing/pool.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    559\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    560\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 561\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    562\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ready\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    563\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python2.7/multiprocessing/pool.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    554\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    555\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ready\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 556\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    557\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    558\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python2.7/threading.pyc\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    338\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m    \u001b[0;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    339\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 340\u001b[0;31m                 \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    341\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0m__debug__\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    342\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_note\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"%s.wait(): got it\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for n_fold, (trn_idx, val_idx) in enumerate(folds.split(data)):\n",
    "    trn_x, trn_y = data[feats].iloc[trn_idx], y.iloc[trn_idx]\n",
    "    val_x, val_y = data[feats].iloc[val_idx], y.iloc[val_idx]\n",
    "    \n",
    "    clf = LogisticRegression(solver='sag',max_iter=1000,verbose=10,n_jobs=-1,C=10,random_state=1700)\n",
    "\n",
    "    clf.fit(trn_x, trn_y)\n",
    "    \n",
    "    oof_preds[val_idx] = clf.predict_proba(val_x)[:, 1]\n",
    "    sub_preds += clf.predict_proba(test[feats])[:, 1] / folds.n_splits\n",
    "    print('Fold %2d Train AUC : %.6f' % (n_fold + 1, roc_auc_score(trn_y, clf.predict_proba(trn_x)[:, 1])))\n",
    "    print('Fold %2d Test AUC : %.6f' % (n_fold + 1, roc_auc_score(val_y, oof_preds[val_idx])))\n",
    "    del clf, trn_x, trn_y, val_x, val_y\n",
    "    gc.collect()\n",
    "    if n_fold == 0:\n",
    "        break\n",
    "    \n",
    "print('Full AUC score %.6f' % roc_auc_score(y, oof_preds)) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lgbcv(num_leaves, colsample_bytree,subsample,reg_alpha,reg_lambda,min_split_gain,min_child_weight):\n",
    "    folds = KFold(n_splits=5, shuffle=True, random_state=1700)\n",
    "    feats = [f for f in data.columns if f not in exclude_fea]\n",
    "    oof_preds = np.zeros(data.shape[0])\n",
    "    for n_fold, (trn_idx, val_idx) in enumerate(folds.split(data)):\n",
    "        trn_x, trn_y = data[feats].iloc[trn_idx], y.iloc[trn_idx]\n",
    "        val_x, val_y = data[feats].iloc[val_idx], y.iloc[val_idx]\n",
    "\n",
    "        clf = LGBMClassifier(\n",
    "        n_estimators=10000,\n",
    "        learning_rate=0.05,\n",
    "        num_leaves=int(num_leaves),\n",
    "        colsample_bytree=max(min(colsample_bytree, 1), 0),\n",
    "        subsample=max(min(subsample, 1), 0),\n",
    "        max_depth=8,\n",
    "        reg_alpha=max(reg_alpha, 0),\n",
    "        reg_lambda=max(reg_lambda, 0),\n",
    "        min_split_gain=max(min_split_gain, 0),\n",
    "        min_child_weight=max(min_child_weight, 0),\n",
    "        random_state=1700,\n",
    "        )\n",
    "        clf.fit(trn_x, trn_y, \n",
    "            eval_set= [(trn_x, trn_y), (val_x, val_y)], \n",
    "            eval_metric='auc', verbose=100, early_stopping_rounds=100  #30\n",
    "           )\n",
    "\n",
    "        oof_preds[val_idx] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)[:, 1]\n",
    "        print('Fold %2d AUC : %.6f' % (n_fold + 1, roc_auc_score(val_y, oof_preds[val_idx])))\n",
    "        del clf, trn_x, trn_y, val_x, val_y\n",
    "        gc.collect()\n",
    "    full_auc = roc_auc_score(y, oof_preds)\n",
    "    print('Full AUC score %.6f' % full_auc) \n",
    "    return(full_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgbBO = BayesianOptimization(\n",
    "        lgbcv,\n",
    "        {'num_leaves': (20, 100),\n",
    "        'colsample_bytree': (0.5, 1.0),\n",
    "        'subsample':(0.5,1.0),\n",
    "        'reg_alpha':(0.001,10),\n",
    "        'reg_lambda':(0.001,10),\n",
    "        'min_split_gain':(0.0001,10),\n",
    "        'min_child_weight':(20,120)}\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mInitialization\u001b[0m\n",
      "\u001b[94m----------------------------------------------------------------------------------------------------------------------------------------------------\u001b[0m\n",
      " Step |   Time |      Value |   colsample_bytree |   min_child_weight |   min_split_gain |   num_leaves |   reg_alpha |   reg_lambda |   subsample | \n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.790898\tvalid_1's auc: 0.771298\n",
      "[200]\ttraining's auc: 0.809141\tvalid_1's auc: 0.783415\n",
      "[300]\ttraining's auc: 0.814748\tvalid_1's auc: 0.786341\n",
      "[400]\ttraining's auc: 0.81748\tvalid_1's auc: 0.787469\n",
      "[500]\ttraining's auc: 0.819464\tvalid_1's auc: 0.788358\n",
      "[600]\ttraining's auc: 0.820843\tvalid_1's auc: 0.788806\n",
      "[700]\ttraining's auc: 0.821864\tvalid_1's auc: 0.788962\n",
      "[800]\ttraining's auc: 0.822801\tvalid_1's auc: 0.789174\n",
      "[900]\ttraining's auc: 0.823705\tvalid_1's auc: 0.789364\n",
      "[1000]\ttraining's auc: 0.824642\tvalid_1's auc: 0.789726\n",
      "[1100]\ttraining's auc: 0.824906\tvalid_1's auc: 0.789708\n",
      "Early stopping, best iteration is:\n",
      "[1035]\ttraining's auc: 0.824894\tvalid_1's auc: 0.789762\n",
      "Fold  1 AUC : 0.789762\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.788735\tvalid_1's auc: 0.775744\n",
      "[200]\ttraining's auc: 0.807516\tvalid_1's auc: 0.78822\n",
      "[300]\ttraining's auc: 0.813058\tvalid_1's auc: 0.791257\n",
      "[400]\ttraining's auc: 0.81622\tvalid_1's auc: 0.792692\n",
      "[500]\ttraining's auc: 0.81812\tvalid_1's auc: 0.793421\n",
      "[600]\ttraining's auc: 0.81955\tvalid_1's auc: 0.793952\n",
      "[700]\ttraining's auc: 0.820702\tvalid_1's auc: 0.794377\n",
      "[800]\ttraining's auc: 0.821794\tvalid_1's auc: 0.794805\n",
      "[900]\ttraining's auc: 0.822148\tvalid_1's auc: 0.794927\n",
      "Early stopping, best iteration is:\n",
      "[821]\ttraining's auc: 0.822015\tvalid_1's auc: 0.794964\n",
      "Fold  2 AUC : 0.794964\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.789664\tvalid_1's auc: 0.773526\n",
      "[200]\ttraining's auc: 0.808861\tvalid_1's auc: 0.78464\n",
      "[300]\ttraining's auc: 0.814642\tvalid_1's auc: 0.786739\n",
      "[400]\ttraining's auc: 0.817674\tvalid_1's auc: 0.787921\n",
      "[500]\ttraining's auc: 0.819534\tvalid_1's auc: 0.788643\n",
      "[600]\ttraining's auc: 0.821058\tvalid_1's auc: 0.788979\n",
      "[700]\ttraining's auc: 0.822346\tvalid_1's auc: 0.789345\n",
      "[800]\ttraining's auc: 0.823362\tvalid_1's auc: 0.789464\n",
      "[900]\ttraining's auc: 0.824235\tvalid_1's auc: 0.789648\n",
      "[1000]\ttraining's auc: 0.825007\tvalid_1's auc: 0.789722\n",
      "Early stopping, best iteration is:\n",
      "[978]\ttraining's auc: 0.824894\tvalid_1's auc: 0.78977\n",
      "Fold  3 AUC : 0.789770\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.789162\tvalid_1's auc: 0.772985\n",
      "[200]\ttraining's auc: 0.809166\tvalid_1's auc: 0.78539\n",
      "[300]\ttraining's auc: 0.814983\tvalid_1's auc: 0.78837\n",
      "[400]\ttraining's auc: 0.817975\tvalid_1's auc: 0.789608\n",
      "[500]\ttraining's auc: 0.819953\tvalid_1's auc: 0.790299\n",
      "[600]\ttraining's auc: 0.821482\tvalid_1's auc: 0.790765\n",
      "[700]\ttraining's auc: 0.822696\tvalid_1's auc: 0.791178\n",
      "[800]\ttraining's auc: 0.823565\tvalid_1's auc: 0.791356\n",
      "[900]\ttraining's auc: 0.824535\tvalid_1's auc: 0.791552\n",
      "[1000]\ttraining's auc: 0.825267\tvalid_1's auc: 0.791594\n",
      "[1100]\ttraining's auc: 0.825975\tvalid_1's auc: 0.791594\n",
      "[1200]\ttraining's auc: 0.826704\tvalid_1's auc: 0.791723\n",
      "Early stopping, best iteration is:\n",
      "[1156]\ttraining's auc: 0.826366\tvalid_1's auc: 0.791753\n",
      "Fold  4 AUC : 0.791702\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.790434\tvalid_1's auc: 0.766892\n",
      "[200]\ttraining's auc: 0.808982\tvalid_1's auc: 0.779247\n",
      "[300]\ttraining's auc: 0.814957\tvalid_1's auc: 0.782382\n",
      "[400]\ttraining's auc: 0.817917\tvalid_1's auc: 0.7836\n",
      "[500]\ttraining's auc: 0.819856\tvalid_1's auc: 0.784375\n",
      "[600]\ttraining's auc: 0.821359\tvalid_1's auc: 0.784701\n",
      "[700]\ttraining's auc: 0.822566\tvalid_1's auc: 0.784951\n",
      "[800]\ttraining's auc: 0.823468\tvalid_1's auc: 0.784999\n",
      "[900]\ttraining's auc: 0.824395\tvalid_1's auc: 0.785156\n",
      "Early stopping, best iteration is:\n",
      "[897]\ttraining's auc: 0.824356\tvalid_1's auc: 0.785172\n",
      "Fold  5 AUC : 0.785160\n",
      "Full AUC score 0.790253\n",
      "    1 | 08m30s | \u001b[35m   0.79025\u001b[0m | \u001b[32m            0.7635\u001b[0m | \u001b[32m          109.3985\u001b[0m | \u001b[32m          5.2167\u001b[0m | \u001b[32m     97.9930\u001b[0m | \u001b[32m     9.0144\u001b[0m | \u001b[32m      0.1190\u001b[0m | \u001b[32m     0.5248\u001b[0m | \n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.793467\tvalid_1's auc: 0.770438\n",
      "[200]\ttraining's auc: 0.820902\tvalid_1's auc: 0.78404\n",
      "[300]\ttraining's auc: 0.837263\tvalid_1's auc: 0.788186\n",
      "[400]\ttraining's auc: 0.851345\tvalid_1's auc: 0.79016\n",
      "[500]\ttraining's auc: 0.863841\tvalid_1's auc: 0.791039\n",
      "[600]\ttraining's auc: 0.874813\tvalid_1's auc: 0.791255\n",
      "[700]\ttraining's auc: 0.885004\tvalid_1's auc: 0.79152\n",
      "[800]\ttraining's auc: 0.894323\tvalid_1's auc: 0.791671\n",
      "[900]\ttraining's auc: 0.90251\tvalid_1's auc: 0.791395\n",
      "Early stopping, best iteration is:\n",
      "[861]\ttraining's auc: 0.899322\tvalid_1's auc: 0.791747\n",
      "Fold  1 AUC : 0.791747\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.791839\tvalid_1's auc: 0.776308\n",
      "[200]\ttraining's auc: 0.82018\tvalid_1's auc: 0.789878\n",
      "[300]\ttraining's auc: 0.836628\tvalid_1's auc: 0.794056\n",
      "[400]\ttraining's auc: 0.850343\tvalid_1's auc: 0.795426\n",
      "[500]\ttraining's auc: 0.861978\tvalid_1's auc: 0.795961\n",
      "[600]\ttraining's auc: 0.872903\tvalid_1's auc: 0.796387\n",
      "Early stopping, best iteration is:\n",
      "[590]\ttraining's auc: 0.87183\tvalid_1's auc: 0.79644\n",
      "Fold  2 AUC : 0.796440\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.792971\tvalid_1's auc: 0.773789\n",
      "[200]\ttraining's auc: 0.820791\tvalid_1's auc: 0.786193\n",
      "[300]\ttraining's auc: 0.837295\tvalid_1's auc: 0.789594\n",
      "[400]\ttraining's auc: 0.851252\tvalid_1's auc: 0.790897\n",
      "[500]\ttraining's auc: 0.862818\tvalid_1's auc: 0.791502\n",
      "[600]\ttraining's auc: 0.873746\tvalid_1's auc: 0.791889\n",
      "[700]\ttraining's auc: 0.883532\tvalid_1's auc: 0.792106\n",
      "Early stopping, best iteration is:\n",
      "[671]\ttraining's auc: 0.880812\tvalid_1's auc: 0.792114\n",
      "Fold  3 AUC : 0.792114\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.792894\tvalid_1's auc: 0.774242\n",
      "[200]\ttraining's auc: 0.820861\tvalid_1's auc: 0.787083\n",
      "[300]\ttraining's auc: 0.837566\tvalid_1's auc: 0.790957\n",
      "[400]\ttraining's auc: 0.8511\tvalid_1's auc: 0.792617\n",
      "[500]\ttraining's auc: 0.862842\tvalid_1's auc: 0.79321\n",
      "Early stopping, best iteration is:\n",
      "[497]\ttraining's auc: 0.862533\tvalid_1's auc: 0.793253\n",
      "Fold  4 AUC : 0.793253\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.793887\tvalid_1's auc: 0.767311\n",
      "[200]\ttraining's auc: 0.821619\tvalid_1's auc: 0.781567\n",
      "[300]\ttraining's auc: 0.837672\tvalid_1's auc: 0.786103\n",
      "[400]\ttraining's auc: 0.8509\tvalid_1's auc: 0.78735\n",
      "[500]\ttraining's auc: 0.862727\tvalid_1's auc: 0.788234\n",
      "[600]\ttraining's auc: 0.873648\tvalid_1's auc: 0.788598\n",
      "Early stopping, best iteration is:\n",
      "[582]\ttraining's auc: 0.871546\tvalid_1's auc: 0.788638\n",
      "Fold  5 AUC : 0.788638\n",
      "Full AUC score 0.792409\n",
      "    2 | 12m14s | \u001b[35m   0.79241\u001b[0m | \u001b[32m            0.8185\u001b[0m | \u001b[32m           76.7848\u001b[0m | \u001b[32m          0.2611\u001b[0m | \u001b[32m     35.8914\u001b[0m | \u001b[32m     7.8412\u001b[0m | \u001b[32m      7.5831\u001b[0m | \u001b[32m     0.8393\u001b[0m | \n",
      "\u001b[31mBayesian Optimization\u001b[0m\n",
      "\u001b[94m----------------------------------------------------------------------------------------------------------------------------------------------------\u001b[0m\n",
      " Step |   Time |      Value |   colsample_bytree |   min_child_weight |   min_split_gain |   num_leaves |   reg_alpha |   reg_lambda |   subsample | \n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.791981\tvalid_1's auc: 0.770544\n",
      "[200]\ttraining's auc: 0.805935\tvalid_1's auc: 0.782091\n",
      "[300]\ttraining's auc: 0.808505\tvalid_1's auc: 0.784505\n",
      "[400]\ttraining's auc: 0.809447\tvalid_1's auc: 0.785327\n",
      "Early stopping, best iteration is:\n",
      "[383]\ttraining's auc: 0.809447\tvalid_1's auc: 0.785327\n",
      "Fold  1 AUC : 0.785327\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.789649\tvalid_1's auc: 0.775599\n",
      "[200]\ttraining's auc: 0.804366\tvalid_1's auc: 0.786796\n",
      "[300]\ttraining's auc: 0.80687\tvalid_1's auc: 0.788755\n",
      "[400]\ttraining's auc: 0.807875\tvalid_1's auc: 0.789754\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopping, best iteration is:\n",
      "[387]\ttraining's auc: 0.807875\tvalid_1's auc: 0.789754\n",
      "Fold  2 AUC : 0.789754\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.791622\tvalid_1's auc: 0.77343\n",
      "[200]\ttraining's auc: 0.80586\tvalid_1's auc: 0.783218\n",
      "[300]\ttraining's auc: 0.808545\tvalid_1's auc: 0.785186\n",
      "[400]\ttraining's auc: 0.809688\tvalid_1's auc: 0.786002\n",
      "[500]\ttraining's auc: 0.809912\tvalid_1's auc: 0.786153\n",
      "Early stopping, best iteration is:\n",
      "[442]\ttraining's auc: 0.809904\tvalid_1's auc: 0.786159\n",
      "Fold  3 AUC : 0.786153\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.791617\tvalid_1's auc: 0.774153\n",
      "[200]\ttraining's auc: 0.806038\tvalid_1's auc: 0.784385\n",
      "[300]\ttraining's auc: 0.808722\tvalid_1's auc: 0.786473\n",
      "[400]\ttraining's auc: 0.809237\tvalid_1's auc: 0.786783\n",
      "Early stopping, best iteration is:\n",
      "[335]\ttraining's auc: 0.809194\tvalid_1's auc: 0.786799\n",
      "Fold  4 AUC : 0.786799\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.79254\tvalid_1's auc: 0.768329\n",
      "[200]\ttraining's auc: 0.806219\tvalid_1's auc: 0.778956\n",
      "[300]\ttraining's auc: 0.808942\tvalid_1's auc: 0.781461\n",
      "[400]\ttraining's auc: 0.809635\tvalid_1's auc: 0.782028\n",
      "Early stopping, best iteration is:\n",
      "[365]\ttraining's auc: 0.809635\tvalid_1's auc: 0.782028\n",
      "Fold  5 AUC : 0.782028\n",
      "Full AUC score 0.785995\n",
      "    3 | 04m47s |    0.78600 |             0.5610 |            20.1188 |           9.9236 |      99.8846 |      7.2057 |       7.2912 |      0.6642 | \n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.782803\tvalid_1's auc: 0.767027\n",
      "[200]\ttraining's auc: 0.804514\tvalid_1's auc: 0.782805\n",
      "[300]\ttraining's auc: 0.808352\tvalid_1's auc: 0.785551\n",
      "[400]\ttraining's auc: 0.809699\tvalid_1's auc: 0.786533\n",
      "[500]\ttraining's auc: 0.809902\tvalid_1's auc: 0.7867\n",
      "Early stopping, best iteration is:\n",
      "[416]\ttraining's auc: 0.809902\tvalid_1's auc: 0.7867\n",
      "Fold  1 AUC : 0.786700\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.7809\tvalid_1's auc: 0.771811\n",
      "[200]\ttraining's auc: 0.802821\tvalid_1's auc: 0.787492\n",
      "[300]\ttraining's auc: 0.806935\tvalid_1's auc: 0.790184\n",
      "[400]\ttraining's auc: 0.80806\tvalid_1's auc: 0.791045\n",
      "[500]\ttraining's auc: 0.808355\tvalid_1's auc: 0.791224\n",
      "Early stopping, best iteration is:\n",
      "[431]\ttraining's auc: 0.808355\tvalid_1's auc: 0.791215\n",
      "Fold  2 AUC : 0.791224\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.781952\tvalid_1's auc: 0.770076\n",
      "[200]\ttraining's auc: 0.804415\tvalid_1's auc: 0.783815\n",
      "[300]\ttraining's auc: 0.808793\tvalid_1's auc: 0.786242\n",
      "[400]\ttraining's auc: 0.810167\tvalid_1's auc: 0.786978\n",
      "[500]\ttraining's auc: 0.810622\tvalid_1's auc: 0.787199\n",
      "Early stopping, best iteration is:\n",
      "[481]\ttraining's auc: 0.810611\tvalid_1's auc: 0.787203\n",
      "Fold  3 AUC : 0.787199\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.781841\tvalid_1's auc: 0.770543\n",
      "[200]\ttraining's auc: 0.803944\tvalid_1's auc: 0.784809\n",
      "[300]\ttraining's auc: 0.808195\tvalid_1's auc: 0.787336\n",
      "[400]\ttraining's auc: 0.809739\tvalid_1's auc: 0.788225\n",
      "[500]\ttraining's auc: 0.810245\tvalid_1's auc: 0.788539\n",
      "Early stopping, best iteration is:\n",
      "[464]\ttraining's auc: 0.810245\tvalid_1's auc: 0.788539\n",
      "Fold  4 AUC : 0.788539\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.783743\tvalid_1's auc: 0.764135\n",
      "[200]\ttraining's auc: 0.805165\tvalid_1's auc: 0.778441\n",
      "[300]\ttraining's auc: 0.809241\tvalid_1's auc: 0.78159\n",
      "[400]\ttraining's auc: 0.810544\tvalid_1's auc: 0.782447\n",
      "Early stopping, best iteration is:\n",
      "[390]\ttraining's auc: 0.810544\tvalid_1's auc: 0.782447\n",
      "Fold  5 AUC : 0.782447\n",
      "Full AUC score 0.787200\n",
      "    4 | 05m25s |    0.78720 |             0.5973 |           119.1125 |           9.5920 |      20.3714 |      0.9413 |       8.5023 |      0.7582 | \n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.782295\tvalid_1's auc: 0.767073\n",
      "[200]\ttraining's auc: 0.806058\tvalid_1's auc: 0.783719\n",
      "[300]\ttraining's auc: 0.819267\tvalid_1's auc: 0.788248\n",
      "[400]\ttraining's auc: 0.829762\tvalid_1's auc: 0.790387\n",
      "[500]\ttraining's auc: 0.83899\tvalid_1's auc: 0.791344\n",
      "[600]\ttraining's auc: 0.847275\tvalid_1's auc: 0.791839\n",
      "[700]\ttraining's auc: 0.85468\tvalid_1's auc: 0.792203\n",
      "[800]\ttraining's auc: 0.862088\tvalid_1's auc: 0.791953\n",
      "Early stopping, best iteration is:\n",
      "[756]\ttraining's auc: 0.858977\tvalid_1's auc: 0.792342\n",
      "Fold  1 AUC : 0.792342\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.780752\tvalid_1's auc: 0.772493\n",
      "[200]\ttraining's auc: 0.804748\tvalid_1's auc: 0.788435\n",
      "[300]\ttraining's auc: 0.818116\tvalid_1's auc: 0.793024\n",
      "[400]\ttraining's auc: 0.828673\tvalid_1's auc: 0.794635\n",
      "[500]\ttraining's auc: 0.838239\tvalid_1's auc: 0.795701\n",
      "[600]\ttraining's auc: 0.846692\tvalid_1's auc: 0.796239\n",
      "[700]\ttraining's auc: 0.854207\tvalid_1's auc: 0.796865\n",
      "[800]\ttraining's auc: 0.861594\tvalid_1's auc: 0.797215\n",
      "[900]\ttraining's auc: 0.868509\tvalid_1's auc: 0.797476\n",
      "[1000]\ttraining's auc: 0.874786\tvalid_1's auc: 0.797696\n",
      "Early stopping, best iteration is:\n",
      "[977]\ttraining's auc: 0.873373\tvalid_1's auc: 0.797828\n",
      "Fold  2 AUC : 0.797828\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.782076\tvalid_1's auc: 0.771047\n",
      "[200]\ttraining's auc: 0.806163\tvalid_1's auc: 0.784606\n",
      "[300]\ttraining's auc: 0.819755\tvalid_1's auc: 0.788849\n",
      "[400]\ttraining's auc: 0.83041\tvalid_1's auc: 0.790583\n",
      "[500]\ttraining's auc: 0.839586\tvalid_1's auc: 0.791111\n",
      "[600]\ttraining's auc: 0.848299\tvalid_1's auc: 0.791501\n",
      "[700]\ttraining's auc: 0.855919\tvalid_1's auc: 0.791684\n",
      "Early stopping, best iteration is:\n",
      "[699]\ttraining's auc: 0.855862\tvalid_1's auc: 0.791692\n",
      "Fold  3 AUC : 0.791692\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.782091\tvalid_1's auc: 0.771186\n",
      "[200]\ttraining's auc: 0.80619\tvalid_1's auc: 0.785548\n",
      "[300]\ttraining's auc: 0.819373\tvalid_1's auc: 0.789638\n",
      "[400]\ttraining's auc: 0.830115\tvalid_1's auc: 0.791767\n",
      "[500]\ttraining's auc: 0.839205\tvalid_1's auc: 0.792308\n",
      "[600]\ttraining's auc: 0.847851\tvalid_1's auc: 0.793101\n",
      "[700]\ttraining's auc: 0.855721\tvalid_1's auc: 0.793358\n",
      "[800]\ttraining's auc: 0.862947\tvalid_1's auc: 0.793418\n",
      "[900]\ttraining's auc: 0.869776\tvalid_1's auc: 0.793534\n",
      "[1000]\ttraining's auc: 0.876093\tvalid_1's auc: 0.793391\n",
      "Early stopping, best iteration is:\n",
      "[957]\ttraining's auc: 0.873415\tvalid_1's auc: 0.793652\n",
      "Fold  4 AUC : 0.793652\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.782796\tvalid_1's auc: 0.763508\n",
      "[200]\ttraining's auc: 0.806631\tvalid_1's auc: 0.779244\n",
      "[300]\ttraining's auc: 0.819656\tvalid_1's auc: 0.783845\n",
      "[400]\ttraining's auc: 0.830311\tvalid_1's auc: 0.785625\n",
      "[500]\ttraining's auc: 0.839572\tvalid_1's auc: 0.786749\n",
      "[600]\ttraining's auc: 0.848209\tvalid_1's auc: 0.787359\n",
      "[700]\ttraining's auc: 0.856124\tvalid_1's auc: 0.787436\n",
      "Early stopping, best iteration is:\n",
      "[637]\ttraining's auc: 0.851203\tvalid_1's auc: 0.787633\n",
      "Fold  5 AUC : 0.787633\n",
      "Full AUC score 0.792628\n",
      "    5 | 10m21s | \u001b[35m   0.79263\u001b[0m | \u001b[32m            0.8281\u001b[0m | \u001b[32m           25.7119\u001b[0m | \u001b[32m          0.6667\u001b[0m | \u001b[32m     20.5985\u001b[0m | \u001b[32m     8.0709\u001b[0m | \u001b[32m      0.0376\u001b[0m | \u001b[32m     0.6215\u001b[0m | \n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.817614\tvalid_1's auc: 0.775251\n",
      "[200]\ttraining's auc: 0.843985\tvalid_1's auc: 0.786559\n",
      "[300]\ttraining's auc: 0.858604\tvalid_1's auc: 0.789525\n",
      "[400]\ttraining's auc: 0.870529\tvalid_1's auc: 0.790732\n",
      "[500]\ttraining's auc: 0.881418\tvalid_1's auc: 0.791368\n",
      "[600]\ttraining's auc: 0.89101\tvalid_1's auc: 0.791484\n",
      "Early stopping, best iteration is:\n",
      "[572]\ttraining's auc: 0.88798\tvalid_1's auc: 0.791709\n",
      "Fold  1 AUC : 0.791709\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.816137\tvalid_1's auc: 0.779889\n",
      "[200]\ttraining's auc: 0.842256\tvalid_1's auc: 0.791302\n",
      "[300]\ttraining's auc: 0.856747\tvalid_1's auc: 0.793915\n",
      "[400]\ttraining's auc: 0.868664\tvalid_1's auc: 0.795783\n",
      "[500]\ttraining's auc: 0.879321\tvalid_1's auc: 0.796223\n",
      "[600]\ttraining's auc: 0.889063\tvalid_1's auc: 0.796255\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[700]\ttraining's auc: 0.897999\tvalid_1's auc: 0.79581\n",
      "Early stopping, best iteration is:\n",
      "[610]\ttraining's auc: 0.889925\tvalid_1's auc: 0.79633\n",
      "Fold  2 AUC : 0.796330\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.816266\tvalid_1's auc: 0.777684\n",
      "[200]\ttraining's auc: 0.842446\tvalid_1's auc: 0.787825\n",
      "[300]\ttraining's auc: 0.857172\tvalid_1's auc: 0.789998\n",
      "[400]\ttraining's auc: 0.869414\tvalid_1's auc: 0.791082\n",
      "[500]\ttraining's auc: 0.880152\tvalid_1's auc: 0.79132\n",
      "[600]\ttraining's auc: 0.88969\tvalid_1's auc: 0.791373\n",
      "Early stopping, best iteration is:\n",
      "[589]\ttraining's auc: 0.888937\tvalid_1's auc: 0.791433\n",
      "Fold  3 AUC : 0.791433\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.815968\tvalid_1's auc: 0.777774\n",
      "[200]\ttraining's auc: 0.842581\tvalid_1's auc: 0.788499\n",
      "[300]\ttraining's auc: 0.85693\tvalid_1's auc: 0.791017\n",
      "[400]\ttraining's auc: 0.869361\tvalid_1's auc: 0.791582\n",
      "[500]\ttraining's auc: 0.879847\tvalid_1's auc: 0.792228\n",
      "[600]\ttraining's auc: 0.890217\tvalid_1's auc: 0.791787\n",
      "Early stopping, best iteration is:\n",
      "[511]\ttraining's auc: 0.880914\tvalid_1's auc: 0.792241\n",
      "Fold  4 AUC : 0.792241\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.816387\tvalid_1's auc: 0.77231\n",
      "[200]\ttraining's auc: 0.842067\tvalid_1's auc: 0.783723\n",
      "[300]\ttraining's auc: 0.855858\tvalid_1's auc: 0.787027\n",
      "[400]\ttraining's auc: 0.868232\tvalid_1's auc: 0.788219\n",
      "[500]\ttraining's auc: 0.878923\tvalid_1's auc: 0.788811\n",
      "[600]\ttraining's auc: 0.888058\tvalid_1's auc: 0.788768\n",
      "[700]\ttraining's auc: 0.896879\tvalid_1's auc: 0.789068\n",
      "Early stopping, best iteration is:\n",
      "[696]\ttraining's auc: 0.8966\tvalid_1's auc: 0.789168\n",
      "Fold  5 AUC : 0.789168\n",
      "Full AUC score 0.792144\n",
      "    6 | 12m28s |    0.79214 |             0.9277 |           100.7248 |           0.1718 |      88.6431 |      0.0346 |       0.2091 |      0.8788 | \n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.784051\tvalid_1's auc: 0.766731\n",
      "[200]\ttraining's auc: 0.808281\tvalid_1's auc: 0.782728\n",
      "[300]\ttraining's auc: 0.821086\tvalid_1's auc: 0.787206\n",
      "[400]\ttraining's auc: 0.831388\tvalid_1's auc: 0.789201\n",
      "[500]\ttraining's auc: 0.840522\tvalid_1's auc: 0.790278\n",
      "[600]\ttraining's auc: 0.848634\tvalid_1's auc: 0.790941\n",
      "[700]\ttraining's auc: 0.856096\tvalid_1's auc: 0.791312\n",
      "[800]\ttraining's auc: 0.863042\tvalid_1's auc: 0.791789\n",
      "[900]\ttraining's auc: 0.869455\tvalid_1's auc: 0.791689\n",
      "Early stopping, best iteration is:\n",
      "[806]\ttraining's auc: 0.863379\tvalid_1's auc: 0.791852\n",
      "Fold  1 AUC : 0.791852\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.782144\tvalid_1's auc: 0.772053\n",
      "[200]\ttraining's auc: 0.806804\tvalid_1's auc: 0.787985\n",
      "[300]\ttraining's auc: 0.819673\tvalid_1's auc: 0.792863\n",
      "[400]\ttraining's auc: 0.830052\tvalid_1's auc: 0.794657\n",
      "[500]\ttraining's auc: 0.838602\tvalid_1's auc: 0.795699\n",
      "[600]\ttraining's auc: 0.846768\tvalid_1's auc: 0.796432\n",
      "[700]\ttraining's auc: 0.854291\tvalid_1's auc: 0.796522\n",
      "[800]\ttraining's auc: 0.861314\tvalid_1's auc: 0.796604\n",
      "[900]\ttraining's auc: 0.867888\tvalid_1's auc: 0.796638\n",
      "Early stopping, best iteration is:\n",
      "[862]\ttraining's auc: 0.865486\tvalid_1's auc: 0.796807\n",
      "Fold  2 AUC : 0.796807\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.783141\tvalid_1's auc: 0.770255\n",
      "[200]\ttraining's auc: 0.808032\tvalid_1's auc: 0.784378\n",
      "[300]\ttraining's auc: 0.820967\tvalid_1's auc: 0.788183\n",
      "[400]\ttraining's auc: 0.831245\tvalid_1's auc: 0.790082\n",
      "[500]\ttraining's auc: 0.840118\tvalid_1's auc: 0.790941\n",
      "[600]\ttraining's auc: 0.848298\tvalid_1's auc: 0.791401\n",
      "[700]\ttraining's auc: 0.855632\tvalid_1's auc: 0.791577\n",
      "[800]\ttraining's auc: 0.862585\tvalid_1's auc: 0.791601\n",
      "Early stopping, best iteration is:\n",
      "[723]\ttraining's auc: 0.857199\tvalid_1's auc: 0.791737\n",
      "Fold  3 AUC : 0.791737\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.783306\tvalid_1's auc: 0.770817\n",
      "[200]\ttraining's auc: 0.807883\tvalid_1's auc: 0.785543\n",
      "[300]\ttraining's auc: 0.820981\tvalid_1's auc: 0.789764\n",
      "[400]\ttraining's auc: 0.831305\tvalid_1's auc: 0.791356\n",
      "[500]\ttraining's auc: 0.840422\tvalid_1's auc: 0.792207\n",
      "[600]\ttraining's auc: 0.848598\tvalid_1's auc: 0.79276\n",
      "[700]\ttraining's auc: 0.856007\tvalid_1's auc: 0.793205\n",
      "[800]\ttraining's auc: 0.862975\tvalid_1's auc: 0.793668\n",
      "[900]\ttraining's auc: 0.869483\tvalid_1's auc: 0.79384\n",
      "[1000]\ttraining's auc: 0.87566\tvalid_1's auc: 0.793371\n",
      "Early stopping, best iteration is:\n",
      "[906]\ttraining's auc: 0.869925\tvalid_1's auc: 0.793876\n",
      "Fold  4 AUC : 0.793876\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.785176\tvalid_1's auc: 0.76419\n",
      "[200]\ttraining's auc: 0.80897\tvalid_1's auc: 0.779052\n",
      "[300]\ttraining's auc: 0.821859\tvalid_1's auc: 0.783923\n",
      "[400]\ttraining's auc: 0.832096\tvalid_1's auc: 0.786186\n",
      "[500]\ttraining's auc: 0.841008\tvalid_1's auc: 0.787207\n",
      "[600]\ttraining's auc: 0.849096\tvalid_1's auc: 0.787873\n",
      "[700]\ttraining's auc: 0.856565\tvalid_1's auc: 0.78815\n",
      "[800]\ttraining's auc: 0.863632\tvalid_1's auc: 0.788498\n",
      "[900]\ttraining's auc: 0.870216\tvalid_1's auc: 0.788216\n",
      "Early stopping, best iteration is:\n",
      "[801]\ttraining's auc: 0.863697\tvalid_1's auc: 0.788505\n",
      "Fold  5 AUC : 0.788505\n",
      "Full AUC score 0.792539\n",
      "    7 | 13m18s |    0.79254 |             0.6163 |           113.5434 |           0.0951 |      22.9561 |      8.9642 |       1.1482 |      0.9040 | \n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.795191\tvalid_1's auc: 0.771858\n",
      "[200]\ttraining's auc: 0.820563\tvalid_1's auc: 0.784344\n",
      "[300]\ttraining's auc: 0.835529\tvalid_1's auc: 0.787986\n",
      "[400]\ttraining's auc: 0.849154\tvalid_1's auc: 0.789303\n",
      "[500]\ttraining's auc: 0.860967\tvalid_1's auc: 0.789812\n",
      "[600]\ttraining's auc: 0.871722\tvalid_1's auc: 0.790135\n",
      "[700]\ttraining's auc: 0.881865\tvalid_1's auc: 0.789729\n",
      "Early stopping, best iteration is:\n",
      "[610]\ttraining's auc: 0.872799\tvalid_1's auc: 0.790187\n",
      "Fold  1 AUC : 0.790187\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.793016\tvalid_1's auc: 0.776638\n",
      "[200]\ttraining's auc: 0.818862\tvalid_1's auc: 0.788978\n",
      "[300]\ttraining's auc: 0.834139\tvalid_1's auc: 0.792557\n",
      "[400]\ttraining's auc: 0.847722\tvalid_1's auc: 0.79407\n",
      "[500]\ttraining's auc: 0.859811\tvalid_1's auc: 0.794797\n",
      "[600]\ttraining's auc: 0.870433\tvalid_1's auc: 0.795333\n",
      "[700]\ttraining's auc: 0.880028\tvalid_1's auc: 0.795697\n",
      "[800]\ttraining's auc: 0.889116\tvalid_1's auc: 0.795655\n",
      "Early stopping, best iteration is:\n",
      "[768]\ttraining's auc: 0.886351\tvalid_1's auc: 0.795854\n",
      "Fold  2 AUC : 0.795854\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.794239\tvalid_1's auc: 0.773482\n",
      "[200]\ttraining's auc: 0.820077\tvalid_1's auc: 0.784843\n",
      "[300]\ttraining's auc: 0.835759\tvalid_1's auc: 0.788061\n",
      "[400]\ttraining's auc: 0.848745\tvalid_1's auc: 0.789277\n",
      "[500]\ttraining's auc: 0.860148\tvalid_1's auc: 0.790046\n",
      "[600]\ttraining's auc: 0.871072\tvalid_1's auc: 0.790308\n",
      "Early stopping, best iteration is:\n",
      "[565]\ttraining's auc: 0.867408\tvalid_1's auc: 0.790564\n",
      "Fold  3 AUC : 0.790564\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.79438\tvalid_1's auc: 0.774192\n",
      "[200]\ttraining's auc: 0.820087\tvalid_1's auc: 0.786423\n",
      "[300]\ttraining's auc: 0.835572\tvalid_1's auc: 0.78982\n",
      "[400]\ttraining's auc: 0.848424\tvalid_1's auc: 0.790716\n",
      "[500]\ttraining's auc: 0.860792\tvalid_1's auc: 0.790891\n",
      "Early stopping, best iteration is:\n",
      "[430]\ttraining's auc: 0.852202\tvalid_1's auc: 0.791037\n",
      "Fold  4 AUC : 0.791037\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.794675\tvalid_1's auc: 0.768436\n",
      "[200]\ttraining's auc: 0.820393\tvalid_1's auc: 0.781683\n",
      "[300]\ttraining's auc: 0.835573\tvalid_1's auc: 0.784904\n",
      "[400]\ttraining's auc: 0.848666\tvalid_1's auc: 0.786059\n",
      "[500]\ttraining's auc: 0.860076\tvalid_1's auc: 0.786515\n",
      "[600]\ttraining's auc: 0.871121\tvalid_1's auc: 0.78646\n",
      "Early stopping, best iteration is:\n",
      "[514]\ttraining's auc: 0.861623\tvalid_1's auc: 0.786573\n",
      "Fold  5 AUC : 0.786573\n",
      "Full AUC score 0.790822\n",
      "    8 | 11m09s |    0.79082 |             0.8102 |           119.5420 |           0.2182 |      96.2088 |      8.9841 |       9.8512 |      0.6194 | \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.800262\tvalid_1's auc: 0.771963\n",
      "[200]\ttraining's auc: 0.82821\tvalid_1's auc: 0.785056\n",
      "[300]\ttraining's auc: 0.845556\tvalid_1's auc: 0.788722\n",
      "[400]\ttraining's auc: 0.860672\tvalid_1's auc: 0.790755\n",
      "[500]\ttraining's auc: 0.873508\tvalid_1's auc: 0.791733\n",
      "[600]\ttraining's auc: 0.885496\tvalid_1's auc: 0.791851\n",
      "Early stopping, best iteration is:\n",
      "[548]\ttraining's auc: 0.879293\tvalid_1's auc: 0.791942\n",
      "Fold  1 AUC : 0.791942\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.798403\tvalid_1's auc: 0.777946\n",
      "[200]\ttraining's auc: 0.827326\tvalid_1's auc: 0.790531\n",
      "[300]\ttraining's auc: 0.844646\tvalid_1's auc: 0.793879\n",
      "[400]\ttraining's auc: 0.858799\tvalid_1's auc: 0.795587\n",
      "[500]\ttraining's auc: 0.871574\tvalid_1's auc: 0.795953\n",
      "[600]\ttraining's auc: 0.883237\tvalid_1's auc: 0.796335\n",
      "[700]\ttraining's auc: 0.894244\tvalid_1's auc: 0.796303\n",
      "Early stopping, best iteration is:\n",
      "[608]\ttraining's auc: 0.884208\tvalid_1's auc: 0.796433\n",
      "Fold  2 AUC : 0.796433\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.799299\tvalid_1's auc: 0.77556\n",
      "[200]\ttraining's auc: 0.827951\tvalid_1's auc: 0.78692\n",
      "[300]\ttraining's auc: 0.845146\tvalid_1's auc: 0.789704\n",
      "[400]\ttraining's auc: 0.859947\tvalid_1's auc: 0.791009\n",
      "[500]\ttraining's auc: 0.873009\tvalid_1's auc: 0.791504\n",
      "[600]\ttraining's auc: 0.884851\tvalid_1's auc: 0.791774\n",
      "[700]\ttraining's auc: 0.895795\tvalid_1's auc: 0.791494\n",
      "Early stopping, best iteration is:\n",
      "[647]\ttraining's auc: 0.890101\tvalid_1's auc: 0.791983\n",
      "Fold  3 AUC : 0.791983\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.799133\tvalid_1's auc: 0.775419\n",
      "[200]\ttraining's auc: 0.827794\tvalid_1's auc: 0.78707\n",
      "[300]\ttraining's auc: 0.845192\tvalid_1's auc: 0.790315\n",
      "[400]\ttraining's auc: 0.859402\tvalid_1's auc: 0.792085\n",
      "[500]\ttraining's auc: 0.872486\tvalid_1's auc: 0.792502\n",
      "Early stopping, best iteration is:\n",
      "[492]\ttraining's auc: 0.871539\tvalid_1's auc: 0.79258\n",
      "Fold  4 AUC : 0.792580\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.800495\tvalid_1's auc: 0.770212\n",
      "[200]\ttraining's auc: 0.828275\tvalid_1's auc: 0.782925\n",
      "[300]\ttraining's auc: 0.844274\tvalid_1's auc: 0.786328\n",
      "[400]\ttraining's auc: 0.859129\tvalid_1's auc: 0.787579\n",
      "[500]\ttraining's auc: 0.872208\tvalid_1's auc: 0.787899\n",
      "[600]\ttraining's auc: 0.883714\tvalid_1's auc: 0.788162\n",
      "Early stopping, best iteration is:\n",
      "[589]\ttraining's auc: 0.882454\tvalid_1's auc: 0.7883\n",
      "Fold  5 AUC : 0.788300\n",
      "Full AUC score 0.792221\n",
      "    9 | 12m43s |    0.79222 |             0.5115 |            64.5362 |           0.1620 |      45.7948 |      9.6616 |       0.5890 |      0.9707 | \n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.783999\tvalid_1's auc: 0.767434\n",
      "[200]\ttraining's auc: 0.809494\tvalid_1's auc: 0.78355\n",
      "[300]\ttraining's auc: 0.823483\tvalid_1's auc: 0.787583\n",
      "[400]\ttraining's auc: 0.835136\tvalid_1's auc: 0.789416\n",
      "[500]\ttraining's auc: 0.845156\tvalid_1's auc: 0.790706\n",
      "[600]\ttraining's auc: 0.854377\tvalid_1's auc: 0.790868\n",
      "[700]\ttraining's auc: 0.86246\tvalid_1's auc: 0.791387\n",
      "[800]\ttraining's auc: 0.870536\tvalid_1's auc: 0.791699\n",
      "[900]\ttraining's auc: 0.878024\tvalid_1's auc: 0.791383\n",
      "Early stopping, best iteration is:\n",
      "[835]\ttraining's auc: 0.873102\tvalid_1's auc: 0.791783\n",
      "Fold  1 AUC : 0.791783\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.782103\tvalid_1's auc: 0.772775\n",
      "[200]\ttraining's auc: 0.808309\tvalid_1's auc: 0.788826\n",
      "[300]\ttraining's auc: 0.822552\tvalid_1's auc: 0.792865\n",
      "[400]\ttraining's auc: 0.834193\tvalid_1's auc: 0.794745\n",
      "[500]\ttraining's auc: 0.844328\tvalid_1's auc: 0.795723\n",
      "[600]\ttraining's auc: 0.853218\tvalid_1's auc: 0.796251\n",
      "[700]\ttraining's auc: 0.86121\tvalid_1's auc: 0.796519\n",
      "[800]\ttraining's auc: 0.869256\tvalid_1's auc: 0.796661\n",
      "Early stopping, best iteration is:\n",
      "[796]\ttraining's auc: 0.868949\tvalid_1's auc: 0.796752\n",
      "Fold  2 AUC : 0.796752\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.783964\tvalid_1's auc: 0.770834\n",
      "[200]\ttraining's auc: 0.809437\tvalid_1's auc: 0.784642\n",
      "[300]\ttraining's auc: 0.823779\tvalid_1's auc: 0.788241\n",
      "[400]\ttraining's auc: 0.835508\tvalid_1's auc: 0.789534\n",
      "[500]\ttraining's auc: 0.845369\tvalid_1's auc: 0.790405\n",
      "[600]\ttraining's auc: 0.854754\tvalid_1's auc: 0.790911\n",
      "Early stopping, best iteration is:\n",
      "[599]\ttraining's auc: 0.854633\tvalid_1's auc: 0.790945\n",
      "Fold  3 AUC : 0.790945\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.783351\tvalid_1's auc: 0.771677\n",
      "[200]\ttraining's auc: 0.809226\tvalid_1's auc: 0.785469\n",
      "[300]\ttraining's auc: 0.82355\tvalid_1's auc: 0.788947\n",
      "[400]\ttraining's auc: 0.835211\tvalid_1's auc: 0.790292\n",
      "[500]\ttraining's auc: 0.845068\tvalid_1's auc: 0.791046\n",
      "[600]\ttraining's auc: 0.854011\tvalid_1's auc: 0.791446\n",
      "[700]\ttraining's auc: 0.86255\tvalid_1's auc: 0.791557\n",
      "[800]\ttraining's auc: 0.870383\tvalid_1's auc: 0.791301\n",
      "Early stopping, best iteration is:\n",
      "[716]\ttraining's auc: 0.86383\tvalid_1's auc: 0.79175\n",
      "Fold  4 AUC : 0.791750\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.785582\tvalid_1's auc: 0.765012\n",
      "[200]\ttraining's auc: 0.810428\tvalid_1's auc: 0.780785\n",
      "[300]\ttraining's auc: 0.824534\tvalid_1's auc: 0.784817\n",
      "[400]\ttraining's auc: 0.836104\tvalid_1's auc: 0.786499\n",
      "[500]\ttraining's auc: 0.845985\tvalid_1's auc: 0.787369\n",
      "[600]\ttraining's auc: 0.855208\tvalid_1's auc: 0.787864\n",
      "[700]\ttraining's auc: 0.863853\tvalid_1's auc: 0.788184\n",
      "[800]\ttraining's auc: 0.87137\tvalid_1's auc: 0.787978\n",
      "Early stopping, best iteration is:\n",
      "[701]\ttraining's auc: 0.863919\tvalid_1's auc: 0.788212\n",
      "Fold  5 AUC : 0.788212\n",
      "Full AUC score 0.791873\n",
      "   10 | 10m25s |    0.79187 |             0.9496 |            57.4442 |           0.0323 |      21.7735 |      0.9982 |       1.3918 |      0.6600 | \n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.78316\tvalid_1's auc: 0.767311\n",
      "[200]\ttraining's auc: 0.80668\tvalid_1's auc: 0.783239\n",
      "[300]\ttraining's auc: 0.819464\tvalid_1's auc: 0.787889\n",
      "[400]\ttraining's auc: 0.829405\tvalid_1's auc: 0.78996\n",
      "[500]\ttraining's auc: 0.837909\tvalid_1's auc: 0.79083\n",
      "[600]\ttraining's auc: 0.845491\tvalid_1's auc: 0.791651\n",
      "[700]\ttraining's auc: 0.852444\tvalid_1's auc: 0.791761\n",
      "[800]\ttraining's auc: 0.859114\tvalid_1's auc: 0.792046\n",
      "[900]\ttraining's auc: 0.865175\tvalid_1's auc: 0.792285\n",
      "[1000]\ttraining's auc: 0.870805\tvalid_1's auc: 0.792129\n",
      "Early stopping, best iteration is:\n",
      "[901]\ttraining's auc: 0.865211\tvalid_1's auc: 0.792288\n",
      "Fold  1 AUC : 0.792288\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.781556\tvalid_1's auc: 0.771537\n",
      "[200]\ttraining's auc: 0.805787\tvalid_1's auc: 0.787134\n",
      "[300]\ttraining's auc: 0.81836\tvalid_1's auc: 0.792213\n",
      "[400]\ttraining's auc: 0.828304\tvalid_1's auc: 0.793748\n",
      "[500]\ttraining's auc: 0.836913\tvalid_1's auc: 0.794651\n",
      "[600]\ttraining's auc: 0.844575\tvalid_1's auc: 0.795388\n",
      "[700]\ttraining's auc: 0.851415\tvalid_1's auc: 0.795905\n",
      "[800]\ttraining's auc: 0.857969\tvalid_1's auc: 0.796091\n",
      "[900]\ttraining's auc: 0.863948\tvalid_1's auc: 0.796391\n",
      "[1000]\ttraining's auc: 0.869414\tvalid_1's auc: 0.796367\n",
      "Early stopping, best iteration is:\n",
      "[911]\ttraining's auc: 0.864518\tvalid_1's auc: 0.796439\n",
      "Fold  2 AUC : 0.796439\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.782475\tvalid_1's auc: 0.770641\n",
      "[200]\ttraining's auc: 0.806729\tvalid_1's auc: 0.78405\n",
      "[300]\ttraining's auc: 0.819457\tvalid_1's auc: 0.788589\n",
      "[400]\ttraining's auc: 0.829377\tvalid_1's auc: 0.79024\n",
      "[500]\ttraining's auc: 0.838022\tvalid_1's auc: 0.791246\n",
      "[600]\ttraining's auc: 0.845753\tvalid_1's auc: 0.791545\n",
      "[700]\ttraining's auc: 0.852751\tvalid_1's auc: 0.791973\n",
      "[800]\ttraining's auc: 0.859023\tvalid_1's auc: 0.792352\n",
      "[900]\ttraining's auc: 0.864854\tvalid_1's auc: 0.792447\n",
      "[1000]\ttraining's auc: 0.870541\tvalid_1's auc: 0.792332\n",
      "Early stopping, best iteration is:\n",
      "[957]\ttraining's auc: 0.868189\tvalid_1's auc: 0.792493\n",
      "Fold  3 AUC : 0.792493\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.782288\tvalid_1's auc: 0.770343\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[200]\ttraining's auc: 0.806705\tvalid_1's auc: 0.785065\n",
      "[300]\ttraining's auc: 0.819436\tvalid_1's auc: 0.789643\n",
      "[400]\ttraining's auc: 0.829244\tvalid_1's auc: 0.791402\n",
      "[500]\ttraining's auc: 0.837522\tvalid_1's auc: 0.792132\n",
      "[600]\ttraining's auc: 0.845067\tvalid_1's auc: 0.792764\n",
      "[700]\ttraining's auc: 0.852202\tvalid_1's auc: 0.793399\n",
      "[800]\ttraining's auc: 0.858625\tvalid_1's auc: 0.793588\n",
      "[900]\ttraining's auc: 0.864715\tvalid_1's auc: 0.793669\n",
      "Early stopping, best iteration is:\n",
      "[831]\ttraining's auc: 0.860524\tvalid_1's auc: 0.793738\n",
      "Fold  4 AUC : 0.793738\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.784168\tvalid_1's auc: 0.765619\n",
      "[200]\ttraining's auc: 0.807559\tvalid_1's auc: 0.780592\n",
      "[300]\ttraining's auc: 0.820025\tvalid_1's auc: 0.785538\n",
      "[400]\ttraining's auc: 0.830176\tvalid_1's auc: 0.788019\n",
      "[500]\ttraining's auc: 0.838719\tvalid_1's auc: 0.789129\n",
      "[600]\ttraining's auc: 0.846344\tvalid_1's auc: 0.789795\n",
      "[700]\ttraining's auc: 0.853291\tvalid_1's auc: 0.790337\n",
      "[800]\ttraining's auc: 0.859879\tvalid_1's auc: 0.790187\n",
      "Early stopping, best iteration is:\n",
      "[741]\ttraining's auc: 0.856054\tvalid_1's auc: 0.79043\n",
      "Fold  5 AUC : 0.790430\n",
      "Full AUC score 0.793068\n",
      "   11 | 13m53s | \u001b[35m   0.79307\u001b[0m | \u001b[32m            0.5802\u001b[0m | \u001b[32m           20.3369\u001b[0m | \u001b[32m          0.7650\u001b[0m | \u001b[32m     21.1524\u001b[0m | \u001b[32m     9.7570\u001b[0m | \u001b[32m      8.7767\u001b[0m | \u001b[32m     0.9370\u001b[0m | \n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.797295\tvalid_1's auc: 0.772626\n",
      "[200]\ttraining's auc: 0.820703\tvalid_1's auc: 0.784127\n",
      "[300]\ttraining's auc: 0.835563\tvalid_1's auc: 0.787283\n",
      "[400]\ttraining's auc: 0.848135\tvalid_1's auc: 0.788682\n",
      "[500]\ttraining's auc: 0.859781\tvalid_1's auc: 0.789366\n",
      "[600]\ttraining's auc: 0.869952\tvalid_1's auc: 0.789574\n",
      "Early stopping, best iteration is:\n",
      "[570]\ttraining's auc: 0.866746\tvalid_1's auc: 0.78993\n",
      "Fold  1 AUC : 0.789930\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.795354\tvalid_1's auc: 0.777016\n",
      "[200]\ttraining's auc: 0.819455\tvalid_1's auc: 0.788959\n",
      "[300]\ttraining's auc: 0.834266\tvalid_1's auc: 0.792864\n",
      "[400]\ttraining's auc: 0.846811\tvalid_1's auc: 0.794628\n",
      "[500]\ttraining's auc: 0.858278\tvalid_1's auc: 0.795326\n",
      "[600]\ttraining's auc: 0.868726\tvalid_1's auc: 0.795454\n",
      "Early stopping, best iteration is:\n",
      "[525]\ttraining's auc: 0.861058\tvalid_1's auc: 0.795617\n",
      "Fold  2 AUC : 0.795617\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.796977\tvalid_1's auc: 0.775554\n",
      "[200]\ttraining's auc: 0.820818\tvalid_1's auc: 0.785917\n",
      "[300]\ttraining's auc: 0.835684\tvalid_1's auc: 0.788523\n",
      "[400]\ttraining's auc: 0.847878\tvalid_1's auc: 0.789927\n",
      "[500]\ttraining's auc: 0.858849\tvalid_1's auc: 0.790164\n",
      "Early stopping, best iteration is:\n",
      "[447]\ttraining's auc: 0.853243\tvalid_1's auc: 0.79024\n",
      "Fold  3 AUC : 0.790240\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.796586\tvalid_1's auc: 0.775988\n",
      "[200]\ttraining's auc: 0.820597\tvalid_1's auc: 0.786266\n",
      "[300]\ttraining's auc: 0.835734\tvalid_1's auc: 0.789297\n",
      "[400]\ttraining's auc: 0.848614\tvalid_1's auc: 0.79042\n",
      "[500]\ttraining's auc: 0.859488\tvalid_1's auc: 0.790702\n",
      "Early stopping, best iteration is:\n",
      "[472]\ttraining's auc: 0.856566\tvalid_1's auc: 0.790951\n",
      "Fold  4 AUC : 0.790951\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.796611\tvalid_1's auc: 0.769476\n",
      "[200]\ttraining's auc: 0.820509\tvalid_1's auc: 0.781566\n",
      "[300]\ttraining's auc: 0.835754\tvalid_1's auc: 0.785368\n",
      "[400]\ttraining's auc: 0.848691\tvalid_1's auc: 0.786403\n",
      "[500]\ttraining's auc: 0.859888\tvalid_1's auc: 0.786434\n",
      "[600]\ttraining's auc: 0.870503\tvalid_1's auc: 0.786379\n",
      "Early stopping, best iteration is:\n",
      "[517]\ttraining's auc: 0.86173\tvalid_1's auc: 0.786543\n",
      "Fold  5 AUC : 0.786543\n",
      "Full AUC score 0.790646\n",
      "   12 | 09m32s |    0.79065 |             0.7254 |           119.8771 |           0.1367 |      60.1944 |      6.8098 |       0.7218 |      0.5510 | \n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.810405\tvalid_1's auc: 0.773464\n",
      "[200]\ttraining's auc: 0.843748\tvalid_1's auc: 0.784756\n",
      "[300]\ttraining's auc: 0.865462\tvalid_1's auc: 0.787781\n",
      "[400]\ttraining's auc: 0.885103\tvalid_1's auc: 0.788698\n",
      "[500]\ttraining's auc: 0.901538\tvalid_1's auc: 0.788887\n",
      "Early stopping, best iteration is:\n",
      "[426]\ttraining's auc: 0.889306\tvalid_1's auc: 0.78923\n",
      "Fold  1 AUC : 0.789230\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.808055\tvalid_1's auc: 0.779998\n",
      "[200]\ttraining's auc: 0.841517\tvalid_1's auc: 0.791184\n",
      "[300]\ttraining's auc: 0.862917\tvalid_1's auc: 0.793504\n",
      "[400]\ttraining's auc: 0.882285\tvalid_1's auc: 0.794292\n",
      "[500]\ttraining's auc: 0.899527\tvalid_1's auc: 0.794473\n",
      "Early stopping, best iteration is:\n",
      "[481]\ttraining's auc: 0.896635\tvalid_1's auc: 0.794636\n",
      "Fold  2 AUC : 0.794636\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.809963\tvalid_1's auc: 0.777537\n",
      "[200]\ttraining's auc: 0.842392\tvalid_1's auc: 0.786322\n",
      "[300]\ttraining's auc: 0.864974\tvalid_1's auc: 0.789041\n",
      "[400]\ttraining's auc: 0.884293\tvalid_1's auc: 0.789304\n",
      "[500]\ttraining's auc: 0.900825\tvalid_1's auc: 0.789784\n",
      "[600]\ttraining's auc: 0.915771\tvalid_1's auc: 0.789176\n",
      "Early stopping, best iteration is:\n",
      "[511]\ttraining's auc: 0.902629\tvalid_1's auc: 0.790034\n",
      "Fold  3 AUC : 0.790034\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.81006\tvalid_1's auc: 0.777326\n",
      "[200]\ttraining's auc: 0.84317\tvalid_1's auc: 0.787852\n",
      "[300]\ttraining's auc: 0.864457\tvalid_1's auc: 0.790026\n",
      "[400]\ttraining's auc: 0.883631\tvalid_1's auc: 0.790256\n",
      "[500]\ttraining's auc: 0.899969\tvalid_1's auc: 0.790583\n",
      "Early stopping, best iteration is:\n",
      "[481]\ttraining's auc: 0.896599\tvalid_1's auc: 0.790895\n",
      "Fold  4 AUC : 0.790895\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.809979\tvalid_1's auc: 0.770252\n",
      "[200]\ttraining's auc: 0.842717\tvalid_1's auc: 0.782599\n",
      "[300]\ttraining's auc: 0.864567\tvalid_1's auc: 0.785023\n",
      "[400]\ttraining's auc: 0.883497\tvalid_1's auc: 0.785888\n",
      "[500]\ttraining's auc: 0.89921\tvalid_1's auc: 0.785514\n",
      "Early stopping, best iteration is:\n",
      "[429]\ttraining's auc: 0.888241\tvalid_1's auc: 0.785953\n",
      "Fold  5 AUC : 0.785953\n",
      "Full AUC score 0.790145\n",
      "   13 | 10m58s |    0.79015 |             0.9695 |            44.0895 |           0.0740 |      99.7662 |      8.1230 |       0.5660 |      0.6265 | \n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.782598\tvalid_1's auc: 0.766058\n",
      "[200]\ttraining's auc: 0.805843\tvalid_1's auc: 0.781951\n",
      "[300]\ttraining's auc: 0.818208\tvalid_1's auc: 0.787029\n",
      "[400]\ttraining's auc: 0.827901\tvalid_1's auc: 0.789313\n",
      "[500]\ttraining's auc: 0.83638\tvalid_1's auc: 0.790193\n",
      "[600]\ttraining's auc: 0.843923\tvalid_1's auc: 0.790799\n",
      "[700]\ttraining's auc: 0.851\tvalid_1's auc: 0.791383\n",
      "[800]\ttraining's auc: 0.857518\tvalid_1's auc: 0.791856\n",
      "Early stopping, best iteration is:\n",
      "[790]\ttraining's auc: 0.856889\tvalid_1's auc: 0.791914\n",
      "Fold  1 AUC : 0.791914\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[100]\ttraining's auc: 0.781056\tvalid_1's auc: 0.771136\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-fc72c263d6a0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlgbBO\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmaximize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minit_points\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mn_iter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m15\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/bayes_opt/bayesian_optimization.pyc\u001b[0m in \u001b[0;36mmaximize\u001b[0;34m(self, init_points, n_iter, acq, kappa, xi, **gp_params)\u001b[0m\n\u001b[1;32m    277\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    278\u001b[0m             \u001b[0;31m# Append most recently generated values to X and Y arrays\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 279\u001b[0;31m             \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobserve_point\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_max\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    280\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    281\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplog\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprint_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpwarning\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/bayes_opt/target_space.pyc\u001b[0m in \u001b[0;36mobserve_point\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    137\u001b[0m             \u001b[0;31m# measure the target function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    138\u001b[0m             \u001b[0mparams\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 139\u001b[0;31m             \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtarget_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    140\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_observation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    141\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-8-19a93491bcd1>\u001b[0m in \u001b[0;36mlgbcv\u001b[0;34m(num_leaves, colsample_bytree, subsample, reg_alpha, reg_lambda, min_split_gain, min_child_weight)\u001b[0m\n\u001b[1;32m     22\u001b[0m         clf.fit(trn_x, trn_y, \n\u001b[1;32m     23\u001b[0m             \u001b[0meval_set\u001b[0m\u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrn_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrn_y\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mval_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_y\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m             \u001b[0meval_metric\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'auc'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mearly_stopping_rounds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m  \u001b[0;31m#30\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m            )\n\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/lightgbm/sklearn.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight, init_score, eval_set, eval_names, eval_sample_weight, eval_class_weight, eval_init_score, eval_metric, early_stopping_rounds, verbose, feature_name, categorical_feature, callbacks)\u001b[0m\n\u001b[1;32m    673\u001b[0m                                         \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeature_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeature_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    674\u001b[0m                                         \u001b[0mcategorical_feature\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcategorical_feature\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 675\u001b[0;31m                                         callbacks=callbacks)\n\u001b[0m\u001b[1;32m    676\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/lightgbm/sklearn.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight, init_score, group, eval_set, eval_names, eval_sample_weight, eval_class_weight, eval_init_score, eval_group, eval_metric, early_stopping_rounds, verbose, feature_name, categorical_feature, callbacks)\u001b[0m\n\u001b[1;32m    467\u001b[0m                               \u001b[0mverbose_eval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeature_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeature_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    468\u001b[0m                               \u001b[0mcategorical_feature\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcategorical_feature\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 469\u001b[0;31m                               callbacks=callbacks)\n\u001b[0m\u001b[1;32m    470\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    471\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mevals_result\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/lightgbm/engine.pyc\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(params, train_set, num_boost_round, valid_sets, valid_names, fobj, feval, init_model, feature_name, categorical_feature, early_stopping_rounds, evals_result, verbose_eval, learning_rates, keep_training_booster, callbacks)\u001b[0m\n\u001b[1;32m    199\u001b[0m                                     evaluation_result_list=None))\n\u001b[1;32m    200\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 201\u001b[0;31m         \u001b[0mbooster\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    202\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m         \u001b[0mevaluation_result_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/lightgbm/basic.pyc\u001b[0m in \u001b[0;36mupdate\u001b[0;34m(self, train_set, fobj)\u001b[0m\n\u001b[1;32m   1522\u001b[0m             _safe_call(_LIB.LGBM_BoosterUpdateOneIter(\n\u001b[1;32m   1523\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1524\u001b[0;31m                 ctypes.byref(is_finished)))\n\u001b[0m\u001b[1;32m   1525\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__is_predicted_cur_iter\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mFalse\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__num_dataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1526\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mis_finished\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "lgbBO.maximize(init_points=2,n_iter=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_params': {'colsample_bytree': 0.5802478232647209,\n",
       "  'min_child_weight': 20.33692569448104,\n",
       "  'min_split_gain': 0.7649917884568475,\n",
       "  'num_leaves': 21.152379900895014,\n",
       "  'reg_alpha': 9.756996480309178,\n",
       "  'reg_lambda': 8.776694575019713,\n",
       "  'subsample': 0.9370453331986632},\n",
       " 'max_val': 0.793068381666508}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgbBO.res['max']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "colsample_bytree |   min_child_weight |   min_split_gain |   num_leaves |   reg_alpha |   reg_lambda |   subsample | "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}