05_House_Price_Prediction_B.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARN: OMP_NUM_THREADS=None =>\n",
      "... If you are using openblas if you are using openblas set OMP_NUM_THREADS=1 or risk subprocess calls hanging indefinitely\n"
     ]
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "from numpy import mean\n",
    "import numpy as np\n",
    "from pandas import read_csv\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from hpsklearn import HyperoptEstimator, pca, normalizer, min_max_scaler, standard_scaler, xgboost_regression\n",
    "from hpsklearn import random_forest_regression, ada_boost_regression, gradient_boosting_regression, extra_trees_regression, sgd_regression\n",
    "from hpsklearn import svr, svr_linear, svr_rbf, svr_poly, svr_sigmoid\n",
    "from hpsklearn import any_regressor\n",
    "from hpsklearn import any_preprocessing\n",
    "from hyperopt import tpe, hp\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error\n",
    "from xgboost import XGBRegressor\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import RepeatedStratifiedKFold\n",
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# load dataset\n",
    "url = 'https://raw.githubusercontent.com/jbrownlee/Datasets/master/housing.csv'\n",
    "dataframe = read_csv(url, header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(506, 14)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataframe.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00632</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0</td>\n",
       "      <td>0.538</td>\n",
       "      <td>6.575</td>\n",
       "      <td>65.2</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1</td>\n",
       "      <td>296.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>396.90</td>\n",
       "      <td>4.98</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.02731</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>6.421</td>\n",
       "      <td>78.9</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.14</td>\n",
       "      <td>21.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.02729</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>7.185</td>\n",
       "      <td>61.1</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>392.83</td>\n",
       "      <td>4.03</td>\n",
       "      <td>34.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.03237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>6.998</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>394.63</td>\n",
       "      <td>2.94</td>\n",
       "      <td>33.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.06905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>7.147</td>\n",
       "      <td>54.2</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.33</td>\n",
       "      <td>36.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        0     1     2   3      4      5     6       7   8      9     10  \\\n",
       "0  0.00632  18.0  2.31   0  0.538  6.575  65.2  4.0900   1  296.0  15.3   \n",
       "1  0.02731   0.0  7.07   0  0.469  6.421  78.9  4.9671   2  242.0  17.8   \n",
       "2  0.02729   0.0  7.07   0  0.469  7.185  61.1  4.9671   2  242.0  17.8   \n",
       "3  0.03237   0.0  2.18   0  0.458  6.998  45.8  6.0622   3  222.0  18.7   \n",
       "4  0.06905   0.0  2.18   0  0.458  7.147  54.2  6.0622   3  222.0  18.7   \n",
       "\n",
       "       11    12    13  \n",
       "0  396.90  4.98  24.0  \n",
       "1  396.90  9.14  21.6  \n",
       "2  392.83  4.03  34.7  \n",
       "3  394.63  2.94  33.4  \n",
       "4  396.90  5.33  36.2  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataframe.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(506, 13) (506,)\n"
     ]
    }
   ],
   "source": [
    "# split into input and output elements\n",
    "data = dataframe.values\n",
    "data = data.astype('float32')\n",
    "X, y = data[:, :-1], data[:, -1]\n",
    "print(X.shape, y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "preproc = hp.choice('myprepros_name', \n",
    "                    [\n",
    "                        [min_max_scaler('myprepros_name.only_norm')],\n",
    "                        [standard_scaler('myprepros_name.only_std_scaler')],\n",
    "                        [pca('myprepros_name.only_pca')],\n",
    "                        [pca('myprepros_name.pca'), min_max_scaler('myprepros_name.norm')],\n",
    "                        [min_max_scaler('myprepros_name.first_norm'), standard_scaler('myprepros_name.second_std_scaler')],\n",
    "                        []\n",
    "                    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg = hp.choice( 'model_name',\n",
    "            [ random_forest_regression ('model_name.random_forest_regression'),\n",
    "            sgd_regression ('model_name.sgd_regression'),\n",
    "            ada_boost_regression ('model_name.ada_boost_regression'),\n",
    "            gradient_boosting_regression ('model_name.gradient_boosting_regression'),\n",
    "            xgboost_regression ('model_name.xgboost_regression'),\n",
    "            extra_trees_regression('model_name.extra_trees_regression'),\n",
    "            svr_linear('model_name.svr_linear'),\n",
    "            svr_rbf('model_name.svr_rbf'),\n",
    "            svr('model_name.svr')]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "#preprocessing=[min_max_scaler('norm_scaler'), pca('my_pca') ]\n",
    "#regressor=xgboost_regression('xgboost')\n",
    "#preprocessing=any_preprocessing('pre')\n",
    "#preprocessing = preproc\n",
    "#regressor=any_regressor('reg')\n",
    "#regressor=reg\n",
    "\n",
    "#01. 100 times with regressor=any_regressor('reg') & preprocessing=any_preprocessing('pre')\n",
    "\n",
    "#02. 100 times with regressor=reg & preprocessing = preproc\n",
    "\n",
    "#03. 100 times with regressor=xgboost_regression(name='xgboost',objective='reg:squarederror') & preprocessing = preproc\n",
    "\n",
    "#04. 100 times with regressor=xgboost_regression(name='xgboost',objective='reg:squarederror') & preprocessing = any_preprocessing('pre')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "tags": [
     "outputPrepend"
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "]\n",
      "100%|██████████| 69/69 [00:06<00:00, 10.17trial/s, best loss: 1.743239080204683]\n",
      "100%|██████████| 70/70 [00:06<00:00, 10.63trial/s, best loss: 1.743239080204683]\n",
      "100%|██████████| 71/71 [00:06<00:00, 10.49trial/s, best loss: 1.743239080204683]\n",
      "100%|██████████| 72/72 [00:06<00:00, 10.63trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 73/73 [00:07<00:00, 10.39trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 74/74 [00:05<00:00, 13.35trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 75/75 [00:06<00:00, 11.16trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 76/76 [00:06<00:00, 12.48trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 77/77 [00:06<00:00, 11.20trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 78/78 [00:06<00:00, 11.47trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 79/79 [00:05<00:00, 13.81trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 80/80 [00:05<00:00, 13.94trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 81/81 [00:06<00:00, 13.20trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 82/82 [00:06<00:00, 12.75trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 83/83 [00:06<00:00, 13.58trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 84/84 [00:05<00:00, 14.39trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 85/85 [00:06<00:00, 13.86trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 86/86 [00:06<00:00, 12.63trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 87/87 [00:05<00:00, 15.57trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 88/88 [00:07<00:00, 11.02trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 89/89 [00:07<00:00, 12.32trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 90/90 [00:04<00:00, 18.18trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 91/91 [00:06<00:00, 14.84trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 92/92 [00:06<00:00, 14.92trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 93/93 [00:08<00:00, 11.54trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 94/94 [00:07<00:00, 13.02trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 95/95 [00:07<00:00, 12.10trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 96/96 [00:09<00:00, 10.04trial/s, best loss: 1.714458893327152]\n",
      "100%|██████████| 97/97 [00:05<00:00, 18.31trial/s, best loss: 1.6898581210304708]\n",
      "100%|██████████| 98/98 [00:04<00:00, 20.42trial/s, best loss: 1.6898581210304708]\n",
      "100%|██████████| 99/99 [00:05<00:00, 17.90trial/s, best loss: 1.6898581210304708]\n",
      "100%|██████████| 100/100 [00:05<00:00, 17.96trial/s, best loss: 1.6898581210304708]\n",
      "[22:51:15] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.1.0/src/objective/regression_obj.cu:170: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[22:51:16] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.1.0/src/objective/regression_obj.cu:170: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "8\n",
      "100%|██████████| 1/1 [00:05<00:00,  5.26s/trial, best loss: 2.679064154624939]\n",
      "100%|██████████| 2/2 [00:05<00:00,  2.76s/trial, best loss: 2.679064154624939]\n",
      "100%|██████████| 3/3 [00:06<00:00,  2.17s/trial, best loss: 2.5795352669323193]\n",
      "100%|██████████| 4/4 [00:06<00:00,  1.55s/trial, best loss: 2.5795352669323193]\n",
      "100%|██████████| 5/5 [00:06<00:00,  1.24s/trial, best loss: 2.5795352669323193]\n",
      "100%|██████████| 6/6 [00:06<00:00,  1.13s/trial, best loss: 2.5795352669323193]\n",
      "100%|██████████| 7/7 [00:06<00:00,  1.02trial/s, best loss: 2.5795352669323193]\n",
      "100%|██████████| 8/8 [00:05<00:00,  1.34trial/s, best loss: 2.5795352669323193]\n",
      "100%|██████████| 9/9 [00:05<00:00,  1.68trial/s, best loss: 2.5795352669323193]\n",
      "100%|██████████| 10/10 [00:06<00:00,  1.44trial/s, best loss: 2.3905647432102874]\n",
      "100%|██████████| 11/11 [00:05<00:00,  2.11trial/s, best loss: 2.3905647432102874]\n",
      "100%|██████████| 12/12 [00:05<00:00,  2.35trial/s, best loss: 2.3905647432102874]\n",
      "100%|██████████| 13/13 [00:06<00:00,  1.96trial/s, best loss: 2.3905647432102874]\n",
      "100%|██████████| 14/14 [00:05<00:00,  2.54trial/s, best loss: 2.3905647432102874]\n",
      "100%|██████████| 15/15 [00:05<00:00,  2.92trial/s, best loss: 2.3905647432102874]\n",
      "100%|██████████| 16/16 [00:05<00:00,  2.76trial/s, best loss: 1.8759238229078405]\n",
      "100%|██████████| 17/17 [00:11<00:00,  1.54trial/s, best loss: 1.8759238229078405]\n",
      "100%|██████████| 18/18 [00:07<00:00,  2.44trial/s, best loss: 1.8759238229078405]\n",
      "100%|██████████| 19/19 [00:09<00:00,  2.08trial/s, best loss: 1.8759238229078405]\n",
      "100%|██████████| 20/20 [00:06<00:00,  2.92trial/s, best loss: 1.8759238229078405]\n",
      "100%|██████████| 21/21 [00:05<00:00,  3.61trial/s, best loss: 1.8759238229078405]\n",
      "100%|██████████| 22/22 [00:05<00:00,  3.81trial/s, best loss: 1.8759238229078405]\n",
      "100%|██████████| 23/23 [00:05<00:00,  4.44trial/s, best loss: 1.8759238229078405]\n",
      "100%|██████████| 24/24 [00:05<00:00,  4.74trial/s, best loss: 1.8759238229078405]\n",
      "100%|██████████| 25/25 [00:04<00:00,  5.55trial/s, best loss: 1.8759238229078405]\n",
      "100%|██████████| 26/26 [00:04<00:00,  5.38trial/s, best loss: 1.8759238229078405]\n",
      "100%|██████████| 27/27 [00:06<00:00,  4.35trial/s, best loss: 1.8759238229078405]\n",
      "100%|██████████| 28/28 [00:04<00:00,  5.73trial/s, best loss: 1.8759238229078405]\n",
      "100%|██████████| 29/29 [00:05<00:00,  5.17trial/s, best loss: 1.87356979005477]\n",
      "100%|██████████| 30/30 [00:05<00:00,  5.47trial/s, best loss: 1.87356979005477]\n",
      "100%|██████████| 31/31 [00:06<00:00,  4.83trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 32/32 [00:07<00:00,  4.16trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 33/33 [00:05<00:00,  5.52trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 34/34 [00:05<00:00,  6.38trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 35/35 [00:07<00:00,  4.65trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 36/36 [00:05<00:00,  7.05trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 37/37 [00:07<00:00,  5.08trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 38/38 [00:05<00:00,  6.43trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 39/39 [00:05<00:00,  6.68trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 40/40 [00:07<00:00,  5.25trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 41/41 [00:07<00:00,  5.37trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 42/42 [00:06<00:00,  6.33trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 43/43 [00:05<00:00,  8.09trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 44/44 [00:06<00:00,  7.09trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 45/45 [00:06<00:00,  7.28trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 46/46 [00:05<00:00,  8.18trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 47/47 [00:08<00:00,  5.40trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 48/48 [00:06<00:00,  7.34trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 49/49 [00:07<00:00,  6.41trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 50/50 [00:05<00:00,  9.47trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 51/51 [00:06<00:00,  7.54trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 52/52 [00:08<00:00,  5.84trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 53/53 [00:04<00:00, 11.30trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 54/54 [00:06<00:00,  7.93trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 55/55 [00:06<00:00,  8.97trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 56/56 [00:04<00:00, 11.29trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 57/57 [00:05<00:00, 10.78trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 58/58 [00:05<00:00, 10.37trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 59/59 [00:05<00:00, 11.55trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 60/60 [00:05<00:00, 10.99trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 61/61 [00:05<00:00, 12.00trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 62/62 [00:06<00:00, 10.26trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 63/63 [00:05<00:00, 12.26trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 64/64 [00:04<00:00, 13.25trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 65/65 [00:06<00:00, 10.57trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 66/66 [00:05<00:00, 13.17trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 67/67 [00:05<00:00, 13.21trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 68/68 [00:05<00:00, 13.29trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 69/69 [00:04<00:00, 14.41trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 70/70 [00:04<00:00, 15.21trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 71/71 [00:06<00:00, 11.49trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 72/72 [00:05<00:00, 13.92trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 73/73 [00:06<00:00, 11.38trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 74/74 [00:04<00:00, 14.96trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 75/75 [00:04<00:00, 15.67trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 76/76 [00:06<00:00, 12.38trial/s, best loss: 1.7149912189034855]\n",
      "100%|██████████| 77/77 [00:07<00:00, 10.58trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 78/78 [00:07<00:00, 10.09trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 79/79 [00:07<00:00, 10.06trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 80/80 [00:07<00:00, 11.19trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 81/81 [00:07<00:00, 10.23trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 82/82 [00:07<00:00, 10.66trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 83/83 [00:07<00:00, 11.58trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 84/84 [00:08<00:00,  9.52trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 85/85 [00:06<00:00, 13.64trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 86/86 [00:07<00:00, 11.56trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 87/87 [00:06<00:00, 13.10trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 88/88 [00:07<00:00, 11.10trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 89/89 [00:07<00:00, 12.39trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 90/90 [00:08<00:00, 10.38trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 91/91 [00:06<00:00, 15.13trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 92/92 [00:07<00:00, 12.25trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 93/93 [00:05<00:00, 16.28trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 94/94 [00:06<00:00, 14.77trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 95/95 [00:10<00:00,  9.49trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 96/96 [00:06<00:00, 14.85trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 97/97 [00:05<00:00, 17.42trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 98/98 [00:07<00:00, 13.02trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 99/99 [00:06<00:00, 14.89trial/s, best loss: 1.7055036530775183]\n",
      "100%|██████████| 100/100 [00:05<00:00, 19.21trial/s, best loss: 1.7055036530775183]\n",
      "[23:01:50] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.1.0/src/objective/regression_obj.cu:170: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[23:01:53] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.1.0/src/objective/regression_obj.cu:170: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "9\n",
      "100%|██████████| 1/1 [00:05<00:00,  5.70s/trial, best loss: 2.782993940746083]\n",
      "100%|██████████| 2/2 [00:07<00:00,  3.76s/trial, best loss: 2.4354907133999992]\n",
      "100%|██████████| 3/3 [00:05<00:00,  1.90s/trial, best loss: 2.044494453598471]\n",
      "100%|██████████| 4/4 [00:07<00:00,  1.80s/trial, best loss: 2.044494453598471]\n",
      "100%|██████████| 5/5 [00:05<00:00,  1.06s/trial, best loss: 2.044494453598471]\n",
      "100%|██████████| 6/6 [00:05<00:00,  1.10trial/s, best loss: 2.044494453598471]\n",
      "100%|██████████| 7/7 [00:05<00:00,  1.19trial/s, best loss: 2.044494453598471]\n",
      "100%|██████████| 8/8 [00:07<00:00,  1.01trial/s, best loss: 2.044494453598471]\n",
      "100%|██████████| 9/9 [00:07<00:00,  1.26trial/s, best loss: 2.044494453598471]\n",
      "100%|██████████| 10/10 [00:05<00:00,  1.73trial/s, best loss: 2.044494453598471]\n",
      "100%|██████████| 11/11 [00:05<00:00,  2.04trial/s, best loss: 2.044494453598471]\n",
      "100%|██████████| 12/12 [00:05<00:00,  2.31trial/s, best loss: 2.044494453598471]\n",
      "100%|██████████| 13/13 [00:07<00:00,  1.82trial/s, best loss: 2.044494453598471]\n",
      "100%|██████████| 14/14 [00:06<00:00,  2.24trial/s, best loss: 2.044494453598471]\n",
      "100%|██████████| 15/15 [00:09<00:00,  1.53trial/s, best loss: 2.044494453598471]\n",
      "100%|██████████| 16/16 [00:07<00:00,  2.26trial/s, best loss: 2.044494453598471]\n",
      "100%|██████████| 17/17 [00:05<00:00,  3.04trial/s, best loss: 2.044494453598471]\n",
      "100%|██████████| 18/18 [00:05<00:00,  3.11trial/s, best loss: 2.044494453598471]\n",
      "100%|██████████| 19/19 [00:04<00:00,  3.99trial/s, best loss: 2.044494453598471]\n",
      "100%|██████████| 20/20 [00:04<00:00,  4.46trial/s, best loss: 2.044494453598471]\n",
      "100%|██████████| 21/21 [00:06<00:00,  3.29trial/s, best loss: 2.010975059340982]\n",
      "100%|██████████| 22/22 [00:06<00:00,  3.66trial/s, best loss: 1.9131748325684492]\n",
      "100%|██████████| 23/23 [00:06<00:00,  3.70trial/s, best loss: 1.9131748325684492]\n",
      "100%|██████████| 24/24 [00:04<00:00,  5.06trial/s, best loss: 1.9124063148218042]\n",
      "100%|██████████| 25/25 [00:04<00:00,  5.03trial/s, best loss: 1.9124063148218042]\n",
      "100%|██████████| 26/26 [00:05<00:00,  4.84trial/s, best loss: 1.9124063148218042]\n",
      "100%|██████████| 27/27 [00:04<00:00,  6.19trial/s, best loss: 1.9124063148218042]\n",
      "100%|██████████| 28/28 [00:05<00:00,  5.22trial/s, best loss: 1.9124063148218042]\n",
      "100%|██████████| 29/29 [00:05<00:00,  5.14trial/s, best loss: 1.9124063148218042]\n",
      "100%|██████████| 30/30 [00:05<00:00,  5.10trial/s, best loss: 1.9124063148218042]\n",
      "100%|██████████| 31/31 [00:06<00:00,  4.68trial/s, best loss: 1.9124063148218042]\n",
      "100%|██████████| 32/32 [00:07<00:00,  4.52trial/s, best loss: 1.9124063148218042]\n",
      "100%|██████████| 33/33 [00:06<00:00,  4.84trial/s, best loss: 1.9013734354692347]\n",
      "100%|██████████| 34/34 [00:07<00:00,  4.26trial/s, best loss: 1.9013734354692347]\n",
      "100%|██████████| 35/35 [00:06<00:00,  5.67trial/s, best loss: 1.9013734354692347]\n",
      "100%|██████████| 36/36 [00:04<00:00,  7.67trial/s, best loss: 1.9013734354692347]\n",
      "100%|██████████| 37/37 [00:06<00:00,  5.57trial/s, best loss: 1.9013734354692347]\n",
      "100%|██████████| 38/38 [00:05<00:00,  6.61trial/s, best loss: 1.9013734354692347]\n",
      "100%|██████████| 39/39 [00:06<00:00,  5.80trial/s, best loss: 1.9013734354692347]\n",
      "100%|██████████| 40/40 [00:07<00:00,  5.57trial/s, best loss: 1.9013734354692347]\n",
      "100%|██████████| 41/41 [00:04<00:00,  8.95trial/s, best loss: 1.9013734354692347]\n",
      "100%|██████████| 42/42 [00:09<00:00,  4.55trial/s, best loss: 1.9013734354692347]\n",
      "100%|██████████| 43/43 [00:05<00:00,  8.07trial/s, best loss: 1.9013734354692347]\n",
      "100%|██████████| 44/44 [00:05<00:00,  7.91trial/s, best loss: 1.9013734354692347]\n",
      "100%|██████████| 45/45 [00:05<00:00,  7.94trial/s, best loss: 1.9013734354692347]\n",
      "100%|██████████| 46/46 [00:07<00:00,  6.37trial/s, best loss: 1.9013734354692347]\n",
      "100%|██████████| 47/47 [00:07<00:00,  6.45trial/s, best loss: 1.9013734354692347]\n",
      "100%|██████████| 48/48 [00:08<00:00,  5.80trial/s, best loss: 1.9013734354692347]\n",
      "100%|██████████| 49/49 [00:08<00:00,  6.06trial/s, best loss: 1.9013734354692347]\n",
      "100%|██████████| 50/50 [00:06<00:00,  7.46trial/s, best loss: 1.9013734354692347]\n",
      "100%|██████████| 51/51 [00:05<00:00,  9.73trial/s, best loss: 1.84302416268517]\n",
      "100%|██████████| 52/52 [00:06<00:00,  8.31trial/s, best loss: 1.830057557891397]\n",
      "100%|██████████| 53/53 [00:05<00:00,  8.96trial/s, best loss: 1.8277358377681059]\n",
      "100%|██████████| 54/54 [00:06<00:00,  8.47trial/s, best loss: 1.8277358377681059]\n",
      "100%|██████████| 55/55 [00:06<00:00,  8.72trial/s, best loss: 1.8277358377681059]\n",
      "100%|██████████| 56/56 [00:06<00:00,  8.15trial/s, best loss: 1.8277358377681059]\n",
      "100%|██████████| 57/57 [00:05<00:00,  9.94trial/s, best loss: 1.8277358377681059]\n",
      "100%|██████████| 58/58 [00:06<00:00,  8.94trial/s, best loss: 1.8277358377681059]\n",
      "100%|██████████| 59/59 [00:06<00:00,  8.87trial/s, best loss: 1.8277358377681059]\n",
      "100%|██████████| 60/60 [00:05<00:00, 11.87trial/s, best loss: 1.8129221271066105]\n",
      "100%|██████████| 61/61 [00:05<00:00, 12.07trial/s, best loss: 1.8129221271066105]\n",
      "100%|██████████| 62/62 [00:04<00:00, 13.67trial/s, best loss: 1.8129221271066105]\n",
      "100%|██████████| 63/63 [00:05<00:00, 11.54trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 64/64 [00:04<00:00, 13.59trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 65/65 [00:05<00:00, 12.61trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 66/66 [00:05<00:00, 11.57trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 67/67 [00:05<00:00, 13.06trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 68/68 [00:05<00:00, 12.91trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 69/69 [00:06<00:00, 10.79trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 70/70 [00:05<00:00, 13.83trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 71/71 [00:04<00:00, 14.51trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 72/72 [00:04<00:00, 15.50trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 73/73 [00:04<00:00, 15.03trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 74/74 [00:04<00:00, 15.32trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 75/75 [00:04<00:00, 15.91trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 76/76 [00:05<00:00, 14.29trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 77/77 [00:05<00:00, 14.02trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 78/78 [00:05<00:00, 14.95trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 79/79 [00:06<00:00, 12.77trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 80/80 [00:05<00:00, 15.79trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 81/81 [00:05<00:00, 14.21trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 82/82 [00:04<00:00, 17.03trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 83/83 [00:04<00:00, 18.22trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 84/84 [00:06<00:00, 12.68trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 85/85 [00:05<00:00, 15.96trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 86/86 [00:04<00:00, 18.16trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 87/87 [00:07<00:00, 12.36trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 88/88 [00:04<00:00, 18.96trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 89/89 [00:05<00:00, 17.56trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 90/90 [00:04<00:00, 18.31trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 91/91 [00:05<00:00, 17.69trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 92/92 [00:05<00:00, 15.53trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 93/93 [00:05<00:00, 18.32trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 94/94 [00:05<00:00, 16.00trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 95/95 [00:07<00:00, 12.62trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 96/96 [00:06<00:00, 13.96trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 97/97 [00:04<00:00, 21.57trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 98/98 [00:05<00:00, 19.31trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 99/99 [00:06<00:00, 16.16trial/s, best loss: 1.789224252981298]\n",
      "100%|██████████| 100/100 [00:06<00:00, 14.63trial/s, best loss: 1.789224252981298]\n",
      "[23:11:47] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.1.0/src/objective/regression_obj.cu:170: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[23:11:48] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.1.0/src/objective/regression_obj.cu:170: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "Execution time :  1:49:31.163775\n"
     ]
    }
   ],
   "source": [
    "# define search\n",
    "init_time = datetime.now()\n",
    "mae_error = 0\n",
    "best_model = None\n",
    "for i in range(10):\n",
    "    print (i)\n",
    "    model = HyperoptEstimator(  regressor=xgboost_regression(name='xgboost',objective='reg:squarederror'), \n",
    "                                preprocessing=any_preprocessing('pre'), \n",
    "                                loss_fn=mean_absolute_error,\n",
    "                                algo=tpe.suggest, \n",
    "                                max_evals=100, \n",
    "                                trial_timeout=120)\n",
    "    # perform the search\n",
    "    model.fit(X_train, y_train)\n",
    "    mae = model.score(X_test, y_test)\n",
    "    #mse = np.sqrt(mse)\n",
    "\n",
    "    if mae > mae_error:\n",
    "        mae_error = mae\n",
    "        best_model = model\n",
    "        print(mae_error)\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE: 0.892\n",
      "{'learner': XGBRegressor(base_score=0.5, booster='gbtree',\n",
      "             colsample_bylevel=0.8297027242206534, colsample_bynode=1,\n",
      "             colsample_bytree=0.9513954029807743, gamma=0.02022174653713924,\n",
      "             gpu_id=-1, importance_type='gain', interaction_constraints='',\n",
      "             learning_rate=0.09281958438582665, max_delta_step=0, max_depth=3,\n",
      "             min_child_weight=2, missing=nan, monotone_constraints='()',\n",
      "             n_estimators=3200, n_jobs=0, num_parallel_tree=1,\n",
      "             objective='reg:linear', random_state=0,\n",
      "             reg_alpha=0.00881786542441721, reg_lambda=3.2649130010873266,\n",
      "             scale_pos_weight=1, seed=0, subsample=0.5058451512836544,\n",
      "             tree_method='exact', validate_parameters=1, verbosity=None), 'preprocs': (StandardScaler(with_mean=False),), 'ex_preprocs': ()}\n"
     ]
    }
   ],
   "source": [
    "# summarize the best\n",
    "print(\"MSE: %.3f\" % mae)\n",
    "print(best_model.best_model())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MSE: 0.892\n",
    "{'learner': XGBRegressor(base_score=0.5, booster='gbtree',\n",
    "             colsample_bylevel=0.8297027242206534, colsample_bynode=1,\n",
    "             colsample_bytree=0.9513954029807743, gamma=0.02022174653713924,\n",
    "             gpu_id=-1, importance_type='gain', interaction_constraints='',\n",
    "             learning_rate=0.09281958438582665, max_delta_step=0, max_depth=3,\n",
    "             min_child_weight=2, missing=nan, monotone_constraints='()',\n",
    "             n_estimators=3200, n_jobs=0, num_parallel_tree=1,\n",
    "             objective='reg:linear', random_state=0,\n",
    "             reg_alpha=0.00881786542441721, reg_lambda=3.2649130010873266,\n",
    "             scale_pos_weight=1, seed=0, subsample=0.5058451512836544,\n",
    "             tree_method='exact', validate_parameters=1, verbosity=None), 'preprocs': (StandardScaler(with_mean=False),), 'ex_preprocs': ()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Squard Error: 2.9499547\n",
      "Execution time :  0:00:01.061171\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "#objective='reg:squarederror'\n",
    "xgb_model = XGBRegressor(base_score=0.5, booster='gbtree',\n",
    "             colsample_bylevel=0.8297027242206534, colsample_bynode=1,\n",
    "             colsample_bytree=0.9513954029807743, gamma=0.02022174653713924,\n",
    "             gpu_id=-1, importance_type='gain', interaction_constraints='',\n",
    "             learning_rate=0.09281958438582665, max_delta_step=0, max_depth=3,\n",
    "             min_child_weight=2, missing=None, monotone_constraints='()',\n",
    "             n_estimators=3200, n_jobs=0, num_parallel_tree=1,\n",
    "             objective='reg:squarederror', random_state=0,\n",
    "             reg_alpha=0.00881786542441721, reg_lambda=3.2649130010873266,\n",
    "             scale_pos_weight=1, seed=0, subsample=0.5058451512836544,\n",
    "             tree_method='exact', validate_parameters=1, verbosity=None)\n",
    "\n",
    "pipeline = Pipeline(steps=[('Stnd', StandardScaler(with_mean=False)), ('m', xgb_model)])\n",
    "pipeline.fit(X_train, y_train)\n",
    "\n",
    "#xgb_model.fit(X_train, y_train, \n",
    "             #early_stopping_rounds=5, \n",
    "             #eval_set=[(X_test, y_test)], \n",
    "             #verbose=False)\n",
    "\n",
    "#xgb_predictions = xgb_model.predict(X_test)\n",
    "xgb_predictions = pipeline.predict(X_test)\n",
    "\n",
    "mae = mean_squared_error(xgb_predictions, y_test)\n",
    "print(\"Mean Squard Error:\" , np.sqrt(mae))\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mean Squard Error: 2.9499547"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}