05_House_Price_Prediction_A.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Load Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import random\n",
    "import operator\n",
    "import pickle\n",
    "from datetime import datetime\n",
    "from scipy.stats import shapiro, normaltest\n",
    "from sklearn.neighbors import LocalOutlierFactor\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.preprocessing import OrdinalEncoder\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import RepeatedKFold\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.linear_model import ElasticNet\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.ensemble import ExtraTreesRegressor\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "from xgboost import XGBRegressor\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import PowerTransformer\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.decomposition import TruncatedSVD"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Load & Prepare Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def load_dataset():\n",
    "    df = pd.read_csv('../data/kc_house_data.csv')\n",
    "    df = df.drop('id',axis=1)\n",
    "    df['date'] = pd.to_datetime(df['date'])\n",
    "    df['month'] = df['date'].apply(lambda date:date.month)\n",
    "    df['year'] = df['date'].apply(lambda date:date.year)\n",
    "    df = df.drop('date',axis=1)\n",
    "    df = df.drop('zipcode',axis=1)\n",
    "    \n",
    "    X = df.drop('price',axis=1)\n",
    "    y = df['price']\n",
    "    \n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Regression Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def get_models():\n",
    "    models = []\n",
    "    models.append(('LR', LinearRegression()))\n",
    "    models.append(('LASSO', Lasso()))\n",
    "    models.append(('EN', ElasticNet()))\n",
    "    models.append(('KNN', KNeighborsRegressor()))\n",
    "    models.append(('CART', DecisionTreeRegressor()))\n",
    "    models.append(('SVR', SVR(gamma='auto')))\n",
    "    return models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Ensemble Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def get_ensemble_models():\n",
    "    ensembles = []\n",
    "    ensembles.append(('AB', AdaBoostRegressor()))\n",
    "    ensembles.append(('GBM', GradientBoostingRegressor())) \n",
    "    ensembles.append(('RF', RandomForestRegressor()))\n",
    "    ensembles.append(('ET', ExtraTreesRegressor()))\n",
    "    ensembles.append(('XGB', XGBRegressor()))\n",
    "    return ensembles"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Model Evaluation Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# evaluate a model\n",
    "def evaluate_model(X, y, model):\n",
    "    # define evaluation procedure\n",
    "    cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "    # evaluate model\n",
    "    scores = cross_val_score(model, X, y, scoring=\"neg_mean_squared_error\", cv=cv, n_jobs=-1)  \n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def display_scores(scores, name=''):\n",
    "    rmse_scores = np.sqrt(-scores)\n",
    "    #print(\"Scores:\", rmse_scores)\n",
    "    print(f\"RMSE Mean {name}: {rmse_scores.mean()}\")\n",
    "    print(f\"RMSE SD {name}: {rmse_scores.std()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Load the Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>sqft_living</th>\n",
       "      <th>sqft_lot</th>\n",
       "      <th>floors</th>\n",
       "      <th>waterfront</th>\n",
       "      <th>view</th>\n",
       "      <th>condition</th>\n",
       "      <th>grade</th>\n",
       "      <th>sqft_above</th>\n",
       "      <th>sqft_basement</th>\n",
       "      <th>yr_built</th>\n",
       "      <th>yr_renovated</th>\n",
       "      <th>lat</th>\n",
       "      <th>long</th>\n",
       "      <th>sqft_living15</th>\n",
       "      <th>sqft_lot15</th>\n",
       "      <th>month</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1180</td>\n",
       "      <td>5650</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>1180</td>\n",
       "      <td>0</td>\n",
       "      <td>1955</td>\n",
       "      <td>0</td>\n",
       "      <td>47.5112</td>\n",
       "      <td>-122.257</td>\n",
       "      <td>1340</td>\n",
       "      <td>5650</td>\n",
       "      <td>10</td>\n",
       "      <td>2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>2.25</td>\n",
       "      <td>2570</td>\n",
       "      <td>7242</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>2170</td>\n",
       "      <td>400</td>\n",
       "      <td>1951</td>\n",
       "      <td>1991</td>\n",
       "      <td>47.7210</td>\n",
       "      <td>-122.319</td>\n",
       "      <td>1690</td>\n",
       "      <td>7639</td>\n",
       "      <td>12</td>\n",
       "      <td>2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1.00</td>\n",
       "      <td>770</td>\n",
       "      <td>10000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>770</td>\n",
       "      <td>0</td>\n",
       "      <td>1933</td>\n",
       "      <td>0</td>\n",
       "      <td>47.7379</td>\n",
       "      <td>-122.233</td>\n",
       "      <td>2720</td>\n",
       "      <td>8062</td>\n",
       "      <td>2</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>3.00</td>\n",
       "      <td>1960</td>\n",
       "      <td>5000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>1050</td>\n",
       "      <td>910</td>\n",
       "      <td>1965</td>\n",
       "      <td>0</td>\n",
       "      <td>47.5208</td>\n",
       "      <td>-122.393</td>\n",
       "      <td>1360</td>\n",
       "      <td>5000</td>\n",
       "      <td>12</td>\n",
       "      <td>2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>2.00</td>\n",
       "      <td>1680</td>\n",
       "      <td>8080</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>1680</td>\n",
       "      <td>0</td>\n",
       "      <td>1987</td>\n",
       "      <td>0</td>\n",
       "      <td>47.6168</td>\n",
       "      <td>-122.045</td>\n",
       "      <td>1800</td>\n",
       "      <td>7503</td>\n",
       "      <td>2</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   bedrooms  bathrooms  sqft_living  sqft_lot  floors  waterfront  view  \\\n",
       "0         3       1.00         1180      5650     1.0           0     0   \n",
       "1         3       2.25         2570      7242     2.0           0     0   \n",
       "2         2       1.00          770     10000     1.0           0     0   \n",
       "3         4       3.00         1960      5000     1.0           0     0   \n",
       "4         3       2.00         1680      8080     1.0           0     0   \n",
       "\n",
       "   condition  grade  sqft_above  sqft_basement  yr_built  yr_renovated  \\\n",
       "0          3      7        1180              0      1955             0   \n",
       "1          3      7        2170            400      1951          1991   \n",
       "2          3      6         770              0      1933             0   \n",
       "3          5      7        1050            910      1965             0   \n",
       "4          3      8        1680              0      1987             0   \n",
       "\n",
       "       lat     long  sqft_living15  sqft_lot15  month  year  \n",
       "0  47.5112 -122.257           1340        5650     10  2014  \n",
       "1  47.7210 -122.319           1690        7639     12  2014  \n",
       "2  47.7379 -122.233           2720        8062      2  2015  \n",
       "3  47.5208 -122.393           1360        5000     12  2014  \n",
       "4  47.6168 -122.045           1800        7503      2  2015  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, y = load_dataset()\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(21597, 19) (21597,)\n",
      "19\n"
     ]
    }
   ],
   "source": [
    "print(X.shape, y.shape)\n",
    "print(X.shape[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['bedrooms', 'bathrooms', 'sqft_living', 'sqft_lot', 'floors',\n",
       "       'waterfront', 'view', 'condition', 'grade', 'sqft_above',\n",
       "       'sqft_basement', 'yr_built', 'yr_renovated', 'lat', 'long',\n",
       "       'sqft_living15', 'sqft_lot15', 'month', 'year'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Object & Numeric Column Transfer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# determine categorical and numerical features\n",
    "#numerical_ix = X.select_dtypes(include=['int64', 'float64']).columns\n",
    "#categorical_ix = X.select_dtypes(include=['object', 'bool']).columns\n",
    "\n",
    "# IT SHOULD BE DON WHILE DATA PRE-PROCESSING\n",
    "\n",
    "# define the data preparation for the columns\n",
    "#t = [('cat', OneHotEncoder(), categorical_ix)]\n",
    "#cat_col_transform = ColumnTransformer(transformers=t, remainder='passthrough')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Normality Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def is_normal(data):\n",
    "    alpha = 0.05\n",
    "    stat, p = normaltest(data)\n",
    "    if p > alpha:\n",
    "        normalTest = True\n",
    "    else:\n",
    "        normalTest = False\n",
    "    \n",
    "    stat, p = shapiro(data)\n",
    "    if p > alpha:\n",
    "        shapiroTest = True\n",
    "    else:\n",
    "        shapiroTest = False\n",
    "        \n",
    "    return normalTest and shapiroTest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gaussian Like columns: []\n",
      "Non-Gaussian Like columns: ['bedrooms', 'bathrooms', 'sqft_living', 'sqft_lot', 'floors', 'waterfront', 'view', 'condition', 'grade', 'sqft_above', 'sqft_basement', 'yr_built', 'yr_renovated', 'lat', 'long', 'sqft_living15', 'sqft_lot15', 'month', 'year']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Z:\\ProgramData\\Anaconda3\\lib\\site-packages\\scipy\\stats\\morestats.py:1681: UserWarning: p-value may not be accurate for N > 5000.\n",
      "  warnings.warn(\"p-value may not be accurate for N > 5000.\")\n"
     ]
    }
   ],
   "source": [
    "Gaussian_Like = []\n",
    "Non_Gaussian = []\n",
    "\n",
    "for i, name in enumerate (X.columns):\n",
    "    if is_normal(X[name]):\n",
    "        Gaussian_Like.append(name)\n",
    "    else:\n",
    "        Non_Gaussian.append(name)\n",
    "        \n",
    "print (f\"Gaussian Like columns: {Gaussian_Like}\")\n",
    "print (f\"Non-Gaussian Like columns: {Non_Gaussian}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Standardize Only Gaussian-Like Input Variables\n",
    "# Normalize Only Non-Gaussian Input Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "if len(Gaussian_Like)> 0 and len(Non_Gaussian) > 0:\n",
    "    print(\"Normality Function is working\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Spot Check Algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE Mean LR: 202148.7774280055\n",
      "RMSE SD LR: 12936.354102455933\n",
      "RMSE Mean LASSO: 202148.73899318252\n",
      "RMSE SD LASSO: 12936.486855114108\n",
      "RMSE Mean EN: 226051.84792081185\n",
      "RMSE SD EN: 12663.621397224997\n",
      "RMSE Mean KNN: 256648.30884374006\n",
      "RMSE SD KNN: 16594.000541308156\n",
      "RMSE Mean CART: 180010.62248766955\n",
      "RMSE SD CART: 15101.962505019203\n",
      "RMSE Mean SVR: 377716.3508651177\n",
      "RMSE SD SVR: 20886.600878293215\n",
      "Execution time :  0:06:10.969464\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "models = get_models()\n",
    "for name, model in models:\n",
    "    scores = evaluate_model(X, y, model)\n",
    "    display_scores(scores, name)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Spot  Check Algorithms with Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE Mean LR: 202145.2867384407\n",
      "RMSE SD LR: 12943.15219668564\n",
      "RMSE Mean LASSO: 202148.96255300718\n",
      "RMSE SD LASSO: 12936.425663093607\n",
      "RMSE Mean EN: 344150.09132223553\n",
      "RMSE SD EN: 19991.857086988337\n",
      "RMSE Mean KNN: 178421.54546147387\n",
      "RMSE SD KNN: 14897.66028131097\n",
      "RMSE Mean CART: 181904.84861288717\n",
      "RMSE SD CART: 17154.193472721832\n",
      "RMSE Mean SVR: 377657.8190079603\n",
      "RMSE SD SVR: 20888.35441889303\n",
      "Execution time :  0:04:05.610742\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "models = get_models()\n",
    "for name, model in models:\n",
    "    # define pipeline\n",
    "    pipeline = Pipeline(steps=[('norm', MinMaxScaler()), ('m', model)])\n",
    "    scores = evaluate_model(X, y, pipeline)\n",
    "    display_scores(scores, name)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Spot  Check Algorithms with Standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE Mean LR: 202154.91237409425\n",
      "RMSE SD LR: 12962.970109882714\n",
      "RMSE Mean LASSO: 202148.75997957124\n",
      "RMSE SD LASSO: 12936.306065518083\n",
      "RMSE Mean EN: 211384.64692688358\n",
      "RMSE SD EN: 15067.324752263878\n",
      "RMSE Mean KNN: 174557.6925252656\n",
      "RMSE SD KNN: 13264.484181634602\n",
      "RMSE Mean CART: 183082.97083737174\n",
      "RMSE SD CART: 16967.704233080956\n",
      "RMSE Mean SVR: 377142.5577756564\n",
      "RMSE SD SVR: 20922.33359715182\n",
      "Execution time :  0:03:33.923775\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "models = get_models()\n",
    "for name, model in models:\n",
    "    # define pipeline\n",
    "    pipeline = Pipeline(steps=[('Stnd', StandardScaler()), ('m', model)])\n",
    "    scores = evaluate_model(X, y, pipeline)\n",
    "    display_scores(scores, name)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Spot Check Ensemble Algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE Mean AB: 350047.7043867652\n",
      "RMSE SD AB: 25434.634143770087\n",
      "RMSE Mean GBM: 134865.46673956417\n",
      "RMSE SD GBM: 10054.581996241734\n",
      "RMSE Mean RF: 127266.23241093171\n",
      "RMSE SD RF: 11035.061423551599\n",
      "RMSE Mean ET: 127482.3896872711\n",
      "RMSE SD ET: 11414.98254748831\n",
      "RMSE Mean XGB: 120784.22063826791\n",
      "RMSE SD XGB: 10614.733343791724\n",
      "Execution time :  0:05:23.735842\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "models = get_ensemble_models()\n",
    "for name, model in models:\n",
    "    scores = evaluate_model(X, y, model)\n",
    "    display_scores(scores, name)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Spot Check Ensemble Algorithms with Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE Mean AB: 343356.1299673539\n",
      "RMSE SD AB: 25251.873982202575\n",
      "RMSE Mean GBM: 135047.7803414584\n",
      "RMSE SD GBM: 10182.124077718094\n",
      "RMSE Mean RF: 126596.66284441769\n",
      "RMSE SD RF: 10809.758322968812\n",
      "RMSE Mean ET: 127517.91862967004\n",
      "RMSE SD ET: 11470.729139705722\n",
      "RMSE Mean XGB: 120651.20252651487\n",
      "RMSE SD XGB: 10521.786036697406\n",
      "Execution time :  0:05:56.099640\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "models = get_ensemble_models()\n",
    "for name, model in models:\n",
    "    # define pipeline\n",
    "    pipeline = Pipeline(steps=[('norm', MinMaxScaler()), ('m', model)])\n",
    "    scores = evaluate_model(X, y, pipeline)\n",
    "    display_scores(scores, name)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Spot Check Ensemble Algorithms with Standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE Mean AB: 344721.44724643056\n",
      "RMSE SD AB: 18986.009189849035\n",
      "RMSE Mean GBM: 135068.75384956726\n",
      "RMSE SD GBM: 10233.561335932562\n",
      "RMSE Mean RF: 126905.5855872733\n",
      "RMSE SD RF: 11039.192063338774\n",
      "RMSE Mean ET: 127626.69448219675\n",
      "RMSE SD ET: 12362.584634625216\n",
      "RMSE Mean XGB: 120788.18884176762\n",
      "RMSE SD XGB: 10616.407416417767\n",
      "Execution time :  0:05:43.918308\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "models = get_ensemble_models()\n",
    "for name, model in models:\n",
    "    # define pipeline\n",
    "    pipeline = Pipeline(steps=[('Stnd', StandardScaler()), ('m', model)])\n",
    "    scores = evaluate_model(X, y, pipeline)\n",
    "    display_scores(scores, name)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### XGB with Normalization & Standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE Mean : 120651.20252651487\n",
      "RMSE SD : 10521.786036697406\n",
      "Execution time :  0:00:25.712544\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "model = XGBRegressor()\n",
    "# define pipeline\n",
    "pipeline = Pipeline(steps=[('Stnd', StandardScaler()), ('norm', MinMaxScaler()), ('m', model)])\n",
    "scores = evaluate_model(X, y, pipeline)\n",
    "display_scores(scores)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Normalize Only Non-Gaussian Input Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "if len(Gaussian_Like)> 0 and len(Non_Gaussian) > 0:\n",
    "    init_time = datetime.now()\n",
    "    # define the selective transforms\n",
    "    t = [('e', MinMaxScaler(), Non_Gaussian)]\n",
    "    norm_transform = ColumnTransformer(transformers=t, remainder='passthrough')\n",
    "    # define pipeline\n",
    "    model = XGBRegressor()\n",
    "    # define pipeline\n",
    "    pipeline = Pipeline(steps=[('s', norm_transform), ('m', model)])\n",
    "    scores = evaluate_model(X, y, pipeline)\n",
    "    display_scores(scores)\n",
    "    fin_time = datetime.now()\n",
    "    print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Standardize Only Gaussian-Like Input Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "if len(Gaussian_Like)> 0 and len(Non_Gaussian) > 0:\n",
    "    init_time = datetime.now()\n",
    "    # define the selective transforms\n",
    "    t = [('n', StandardScaler(), Gaussian_Like)]\n",
    "    stnd_transform = ColumnTransformer(transformers=t, remainder='passthrough')\n",
    "    # define pipeline\n",
    "    model = XGBRegressor()\n",
    "    # define pipeline\n",
    "    pipeline = Pipeline(steps=[('s', stnd_transform), ('m', model)])\n",
    "    scores = evaluate_model(X, y, pipeline)\n",
    "    display_scores(scores)\n",
    "    fin_time = datetime.now()\n",
    "    print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### XGB with Selectively Normalize and Standardize Input Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "if len(Gaussian_Like)> 0 and len(Non_Gaussian) > 0:\n",
    "    init_time = datetime.now()\n",
    "    # define the selective transforms\n",
    "    t = [('e', MinMaxScaler(), Non_Gaussian), ('n', StandardScaler(), Gaussian_Like)]\n",
    "    selective = ColumnTransformer(transformers=t)\n",
    "    # define pipeline\n",
    "    model = XGBRegressor()\n",
    "    # define pipeline\n",
    "    pipeline = Pipeline(steps=[('s', selective), ('m', model)])\n",
    "    scores = evaluate_model(X, y, pipeline)\n",
    "    display_scores(scores)\n",
    "    fin_time = datetime.now()\n",
    "    print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### XGB with PowerTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE Mean: 120835.38353046724\n",
      "RMSE SD: 9856.333672781506\n",
      "Execution time :  0:00:35.081420\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "model = XGBRegressor()\n",
    "# define pipeline\n",
    "pipeline = Pipeline(steps=[('t', PowerTransformer()), ('m', model)])\n",
    "scores = evaluate_model(X, y, pipeline)\n",
    "display_scores(scores)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### XGB with PCA & Outlier Removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE of SVD value 18 is 156075.45787923344\n",
      "[366130.19229220535, 360130.5543315672, 254319.91156130133, 253508.49515443415, 236552.1307046798, 233271.04619099796, 231821.95428368897, 221233.67172203813, 219728.49963490356, 204976.57683429297, 200407.09060808478, 196228.44623024564, 195671.43706768172, 195575.0672501937, 195796.2543774463, 168123.55253226738, 163228.67825907172, 156075.45787923344, nan]\n",
      "['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19']\n",
      "Execution time :  0:11:25.416230\n",
      "18\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "# get a list of models to evaluate\n",
    "def get_models():\n",
    "    models = dict()\n",
    "    for i in range(1,20):\n",
    "        #steps = [('pca', PCA(n_components=i)), ('m', XGBRegressor())]\n",
    "        steps = [('svd', TruncatedSVD(n_components=i)), ('m', XGBRegressor())]\n",
    "        models[str(i)] = Pipeline(steps=steps)\n",
    "    return models\n",
    "\n",
    "# get the models to evaluate\n",
    "models = get_models() \n",
    "# evaluate the models and store results\n",
    "results, names = list(), list()\n",
    "for name, model in models.items():\n",
    "    scores = evaluate_model(X, y, model)\n",
    "    scores = np.sqrt(-scores)\n",
    "    results.append(scores.mean())\n",
    "    names.append(name)\n",
    "\n",
    "minIndex = results.index(min(results))\n",
    "print(f\"RMSE of SVD value {names[minIndex]} is {results[minIndex]}\")\n",
    "print(results)\n",
    "print(names)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))\n",
    "best_n_components = int(names[minIndex])\n",
    "print(best_n_components)\n",
    "#RMSE of PCA value 18 is 153401.49704230417"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE Mean : 146503.68059951623\n",
      "RMSE SD: 5925.882450907686\n",
      "Execution time :  0:00:59.039002\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "\n",
    "# define transform\n",
    "#pca = PCA(n_components=best_n_components)\n",
    "svd = TruncatedSVD(n_components=best_n_components)\n",
    "# prepare transform on dataset\n",
    "#pca.fit(X)\n",
    "svd.fit(X)\n",
    "# apply transform to dataset\n",
    "# X_transformed = pca.transform(X)\n",
    "X_transformed = svd.transform(X)\n",
    "\n",
    "# identify outliers in the training dataset\n",
    "lof = LocalOutlierFactor()\n",
    "yhat = lof.fit_predict(X_transformed)\n",
    "\n",
    "# select all rows that are not outliers\n",
    "mask = yhat != -1\n",
    "X_transformed, y_transformed = X_transformed[mask, :], y[mask]\n",
    "\n",
    "# fit the model\n",
    "model = XGBRegressor()\n",
    "\n",
    "# evaluate the model\n",
    "scores = evaluate_model(X_transformed, y_transformed, model)\n",
    "display_scores(scores)\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))\n",
    "\n",
    "# PCA RMSE Mean : 143220.67129159492"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'(array([ True,  True, False, ...,  True,  True,  True]), slice(None, None, None))' is an invalid key",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-25-8f5e1cdbb84a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;31m# select all rows that are not outliers\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[0mmask\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0myhat\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m \u001b[0mX_transformed\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_transformed\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mmask\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mmask\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[1;31m# fit the model\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mZ:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   2798\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnlevels\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2799\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2800\u001b[1;33m             \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2801\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mis_integer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2802\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mindexer\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mZ:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py\u001b[0m in \u001b[0;36mget_loc\u001b[1;34m(self, key, method, tolerance)\u001b[0m\n\u001b[0;32m   2644\u001b[0m                 )\n\u001b[0;32m   2645\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2646\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2647\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2648\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_maybe_cast_indexer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: '(array([ True,  True, False, ...,  True,  True,  True]), slice(None, None, None))' is an invalid key"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "\n",
    "# identify outliers in the training dataset\n",
    "lof = LocalOutlierFactor()\n",
    "yhat = lof.fit_predict(X)\n",
    "\n",
    "# select all rows that are not outliers\n",
    "mask = yhat != -1\n",
    "X_transformed, y_transformed = X[mask, :], y[mask]\n",
    "\n",
    "# fit the model\n",
    "model = XGBRegressor()\n",
    "\n",
    "# evaluate the model\n",
    "scores = evaluate_model(X_transformed, y_transformed, model)\n",
    "display_scores(scores)\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### XGB with Feature Union (PCA + TruncatedSVD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# transforms for the feature union\n",
    "transforms = list()\n",
    "transforms.append(('norm', MinMaxScaler()))\n",
    "transforms.append(('Stnd', StandardScaler()))\n",
    "transforms.append(('pca', PCA()))\n",
    "transforms.append(('svd', TruncatedSVD()))\n",
    "\n",
    "# create the feature union\n",
    "fu = FeatureUnion(transforms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE Mean : 129434.03817500819\n",
      "RMSE SD : 10857.366984540666\n",
      "Execution time :  0:02:27.711553\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "model = XGBRegressor()\n",
    "# define pipeline\n",
    "steps = list()\n",
    "steps.append(('fu', fu))\n",
    "steps.append(('m', model))\n",
    "pipeline = Pipeline(steps=steps)\n",
    "scores = evaluate_model(X, y, pipeline)\n",
    "display_scores(scores)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### XGB with Normalization and PowerTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE Mean: 120431.06133167246\n",
      "RMSE SD: 10085.271522888474\n",
      "Execution time :  0:00:33.817571\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "model = XGBRegressor()\n",
    "# define pipeline\n",
    "steps = list()\n",
    "steps.append(('norm', MinMaxScaler()))\n",
    "steps.append(('power', PowerTransformer()))\n",
    "steps.append(('m', model))\n",
    "pipeline = Pipeline(steps=steps)\n",
    "scores = evaluate_model(X, y, pipeline)\n",
    "display_scores(scores)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### XGB with Normalization & PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE Mean: 167931.7710757437\n",
      "RMSE SD: 13606.865201620762\n",
      "Execution time :  0:01:12.338696\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "model = XGBRegressor()\n",
    "# define pipeline\n",
    "steps = list()\n",
    "steps.append(('norm', MinMaxScaler()))\n",
    "steps.append(('pca', PCA()))\n",
    "steps.append(('m', model))\n",
    "pipeline = Pipeline(steps=steps)\n",
    "scores = evaluate_model(X, y, pipeline)\n",
    "display_scores(scores)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### XGB with Normalization, PCA and PowerTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE Mean: 164418.13908667807\n",
      "RMSE SD: 14092.657334348532\n",
      "Execution time :  0:01:15.276995\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "model = XGBRegressor()\n",
    "# define pipeline\n",
    "steps = list()\n",
    "steps.append(('norm', MinMaxScaler()))\n",
    "steps.append(('power', PowerTransformer()))\n",
    "steps.append(('pca', PCA()))\n",
    "steps.append(('m', model))\n",
    "pipeline = Pipeline(steps=steps)\n",
    "scores = evaluate_model(X, y, pipeline)\n",
    "display_scores(scores)\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGB with Grid Search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### Train, Test Validation Data Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.20, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### Data Scaling & Transforming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "scaler = MinMaxScaler()\n",
    "transformer = PowerTransformer()\n",
    "\n",
    "scaler.fit(X_train)\n",
    "X_train_scaled = scaler.transform(X_train)\n",
    "X_test_scaled = scaler.transform(X_test)\n",
    "X_valid_scaled = scaler.transform(X_valid)\n",
    "\n",
    "transformer.fit(X_train_scaled)\n",
    "X_train_transformed = transformer.transform(X_train_scaled)\n",
    "X_test_transformed = transformer.transform(X_test_scaled)\n",
    "X_valid_transformed = transformer.transform(X_valid_scaled)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Randomized Search Function "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_RandomizedSearch_model(parameters):\n",
    "    xgb_reg = XGBRegressor(objective='reg:squarederror')\n",
    "    xgb_grid = RandomizedSearchCV(xgb_reg,\n",
    "                        parameters,\n",
    "                        scoring='neg_mean_squared_error',\n",
    "                        cv = 2,\n",
    "                        n_jobs = 10,\n",
    "                        verbose=False)\n",
    "    xgb_grid.fit(X_train, y_train)\n",
    "    return xgb_grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### First Grid Search Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = {\n",
    "                'n_estimators': [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000],\n",
    "                'max_depth': [3, 4, 5, 6, 7, 8, 9, 10],\n",
    "                'learning_rate': [0.0001, 0.001, 0.01, 0.02, 0.1, 0.2, 0.3], \n",
    "                'subsample' : [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 1.0],\n",
    "                'colsample_bytree' : [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 1.0],\n",
    "                'colsample_bylevel' : [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 1.0],\n",
    "                'gamma': [0],\n",
    "                'reg_lambda': [0, 1.0, 3.0, 5.0, 7.0, 10.0, 12.0]\n",
    "              }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Execution time :  0:05:05.905865\n",
      "Minimum Error: 119579.95917679991\n",
      "Parameters: {'subsample': 0.8, 'reg_lambda': 3.0, 'n_estimators': 400, 'max_depth': 6, 'learning_rate': 0.1, 'gamma': 0, 'colsample_bytree': 0.7, 'colsample_bylevel': 0.8}\n"
     ]
    }
   ],
   "source": [
    "init_time = datetime.now()\n",
    "results = dict()\n",
    "for _ in range(10):\n",
    "  xgb_grid = get_RandomizedSearch_model(parameters)\n",
    "  xgb_predictions = xgb_grid.predict(X_test)\n",
    "  rmse = np.sqrt(metrics.mean_squared_error(y_test, xgb_predictions))\n",
    "  results[rmse] = xgb_grid.best_params_\n",
    "\n",
    "fin_time = datetime.now()\n",
    "print(\"Execution time : \", (fin_time-init_time))\n",
    "min_error = min(results.keys())\n",
    "print(f\"Minimum Error: {min_error}\")\n",
    "print(f\"Parameters: {results[min_error]}\")\n",
    "\n",
    "#Minimum Error: 117461.46650171727"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### Grid Search Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def get_GridSearch_model(parameters):\n",
    "    init_time = datetime.now()\n",
    "    xgb_reg = XGBRegressor(objective='reg:squarederror')\n",
    "    # define pipeline\n",
    "    steps = list()\n",
    "    steps.append(('norm', MinMaxScaler()))\n",
    "    steps.append(('power', PowerTransformer()))\n",
    "    steps.append(('xgb', xgb_reg))\n",
    "    pipeline = Pipeline(steps=steps)\n",
    "    xgb_grid = GridSearchCV(pipeline,\n",
    "                        parameters,\n",
    "                        scoring='neg_mean_squared_error',\n",
    "                        cv = 5,\n",
    "                        n_jobs = 10,\n",
    "                        verbose=False)\n",
    "    xgb_grid.fit(X_train, y_train)\n",
    "    xgb_predictions = xgb_grid.predict(X_test)\n",
    "    print('RMSE:', np.sqrt(metrics.mean_squared_error(y_test, xgb_predictions)))\n",
    "    #print(f\"Param: {xgb_grid.best_params_}\")\n",
    "    fin_time = datetime.now()\n",
    "    print(\"Execution time : \", (fin_time-init_time))\n",
    "    \n",
    "    return xgb_grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 114439.38493937258\n",
      "Execution time :  0:00:21.328859\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    'xgb__subsample': [0.8], \n",
    "    'xgb__reg_lambda': [9.0], \n",
    "    'xgb__n_estimators': [900], \n",
    "    'xgb__max_depth': [5], \n",
    "    'xgb__learning_rate': [0.1], \n",
    "    'xgb__gamma': [0], \n",
    "    'xgb__colsample_bytree': [0.6], \n",
    "    'xgb__colsample_bylevel': [0.9]\n",
    "}\n",
    "model = get_GridSearch_model(parameters)\n",
    "\n",
    "#RMSE: 114726.49126301783\n",
    "#Param: {'colsample_bylevel': 0.9, 'colsample_bytree': 0.6, 'gamma': 0, 'learning_rate': 0.1, \n",
    "#'max_depth': 5, 'n_estimators': 900, 'reg_lambda': 9.0, 'subsample': 0.8 \n",
    "\n",
    "#RMSE: 114439.38493937258 with Norm + Power"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "535407.6787037037\n"
     ]
    }
   ],
   "source": [
    "avgValue = np.mean(y_test)\n",
    "print(avgValue)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### Save Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# write the actual face recognition model to disk\n",
    "f = open('best_param_model', \"wb\")\n",
    "f.write(pickle.dumps(model.best_estimator_))\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# write the label encoder to disk\n",
    "#le = LabelEncoder()\n",
    "#labels = le.fit_transform(data[\"names\"])\n",
    "#f = open(args[\"le\"], \"wb\")\n",
    "#f.write(pickle.dumps(le))\n",
    "#f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### Load Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 114439.38493937258\n"
     ]
    }
   ],
   "source": [
    "model = pickle.loads(open('best_param_model', \"rb\").read())\n",
    "#le = pickle.loads(open(args[\"le\"], \"rb\").read())\n",
    "predictions = model.predict(X_test)\n",
    "print('RMSE:', np.sqrt(metrics.mean_squared_error(y_test, predictions)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}