2018/day40_house_price_prediction_modelling_ml.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# House price prediction using multiple regression analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/adhiman/anaconda3/envs/python3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # NumPy is the fundamental package for scientific computing\n",
    "\n",
    "import pandas as pd # Pandas is an easy-to-use data structures and data analysis tools\n",
    "pd.set_option('display.max_columns', None) # To display all columns\n",
    "\n",
    "import matplotlib.pyplot as plt # Matplotlib is a python 2D plotting library\n",
    "%matplotlib inline \n",
    "# A magic command that tells matplotlib to render figures as static images in the Notebook.\n",
    "\n",
    "import seaborn as sns # Seaborn is a visualization library based on matplotlib (attractive statistical graphics).\n",
    "sns.set_style('whitegrid') # One of the five seaborn themes\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore') # To ignore some of seaborn warning msg\n",
    "\n",
    "from scipy import stats\n",
    "\n",
    "from sklearn import linear_model # Scikit learn library that implements generalized linear models\n",
    "from sklearn import neighbors # provides functionality for unsupervised and supervised neighbors-based learning methods\n",
    "from sklearn.metrics import mean_squared_error # Mean squared error regression loss\n",
    "from sklearn import preprocessing # provides functions and classes to change raw feature vectors\n",
    "\n",
    "from math import log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date</th>\n",
       "      <th>price</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>sqft_living</th>\n",
       "      <th>sqft_lot</th>\n",
       "      <th>floors</th>\n",
       "      <th>waterfront</th>\n",
       "      <th>view</th>\n",
       "      <th>condition</th>\n",
       "      <th>grade</th>\n",
       "      <th>sqft_above</th>\n",
       "      <th>sqft_basement</th>\n",
       "      <th>yr_built</th>\n",
       "      <th>yr_renovated</th>\n",
       "      <th>zipcode</th>\n",
       "      <th>lat</th>\n",
       "      <th>long</th>\n",
       "      <th>sqft_living15</th>\n",
       "      <th>sqft_lot15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7129300520</td>\n",
       "      <td>2014-10-13</td>\n",
       "      <td>221900.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1180</td>\n",
       "      <td>5650</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>1180</td>\n",
       "      <td>0</td>\n",
       "      <td>1955</td>\n",
       "      <td>0</td>\n",
       "      <td>98178</td>\n",
       "      <td>47.5112</td>\n",
       "      <td>-122.257</td>\n",
       "      <td>1340</td>\n",
       "      <td>5650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6414100192</td>\n",
       "      <td>2014-12-09</td>\n",
       "      <td>538000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2.25</td>\n",
       "      <td>2570</td>\n",
       "      <td>7242</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>2170</td>\n",
       "      <td>400</td>\n",
       "      <td>1951</td>\n",
       "      <td>1991</td>\n",
       "      <td>98125</td>\n",
       "      <td>47.7210</td>\n",
       "      <td>-122.319</td>\n",
       "      <td>1690</td>\n",
       "      <td>7639</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id       date     price  bedrooms  bathrooms  sqft_living  \\\n",
       "0  7129300520 2014-10-13  221900.0         3       1.00         1180   \n",
       "1  6414100192 2014-12-09  538000.0         3       2.25         2570   \n",
       "\n",
       "   sqft_lot  floors  waterfront  view  condition  grade  sqft_above  \\\n",
       "0      5650     1.0           0     0          3      7        1180   \n",
       "1      7242     2.0           0     0          3      7        2170   \n",
       "\n",
       "   sqft_basement  yr_built  yr_renovated  zipcode      lat     long  \\\n",
       "0              0      1955             0    98178  47.5112 -122.257   \n",
       "1            400      1951          1991    98125  47.7210 -122.319   \n",
       "\n",
       "   sqft_living15  sqft_lot15  \n",
       "0           1340        5650  \n",
       "1           1690        7639  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"data/kc_house_data.csv.zip\", parse_dates = ['date']) # load the data into a pandas dataframe\n",
    "data.head(2) # Show the first 2 lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.drop(['id', 'date'], axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['basement_present'] = data['sqft_basement'].apply(lambda x: 1 if x > 0 else 0) # Indicate whether there is a basement or not\n",
    "data['renovated'] = data['yr_renovated'].apply(lambda x: 1 if x > 0 else 0) # 1 if the house has been renovated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorial_cols = ['floors', 'view', 'condition', 'grade']\n",
    "\n",
    "for cc in categorial_cols:\n",
    "    dummies = pd.get_dummies(data[cc], drop_first=False)\n",
    "    dummies = dummies.add_prefix(\"{}#\".format(cc))\n",
    "    data.drop(cc, axis=1, inplace=True)\n",
    "    data = data.join(dummies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "price               float64\n",
       "bedrooms              int64\n",
       "bathrooms           float64\n",
       "sqft_living           int64\n",
       "sqft_lot              int64\n",
       "waterfront            int64\n",
       "sqft_above            int64\n",
       "sqft_basement         int64\n",
       "yr_built              int64\n",
       "yr_renovated          int64\n",
       "lat                 float64\n",
       "long                float64\n",
       "sqft_living15         int64\n",
       "sqft_lot15            int64\n",
       "basement_present      int64\n",
       "renovated             int64\n",
       "floors#1.0            uint8\n",
       "floors#1.5            uint8\n",
       "floors#2.0            uint8\n",
       "floors#2.5            uint8\n",
       "floors#3.0            uint8\n",
       "floors#3.5            uint8\n",
       "view#0                uint8\n",
       "view#1                uint8\n",
       "view#2                uint8\n",
       "view#3                uint8\n",
       "view#4                uint8\n",
       "condition#1           uint8\n",
       "condition#2           uint8\n",
       "condition#3           uint8\n",
       "condition#4           uint8\n",
       "condition#5           uint8\n",
       "grade#1               uint8\n",
       "grade#3               uint8\n",
       "grade#4               uint8\n",
       "grade#5               uint8\n",
       "grade#6               uint8\n",
       "grade#7               uint8\n",
       "grade#8               uint8\n",
       "grade#9               uint8\n",
       "grade#10              uint8\n",
       "grade#11              uint8\n",
       "grade#12              uint8\n",
       "grade#13              uint8\n",
       "zipcode#98004         uint8\n",
       "zipcode#98102         uint8\n",
       "zipcode#98109         uint8\n",
       "zipcode#98112         uint8\n",
       "zipcode#98039         uint8\n",
       "zipcode#98040         uint8\n",
       "dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummies_zipcodes = pd.get_dummies(data['zipcode'], drop_first=False)\n",
    "dummies_zipcodes.reset_index(inplace=True)\n",
    "dummies_zipcodes = dummies_zipcodes.add_prefix(\"{}#\".format('zipcode'))\n",
    "dummies_zipcodes = dummies_zipcodes[['zipcode#98004','zipcode#98102','zipcode#98109','zipcode#98112','zipcode#98039','zipcode#98040']]\n",
    "data.drop('zipcode', axis=1, inplace=True)\n",
    "data = data.join(dummies_zipcodes)\n",
    "\n",
    "data.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/adhiman/anaconda3/envs/python3/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "train_data, test_data = train_test_split(data, train_size = 0.8, random_state = 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regression models\n",
    "\n",
    "### Simple linear regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A function that take one input of the dataset and return the RMSE (of the test data), and the intercept and coefficient\n",
    "def simple_linear_model(train, test, input_feature):\n",
    "    regr = linear_model.LinearRegression() # Create a linear regression object\n",
    "    regr.fit(train.as_matrix(columns = [input_feature]), train.as_matrix(columns = ['price'])) # Train the model\n",
    "    RMSE = mean_squared_error(test.as_matrix(columns = ['price']), \n",
    "                              regr.predict(test.as_matrix(columns = [input_feature])))**0.5 # Calculate the RMSE on test data\n",
    "    return RMSE, regr.intercept_[0], regr.coef_[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE for sqft_living is: 268279.6438833362 \n",
      "intercept is: -36738.177346383105\n",
      "coefficient is: 277.36412987021157\n"
     ]
    }
   ],
   "source": [
    "RMSE, w0, w1 = simple_linear_model(train_data, test_data, 'sqft_living')\n",
    "print ('RMSE for sqft_living is: %s ' %RMSE)\n",
    "print ('intercept is: %s' %w0)\n",
    "print ('coefficient is: %s' %w1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>intercept</th>\n",
       "      <th>coefficient</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sqft_living</td>\n",
       "      <td>268279.643883</td>\n",
       "      <td>277.364130</td>\n",
       "      <td>-3.673818e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>sqft_above</td>\n",
       "      <td>304131.310592</td>\n",
       "      <td>266.306764</td>\n",
       "      <td>6.461714e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>sqft_living15</td>\n",
       "      <td>320686.541323</td>\n",
       "      <td>314.359911</td>\n",
       "      <td>-8.502590e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bathrooms</td>\n",
       "      <td>324082.781919</td>\n",
       "      <td>246523.891877</td>\n",
       "      <td>1.863279e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>view#0</td>\n",
       "      <td>356019.001320</td>\n",
       "      <td>-435033.777431</td>\n",
       "      <td>9.322014e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>sqft_basement</td>\n",
       "      <td>357843.745395</td>\n",
       "      <td>258.126523</td>\n",
       "      <td>4.642966e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>grade#11</td>\n",
       "      <td>357964.423743</td>\n",
       "      <td>965286.415396</td>\n",
       "      <td>5.222663e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>grade#10</td>\n",
       "      <td>360773.700418</td>\n",
       "      <td>556992.601325</td>\n",
       "      <td>5.107024e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bedrooms</td>\n",
       "      <td>361295.375626</td>\n",
       "      <td>117579.891853</td>\n",
       "      <td>1.436815e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>lat</td>\n",
       "      <td>365041.433662</td>\n",
       "      <td>814499.981062</td>\n",
       "      <td>-3.819789e+07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          feature           RMSE      intercept   coefficient\n",
       "2     sqft_living  268279.643883     277.364130 -3.673818e+04\n",
       "5      sqft_above  304131.310592     266.306764  6.461714e+04\n",
       "11  sqft_living15  320686.541323     314.359911 -8.502590e+04\n",
       "1       bathrooms  324082.781919  246523.891877  1.863279e+04\n",
       "21         view#0  356019.001320 -435033.777431  9.322014e+05\n",
       "6   sqft_basement  357843.745395     258.126523  4.642966e+05\n",
       "40       grade#11  357964.423743  965286.415396  5.222663e+05\n",
       "39       grade#10  360773.700418  556992.601325  5.107024e+05\n",
       "0        bedrooms  361295.375626  117579.891853  1.436815e+05\n",
       "9             lat  365041.433662  814499.981062 -3.819789e+07"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_list = data.columns.values.tolist() # list of column name\n",
    "input_list.remove('price')\n",
    "simple_linear_result = pd.DataFrame(columns = ['feature', 'RMSE', 'intercept', 'coefficient'])\n",
    "\n",
    "# loop that calculate the RMSE of the test data for each input \n",
    "for p in input_list:\n",
    "    RMSE, w1, w0 = simple_linear_model(train_data, test_data, p)\n",
    "    simple_linear_result = simple_linear_result.append({'feature':p, 'RMSE':RMSE, 'intercept':w0, 'coefficient': w1}\n",
    "                                                       ,ignore_index=True)\n",
    "simple_linear_result.sort_values('RMSE').head(10) # display the 10 best estimators"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multiple linear regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A function that take multiple features as input and return the RMSE (of the test data), and the  intercept and coefficients\n",
    "def multiple_regression_model(train, test, input_features):\n",
    "    regr = linear_model.LinearRegression() # Create a linear regression object\n",
    "    regr.fit(train.as_matrix(columns = input_features), train.as_matrix(columns = ['price'])) # Train the model\n",
    "    RMSE = mean_squared_error(test.as_matrix(columns = ['price']), \n",
    "                              regr.predict(test.as_matrix(columns = input_features)))**0.5 # Calculate the RMSE on test data\n",
    "    return RMSE, regr.intercept_[0], regr.coef_ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 264872.2835550953, intercept: 81100.95967753517, coefficients: [[   306.15090562   7913.53847651 -57658.90103459]]\n",
      "RMSE: 282802.3649615518, intercept: 303531.2921883529, coefficients: [[ 1.98928518e+02 -3.17760670e+05  7.92684094e+04]]\n",
      "RMSE: 323412.2692762603, intercept: -18432.90570721531, coefficients: [[235300.89998266  18030.65120532]]\n",
      "RMSE: 320893.6584322429, intercept: 507958.8899922919, coefficients: [[-3.31957675e+05  1.35299763e+06  8.56179339e+04  1.24073575e+02]]\n",
      "RMSE: 260210.04852365193, intercept: 205076.198921312, coefficients: [[    258.00653033    -244.89749378 -223120.61245789]]\n"
     ]
    }
   ],
   "source": [
    "print ('RMSE: %s, intercept: %s, coefficients: %s' %multiple_regression_model(train_data, \n",
    "                                                                             test_data, ['sqft_living','bathrooms','bedrooms']))\n",
    "print ('RMSE: %s, intercept: %s, coefficients: %s' %multiple_regression_model(train_data, \n",
    "                                                                             test_data, ['sqft_above','view#0','bathrooms']))\n",
    "print ('RMSE: %s, intercept: %s, coefficients: %s' %multiple_regression_model(train_data, \n",
    "                                                                             test_data, ['bathrooms','bedrooms']))\n",
    "print ('RMSE: %s, intercept: %s, coefficients: %s' %multiple_regression_model(train_data, \n",
    "                                                                             test_data, ['view#0','grade#12','bedrooms','sqft_basement']))\n",
    "print ('RMSE: %s, intercept: %s, coefficients: %s' %multiple_regression_model(train_data, \n",
    "                                                                             test_data, ['sqft_living','bathrooms','view#0']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 246063.95920692245, intercept: 172106.56010562018, coefficients: [[9.31721994e+01 3.37454932e-02]]\n"
     ]
    }
   ],
   "source": [
    "train_data['sqft_living_squared'] = train_data['sqft_living'].apply(lambda x: x**2) # create a new column in train_data\n",
    "test_data['sqft_living_squared'] = test_data['sqft_living'].apply(lambda x: x**2) # create a new column in test_data\n",
    "print ('RMSE: %s, intercept: %s, coefficients: %s' %multiple_regression_model(train_data, \n",
    "                                                                             test_data, ['sqft_living','sqft_living_squared']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Which features to select?**\n",
    "\n",
    "Lets use a greedy technique like a forward stepwise algorithm where the best estimator feature is added to the set of already selected features at each iteration. For example, let's pretend that the best single estimator is `sqft_living`. In the 2nd step of the greedy algorithm, we test all the remaining features one by one in combinations with `sqft_living` (e.g., `sqft_living` and `bedrooms`, `sqft_living` and `waterfront`, etc) and select the best combination using training error. \n",
    "\n",
    "At the end, we select the model complexity (number of features) using the validation error and estimate the generalization error using the test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(17290, 55)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# we're first going to add more features into the dataset.\n",
    "\n",
    "# sqft_living cubed\n",
    "train_data['sqft_living_cubed'] = train_data['sqft_living'].apply(lambda x: x**3) \n",
    "test_data['sqft_living_cubed'] = test_data['sqft_living'].apply(lambda x: x**3) \n",
    "\n",
    "# bedrooms_squared: this feature will mostly affect houses with many bedrooms.\n",
    "train_data['bedrooms_squared'] = train_data['bedrooms'].apply(lambda x: x**2) \n",
    "test_data['bedrooms_squared'] = test_data['bedrooms'].apply(lambda x: x**2)\n",
    "\n",
    "# bedrooms times bathrooms gives what's called an \"interaction\" feature. It is large when both of them are large.\n",
    "train_data['bed_bath_rooms'] = train_data['bedrooms']*train_data['bathrooms']\n",
    "test_data['bed_bath_rooms'] = test_data['bedrooms']*test_data['bathrooms']\n",
    "\n",
    "# Taking the log of squarefeet has the effect of bringing large values closer together and spreading out small values.\n",
    "train_data['log_sqft_living'] = train_data['sqft_living'].apply(lambda x: log(x))\n",
    "test_data['log_sqft_living'] = test_data['sqft_living'].apply(lambda x: log(x))\n",
    "\n",
    "train_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split the train_data to include a validation set (train_data2 = 60%, validation_data = 20%, test_data = 20%)\n",
    "train_data_2, validation_data = train_test_split(train_data, train_size = 0.75, random_state = 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A function that take multiple features as input and return the RMSE (of the train and validation data)\n",
    "def RMSE(train, validation, features, new_input):\n",
    "    features_list = list(features)\n",
    "    features_list.append(new_input)\n",
    "    regr = linear_model.LinearRegression() # Create a linear regression object\n",
    "    regr.fit(train.as_matrix(columns = features_list), train.as_matrix(columns = ['price'])) # Train the model\n",
    "    RMSE_train = mean_squared_error(train.as_matrix(columns = ['price']), \n",
    "                              regr.predict(train.as_matrix(columns = features_list)))**0.5 # Calculate the RMSE on train data\n",
    "    RMSE_validation = mean_squared_error(validation.as_matrix(columns = ['price']), \n",
    "                              regr.predict(validation.as_matrix(columns = features_list)))**0.5 # Calculate the RMSE on train data\n",
    "    return RMSE_train, RMSE_validation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>train_error</th>\n",
       "      <th>validation_error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sqft_living_squared</td>\n",
       "      <td>249721.440480</td>\n",
       "      <td>268747.213950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>lat</td>\n",
       "      <td>229419.062478</td>\n",
       "      <td>252069.707424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>waterfront</td>\n",
       "      <td>219240.069871</td>\n",
       "      <td>241979.268406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>zipcode#98004</td>\n",
       "      <td>208411.508254</td>\n",
       "      <td>229000.747130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>zipcode#98112</td>\n",
       "      <td>202604.911359</td>\n",
       "      <td>225124.826562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>zipcode#98039</td>\n",
       "      <td>196885.794407</td>\n",
       "      <td>217328.215296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>view#0</td>\n",
       "      <td>191119.193990</td>\n",
       "      <td>209679.561580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>zipcode#98040</td>\n",
       "      <td>186849.129730</td>\n",
       "      <td>204450.501487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>sqft_living15</td>\n",
       "      <td>185134.594156</td>\n",
       "      <td>197733.047694</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>grade#13</td>\n",
       "      <td>183383.653232</td>\n",
       "      <td>197321.800871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>condition#5</td>\n",
       "      <td>181891.092067</td>\n",
       "      <td>196541.377903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>grade#11</td>\n",
       "      <td>180532.105926</td>\n",
       "      <td>194534.976260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>grade#10</td>\n",
       "      <td>178492.121187</td>\n",
       "      <td>190462.363671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>grade#12</td>\n",
       "      <td>175704.660910</td>\n",
       "      <td>185437.534555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>grade#9</td>\n",
       "      <td>172510.818377</td>\n",
       "      <td>179597.162787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>grade#8</td>\n",
       "      <td>170635.242038</td>\n",
       "      <td>176297.524718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>yr_built</td>\n",
       "      <td>167977.636383</td>\n",
       "      <td>173665.533961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>bathrooms</td>\n",
       "      <td>166456.154777</td>\n",
       "      <td>170353.745230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>view#4</td>\n",
       "      <td>165260.808366</td>\n",
       "      <td>170390.119346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>grade#7</td>\n",
       "      <td>164246.940788</td>\n",
       "      <td>168940.611700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>zipcode#98109</td>\n",
       "      <td>163458.010133</td>\n",
       "      <td>168140.236755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>zipcode#98102</td>\n",
       "      <td>162926.763099</td>\n",
       "      <td>167535.616923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>view#3</td>\n",
       "      <td>162643.227969</td>\n",
       "      <td>167797.150367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>floors#1.0</td>\n",
       "      <td>162364.749446</td>\n",
       "      <td>167570.147704</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>condition#4</td>\n",
       "      <td>162071.950582</td>\n",
       "      <td>167362.587289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>sqft_living_cubed</td>\n",
       "      <td>161840.708258</td>\n",
       "      <td>177707.843165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>bed_bath_rooms</td>\n",
       "      <td>161677.933947</td>\n",
       "      <td>176365.316470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>long</td>\n",
       "      <td>161531.531519</td>\n",
       "      <td>176130.989418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>yr_renovated</td>\n",
       "      <td>161431.315244</td>\n",
       "      <td>176382.302595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>renovated</td>\n",
       "      <td>161074.570625</td>\n",
       "      <td>175973.235549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>floors#2.5</td>\n",
       "      <td>160989.462058</td>\n",
       "      <td>175932.983373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>basement_present</td>\n",
       "      <td>160917.920082</td>\n",
       "      <td>176837.875928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>sqft_above</td>\n",
       "      <td>160757.391978</td>\n",
       "      <td>179877.263575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>sqft_lot15</td>\n",
       "      <td>160729.840694</td>\n",
       "      <td>179740.258551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>sqft_lot</td>\n",
       "      <td>160687.168393</td>\n",
       "      <td>179658.484669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>grade#5</td>\n",
       "      <td>160665.085264</td>\n",
       "      <td>179610.203557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>view#1</td>\n",
       "      <td>160647.462962</td>\n",
       "      <td>179635.831098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>floors#3.5</td>\n",
       "      <td>160634.745799</td>\n",
       "      <td>179652.204039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>bedrooms_squared</td>\n",
       "      <td>160622.498961</td>\n",
       "      <td>179667.784872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>grade#3</td>\n",
       "      <td>160613.317262</td>\n",
       "      <td>179659.223511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>condition#3</td>\n",
       "      <td>160605.906251</td>\n",
       "      <td>179667.873423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>floors#3.0</td>\n",
       "      <td>160599.095301</td>\n",
       "      <td>179650.091515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>bedrooms</td>\n",
       "      <td>160595.259103</td>\n",
       "      <td>179351.104058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>sqft_basement</td>\n",
       "      <td>160593.819733</td>\n",
       "      <td>180426.961334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>log_sqft_living</td>\n",
       "      <td>160592.873743</td>\n",
       "      <td>181502.353584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>grade#4</td>\n",
       "      <td>160592.296966</td>\n",
       "      <td>181675.251253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>floors#2.0</td>\n",
       "      <td>160591.892814</td>\n",
       "      <td>181690.559092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>condition#2</td>\n",
       "      <td>160591.534791</td>\n",
       "      <td>181688.669859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>grade#6</td>\n",
       "      <td>160591.534790</td>\n",
       "      <td>181688.433984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>grade#1</td>\n",
       "      <td>160591.534790</td>\n",
       "      <td>181688.433984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>floors#1.5</td>\n",
       "      <td>160591.534792</td>\n",
       "      <td>181688.895179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>view#2</td>\n",
       "      <td>160591.534790</td>\n",
       "      <td>181688.445838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>condition#1</td>\n",
       "      <td>160591.534816</td>\n",
       "      <td>181690.398960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>sqft_living</td>\n",
       "      <td>160591.534798</td>\n",
       "      <td>181690.100941</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                feature    train_error  validation_error\n",
       "0   sqft_living_squared  249721.440480     268747.213950\n",
       "1                   lat  229419.062478     252069.707424\n",
       "2            waterfront  219240.069871     241979.268406\n",
       "3         zipcode#98004  208411.508254     229000.747130\n",
       "4         zipcode#98112  202604.911359     225124.826562\n",
       "5         zipcode#98039  196885.794407     217328.215296\n",
       "6                view#0  191119.193990     209679.561580\n",
       "7         zipcode#98040  186849.129730     204450.501487\n",
       "8         sqft_living15  185134.594156     197733.047694\n",
       "9              grade#13  183383.653232     197321.800871\n",
       "10          condition#5  181891.092067     196541.377903\n",
       "11             grade#11  180532.105926     194534.976260\n",
       "12             grade#10  178492.121187     190462.363671\n",
       "13             grade#12  175704.660910     185437.534555\n",
       "14              grade#9  172510.818377     179597.162787\n",
       "15              grade#8  170635.242038     176297.524718\n",
       "16             yr_built  167977.636383     173665.533961\n",
       "17            bathrooms  166456.154777     170353.745230\n",
       "18               view#4  165260.808366     170390.119346\n",
       "19              grade#7  164246.940788     168940.611700\n",
       "20        zipcode#98109  163458.010133     168140.236755\n",
       "21        zipcode#98102  162926.763099     167535.616923\n",
       "22               view#3  162643.227969     167797.150367\n",
       "23           floors#1.0  162364.749446     167570.147704\n",
       "24          condition#4  162071.950582     167362.587289\n",
       "25    sqft_living_cubed  161840.708258     177707.843165\n",
       "26       bed_bath_rooms  161677.933947     176365.316470\n",
       "27                 long  161531.531519     176130.989418\n",
       "28         yr_renovated  161431.315244     176382.302595\n",
       "29            renovated  161074.570625     175973.235549\n",
       "30           floors#2.5  160989.462058     175932.983373\n",
       "31     basement_present  160917.920082     176837.875928\n",
       "32           sqft_above  160757.391978     179877.263575\n",
       "33           sqft_lot15  160729.840694     179740.258551\n",
       "34             sqft_lot  160687.168393     179658.484669\n",
       "35              grade#5  160665.085264     179610.203557\n",
       "36               view#1  160647.462962     179635.831098\n",
       "37           floors#3.5  160634.745799     179652.204039\n",
       "38     bedrooms_squared  160622.498961     179667.784872\n",
       "39              grade#3  160613.317262     179659.223511\n",
       "40          condition#3  160605.906251     179667.873423\n",
       "41           floors#3.0  160599.095301     179650.091515\n",
       "42             bedrooms  160595.259103     179351.104058\n",
       "43        sqft_basement  160593.819733     180426.961334\n",
       "44      log_sqft_living  160592.873743     181502.353584\n",
       "45              grade#4  160592.296966     181675.251253\n",
       "46           floors#2.0  160591.892814     181690.559092\n",
       "47          condition#2  160591.534791     181688.669859\n",
       "48              grade#6  160591.534790     181688.433984\n",
       "49              grade#1  160591.534790     181688.433984\n",
       "50           floors#1.5  160591.534792     181688.895179\n",
       "51               view#2  160591.534790     181688.445838\n",
       "52          condition#1  160591.534816     181690.398960\n",
       "53          sqft_living  160591.534798     181690.100941"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_list = train_data_2.columns.values.tolist() # list of column name\n",
    "input_list.remove('price')\n",
    "\n",
    "# list of features included in the regression model and the calculated train and validation errors (RMSE)\n",
    "regression_greedy_algorithm = pd.DataFrame(columns = ['feature', 'train_error', 'validation_error'])  \n",
    "i = 0\n",
    "temp_list = []\n",
    "\n",
    "# a while loop going through all the features in the dataframe\n",
    "while i < len(train_data_2.columns)-1:\n",
    "    \n",
    "    # a temporary dataframe to select the best feature at each iteration\n",
    "    temp = pd.DataFrame(columns = ['feature', 'train_error', 'validation_error'])\n",
    "    \n",
    "    # a for loop to test all the remaining features\n",
    "    for p in input_list:\n",
    "        RMSE_train, RMSE_validation = RMSE(train_data_2, validation_data, temp_list, p)\n",
    "        temp = temp.append({'feature':p, 'train_error':RMSE_train, 'validation_error':RMSE_validation}, ignore_index=True)\n",
    "        \n",
    "    temp = temp.sort_values('train_error') # select the best feature using train error\n",
    "    best = temp.iloc[0,0]\n",
    "    temp_list.append(best)\n",
    "    regression_greedy_algorithm = regression_greedy_algorithm.append({'feature': best, \n",
    "                                                  'train_error': temp.iloc[0,1], 'validation_error': temp.iloc[0,2]}, \n",
    "                                                 ignore_index=True) # add the feature to the dataframe\n",
    "    input_list.remove(best) # remove the best feature from the list of available features\n",
    "    i += 1\n",
    "regression_greedy_algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test error (RMSE) is: 167570.14770428356\n"
     ]
    }
   ],
   "source": [
    "greedy_algo_features_list = regression_greedy_algorithm['feature'].tolist()[:24] # select the first 30 features\n",
    "test_error, _, _ = multiple_regression_model(train_data_2, validation_data, greedy_algo_features_list)\n",
    "print ('test error (RMSE) is: %s' %test_error)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ridge regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test error (RMSE) is: 190153.37221800248\n",
      "test error (RMSE) is: 287976.6316505086\n"
     ]
    }
   ],
   "source": [
    "input_feature = train_data.columns.values.tolist() # list of column name\n",
    "input_feature.remove('price')\n",
    "\n",
    "for i in [1,10]:\n",
    "    ridge = linear_model.Ridge(alpha = i, normalize = True) # initialize the model\n",
    "    ridge.fit(train_data.as_matrix(columns = input_feature), train_data.as_matrix(columns = ['price'])) # fit the train data\n",
    "    print ('test error (RMSE) is: %s' %mean_squared_error(test_data.as_matrix(columns = ['price']), \n",
    "                              ridge.predict(test_data.as_matrix(columns = input_feature)))**0.5) # predict price and test error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best alpha is: 0.0707070708\n",
      "test error (RMSE) is: 171566.79185920305\n"
     ]
    }
   ],
   "source": [
    "ridgeCV = linear_model.RidgeCV(alphas = np.linspace(1.0e-10,1,num = 100), normalize = True, store_cv_values = True) # initialize the model\n",
    "ridgeCV.fit(train_data.as_matrix(columns = input_feature), train_data.as_matrix(columns = ['price'])) # fit the train data\n",
    "print ('best alpha is: %s' %ridgeCV.alpha_) # get the best alpha\n",
    "print ('test error (RMSE) is: %s' %mean_squared_error(test_data.as_matrix(columns = ['price']), \n",
    "                              ridgeCV.predict(test_data.as_matrix(columns = input_feature)))**0.5) # predict price and test error"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lasso regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<bound method _cs_matrix.getnnz of <1x54 sparse matrix of type '<class 'numpy.float64'>'\n",
      "\twith 53 stored elements in Compressed Sparse Row format>>\n",
      "test error (RMSE) is: 174601.68260073222\n",
      "<bound method _cs_matrix.getnnz of <1x54 sparse matrix of type '<class 'numpy.float64'>'\n",
      "\twith 50 stored elements in Compressed Sparse Row format>>\n",
      "test error (RMSE) is: 174578.55878672577\n",
      "<bound method _cs_matrix.getnnz of <1x54 sparse matrix of type '<class 'numpy.float64'>'\n",
      "\twith 45 stored elements in Compressed Sparse Row format>>\n",
      "test error (RMSE) is: 174412.93966589426\n",
      "<bound method _cs_matrix.getnnz of <1x54 sparse matrix of type '<class 'numpy.float64'>'\n",
      "\twith 14 stored elements in Compressed Sparse Row format>>\n",
      "test error (RMSE) is: 203694.35400709457\n",
      "<bound method _cs_matrix.getnnz of <1x54 sparse matrix of type '<class 'numpy.float64'>'\n",
      "\twith 8 stored elements in Compressed Sparse Row format>>\n",
      "test error (RMSE) is: 241504.25938546794\n",
      "<bound method _cs_matrix.getnnz of <1x54 sparse matrix of type '<class 'numpy.float64'>'\n",
      "\twith 2 stored elements in Compressed Sparse Row format>>\n",
      "test error (RMSE) is: 289827.5897998794\n"
     ]
    }
   ],
   "source": [
    "for i in [0.01,0.1,1,250,500,1000]:\n",
    "    lasso = linear_model.Lasso(alpha = i, normalize = True) # initialize the model\n",
    "    lasso.fit(train_data.as_matrix(columns = input_feature), train_data.as_matrix(columns = ['price'])) # fit the train data\n",
    "    print (lasso.sparse_coef_.getnnz) # number of non zero weights\n",
    "    print ('test error (RMSE) is: %s' %mean_squared_error(test_data.as_matrix(columns = ['price']), \n",
    "                              lasso.predict(test_data.as_matrix(columns = input_feature)))**0.5) # predict price and test error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best alpha is: 14.936078017798543\n",
      "number of non zero weigths is: 39\n",
      "test error (RMSE) is: 171369.07571302887\n"
     ]
    }
   ],
   "source": [
    "lassoCV = linear_model.LassoCV(normalize = True) # initialize the model (alphas are set automatically)\n",
    "lassoCV.fit(train_data.as_matrix(columns = input_feature), np.ravel(train_data.as_matrix(columns = ['price']))) # fit the train data\n",
    "print ('best alpha is: %s' %lassoCV.alpha_) # get the best alpha\n",
    "print ('number of non zero weigths is: %s' %np.count_nonzero(lassoCV.coef_)) # number of non zero weights\n",
    "print ('test error (RMSE) is: %s' %mean_squared_error(test_data.as_matrix(columns = ['price']), \n",
    "                              lassoCV.predict(test_data.as_matrix(columns = input_feature)))**0.5) # predict price and test error"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNN Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test error (RMSE) is: 184936.00865427667\n"
     ]
    }
   ],
   "source": [
    "# normalize the data\n",
    "train_X = train_data.as_matrix(columns = input_feature)\n",
    "scaler = preprocessing.StandardScaler().fit(train_X)\n",
    "train_X_scaled = scaler.transform(train_X)\n",
    "test_X = test_data.as_matrix(columns = input_feature)\n",
    "test_X_scaled = scaler.transform(test_X)\n",
    "\n",
    "knn = neighbors.KNeighborsRegressor(n_neighbors=10, weights='distance') # initialize the model\n",
    "knn.fit(train_X_scaled, train_data.as_matrix(columns = ['price'])) # fit the train data\n",
    "print ('test error (RMSE) is: %s' %mean_squared_error(test_data.as_matrix(columns = ['price']), \n",
    "                              knn.predict(test_X_scaled))**0.5) # predict price and test error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}