data_preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing necessary libraries\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('ggplot')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date</th>\n",
       "      <th>price</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>sqft_living</th>\n",
       "      <th>sqft_lot</th>\n",
       "      <th>floors</th>\n",
       "      <th>waterfront</th>\n",
       "      <th>view</th>\n",
       "      <th>...</th>\n",
       "      <th>grade</th>\n",
       "      <th>sqft_above</th>\n",
       "      <th>sqft_basement</th>\n",
       "      <th>yr_built</th>\n",
       "      <th>yr_renovated</th>\n",
       "      <th>zipcode</th>\n",
       "      <th>lat</th>\n",
       "      <th>long</th>\n",
       "      <th>sqft_living15</th>\n",
       "      <th>sqft_lot15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7129300520</td>\n",
       "      <td>10/13/2014</td>\n",
       "      <td>221900.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1180</td>\n",
       "      <td>5650</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NONE</td>\n",
       "      <td>...</td>\n",
       "      <td>7 Average</td>\n",
       "      <td>1180</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1955</td>\n",
       "      <td>0.0</td>\n",
       "      <td>98178</td>\n",
       "      <td>47.5112</td>\n",
       "      <td>-122.257</td>\n",
       "      <td>1340</td>\n",
       "      <td>5650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6414100192</td>\n",
       "      <td>12/9/2014</td>\n",
       "      <td>538000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2.25</td>\n",
       "      <td>2570</td>\n",
       "      <td>7242</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>...</td>\n",
       "      <td>7 Average</td>\n",
       "      <td>2170</td>\n",
       "      <td>400.0</td>\n",
       "      <td>1951</td>\n",
       "      <td>1991.0</td>\n",
       "      <td>98125</td>\n",
       "      <td>47.7210</td>\n",
       "      <td>-122.319</td>\n",
       "      <td>1690</td>\n",
       "      <td>7639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5631500400</td>\n",
       "      <td>2/25/2015</td>\n",
       "      <td>180000.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.00</td>\n",
       "      <td>770</td>\n",
       "      <td>10000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>...</td>\n",
       "      <td>6 Low Average</td>\n",
       "      <td>770</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1933</td>\n",
       "      <td>NaN</td>\n",
       "      <td>98028</td>\n",
       "      <td>47.7379</td>\n",
       "      <td>-122.233</td>\n",
       "      <td>2720</td>\n",
       "      <td>8062</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2487200875</td>\n",
       "      <td>12/9/2014</td>\n",
       "      <td>604000.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3.00</td>\n",
       "      <td>1960</td>\n",
       "      <td>5000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>...</td>\n",
       "      <td>7 Average</td>\n",
       "      <td>1050</td>\n",
       "      <td>910.0</td>\n",
       "      <td>1965</td>\n",
       "      <td>0.0</td>\n",
       "      <td>98136</td>\n",
       "      <td>47.5208</td>\n",
       "      <td>-122.393</td>\n",
       "      <td>1360</td>\n",
       "      <td>5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1954400510</td>\n",
       "      <td>2/18/2015</td>\n",
       "      <td>510000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2.00</td>\n",
       "      <td>1680</td>\n",
       "      <td>8080</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>...</td>\n",
       "      <td>8 Good</td>\n",
       "      <td>1680</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1987</td>\n",
       "      <td>0.0</td>\n",
       "      <td>98074</td>\n",
       "      <td>47.6168</td>\n",
       "      <td>-122.045</td>\n",
       "      <td>1800</td>\n",
       "      <td>7503</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id        date     price  bedrooms  bathrooms  sqft_living  \\\n",
       "0  7129300520  10/13/2014  221900.0         3       1.00         1180   \n",
       "1  6414100192   12/9/2014  538000.0         3       2.25         2570   \n",
       "2  5631500400   2/25/2015  180000.0         2       1.00          770   \n",
       "3  2487200875   12/9/2014  604000.0         4       3.00         1960   \n",
       "4  1954400510   2/18/2015  510000.0         3       2.00         1680   \n",
       "\n",
       "   sqft_lot  floors waterfront  view  ...          grade sqft_above  \\\n",
       "0      5650     1.0        NaN  NONE  ...      7 Average       1180   \n",
       "1      7242     2.0         NO  NONE  ...      7 Average       2170   \n",
       "2     10000     1.0         NO  NONE  ...  6 Low Average        770   \n",
       "3      5000     1.0         NO  NONE  ...      7 Average       1050   \n",
       "4      8080     1.0         NO  NONE  ...         8 Good       1680   \n",
       "\n",
       "   sqft_basement yr_built  yr_renovated  zipcode      lat     long  \\\n",
       "0            0.0     1955           0.0    98178  47.5112 -122.257   \n",
       "1          400.0     1951        1991.0    98125  47.7210 -122.319   \n",
       "2            0.0     1933           NaN    98028  47.7379 -122.233   \n",
       "3          910.0     1965           0.0    98136  47.5208 -122.393   \n",
       "4            0.0     1987           0.0    98074  47.6168 -122.045   \n",
       "\n",
       "   sqft_living15  sqft_lot15  \n",
       "0           1340        5650  \n",
       "1           1690        7639  \n",
       "2           2720        8062  \n",
       "3           1360        5000  \n",
       "4           1800        7503  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load and preview the dataset\n",
    "\n",
    "data = pd.read_csv('./data/kc_house_data.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 21597 entries, 0 to 21596\n",
      "Data columns (total 21 columns):\n",
      " #   Column         Non-Null Count  Dtype  \n",
      "---  ------         --------------  -----  \n",
      " 0   id             21597 non-null  int64  \n",
      " 1   date           21597 non-null  object \n",
      " 2   price          21597 non-null  float64\n",
      " 3   bedrooms       21597 non-null  int64  \n",
      " 4   bathrooms      21597 non-null  float64\n",
      " 5   sqft_living    21597 non-null  int64  \n",
      " 6   sqft_lot       21597 non-null  int64  \n",
      " 7   floors         21597 non-null  float64\n",
      " 8   waterfront     19221 non-null  object \n",
      " 9   view           21534 non-null  object \n",
      " 10  condition      21597 non-null  object \n",
      " 11  grade          21597 non-null  object \n",
      " 12  sqft_above     21597 non-null  int64  \n",
      " 13  sqft_basement  21597 non-null  object \n",
      " 14  yr_built       21597 non-null  int64  \n",
      " 15  yr_renovated   17755 non-null  float64\n",
      " 16  zipcode        21597 non-null  int64  \n",
      " 17  lat            21597 non-null  float64\n",
      " 18  long           21597 non-null  float64\n",
      " 19  sqft_living15  21597 non-null  int64  \n",
      " 20  sqft_lot15     21597 non-null  int64  \n",
      "dtypes: float64(6), int64(9), object(6)\n",
      "memory usage: 3.5+ MB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exploring null values, object types, and dropping irrelevant variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                  0\n",
       "date                0\n",
       "price               0\n",
       "bedrooms            0\n",
       "bathrooms           0\n",
       "sqft_living         0\n",
       "sqft_lot            0\n",
       "floors              0\n",
       "waterfront       2376\n",
       "view               63\n",
       "condition           0\n",
       "grade               0\n",
       "sqft_above          0\n",
       "sqft_basement       0\n",
       "yr_built            0\n",
       "yr_renovated     3842\n",
       "zipcode             0\n",
       "lat                 0\n",
       "long                0\n",
       "sqft_living15       0\n",
       "sqft_lot15          0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['NONE', nan, 'GOOD', 'EXCELLENT', 'AVERAGE', 'FAIR'], dtype=object)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['view'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date</th>\n",
       "      <th>price</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>sqft_living</th>\n",
       "      <th>sqft_lot</th>\n",
       "      <th>floors</th>\n",
       "      <th>waterfront</th>\n",
       "      <th>view</th>\n",
       "      <th>...</th>\n",
       "      <th>grade</th>\n",
       "      <th>sqft_above</th>\n",
       "      <th>sqft_basement</th>\n",
       "      <th>yr_built</th>\n",
       "      <th>yr_renovated</th>\n",
       "      <th>zipcode</th>\n",
       "      <th>lat</th>\n",
       "      <th>long</th>\n",
       "      <th>sqft_living15</th>\n",
       "      <th>sqft_lot15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1364</th>\n",
       "      <td>9468200140</td>\n",
       "      <td>8/19/2014</td>\n",
       "      <td>450000.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.75</td>\n",
       "      <td>1250</td>\n",
       "      <td>2890</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>7 Average</td>\n",
       "      <td>790</td>\n",
       "      <td>460.0</td>\n",
       "      <td>1920</td>\n",
       "      <td>0.0</td>\n",
       "      <td>98103</td>\n",
       "      <td>47.6795</td>\n",
       "      <td>-122.353</td>\n",
       "      <td>1500</td>\n",
       "      <td>3225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20148</th>\n",
       "      <td>291310170</td>\n",
       "      <td>8/4/2014</td>\n",
       "      <td>384500.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2.50</td>\n",
       "      <td>1600</td>\n",
       "      <td>2610</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>8 Good</td>\n",
       "      <td>1600</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2005</td>\n",
       "      <td>NaN</td>\n",
       "      <td>98027</td>\n",
       "      <td>47.5344</td>\n",
       "      <td>-122.068</td>\n",
       "      <td>1445</td>\n",
       "      <td>1288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17222</th>\n",
       "      <td>316000190</td>\n",
       "      <td>2/15/2015</td>\n",
       "      <td>219000.0</td>\n",
       "      <td>4</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1370</td>\n",
       "      <td>5339</td>\n",
       "      <td>1.5</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>6 Low Average</td>\n",
       "      <td>1370</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1948</td>\n",
       "      <td>0.0</td>\n",
       "      <td>98168</td>\n",
       "      <td>47.5046</td>\n",
       "      <td>-122.300</td>\n",
       "      <td>1280</td>\n",
       "      <td>7048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9853</th>\n",
       "      <td>2719100400</td>\n",
       "      <td>9/12/2014</td>\n",
       "      <td>499950.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1.75</td>\n",
       "      <td>1340</td>\n",
       "      <td>6250</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>7 Average</td>\n",
       "      <td>1090</td>\n",
       "      <td>250.0</td>\n",
       "      <td>1941</td>\n",
       "      <td>0.0</td>\n",
       "      <td>98136</td>\n",
       "      <td>47.5427</td>\n",
       "      <td>-122.386</td>\n",
       "      <td>1550</td>\n",
       "      <td>1755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14975</th>\n",
       "      <td>8562740480</td>\n",
       "      <td>4/14/2015</td>\n",
       "      <td>840000.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3.25</td>\n",
       "      <td>3160</td>\n",
       "      <td>6327</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>9 Better</td>\n",
       "      <td>2280</td>\n",
       "      <td>880.0</td>\n",
       "      <td>2004</td>\n",
       "      <td>0.0</td>\n",
       "      <td>98027</td>\n",
       "      <td>47.5360</td>\n",
       "      <td>-122.066</td>\n",
       "      <td>3160</td>\n",
       "      <td>5946</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               id       date     price  bedrooms  bathrooms  sqft_living  \\\n",
       "1364   9468200140  8/19/2014  450000.0         2       1.75         1250   \n",
       "20148   291310170   8/4/2014  384500.0         3       2.50         1600   \n",
       "17222   316000190  2/15/2015  219000.0         4       1.00         1370   \n",
       "9853   2719100400  9/12/2014  499950.0         3       1.75         1340   \n",
       "14975  8562740480  4/14/2015  840000.0         4       3.25         3160   \n",
       "\n",
       "       sqft_lot  floors waterfront view  ...          grade sqft_above  \\\n",
       "1364       2890     1.0         NO  NaN  ...      7 Average        790   \n",
       "20148      2610     2.0         NO  NaN  ...         8 Good       1600   \n",
       "17222      5339     1.5         NO  NaN  ...  6 Low Average       1370   \n",
       "9853       6250     1.0         NO  NaN  ...      7 Average       1090   \n",
       "14975      6327     2.0         NO  NaN  ...       9 Better       2280   \n",
       "\n",
       "       sqft_basement yr_built  yr_renovated  zipcode      lat     long  \\\n",
       "1364           460.0     1920           0.0    98103  47.6795 -122.353   \n",
       "20148            0.0     2005           NaN    98027  47.5344 -122.068   \n",
       "17222            0.0     1948           0.0    98168  47.5046 -122.300   \n",
       "9853           250.0     1941           0.0    98136  47.5427 -122.386   \n",
       "14975          880.0     2004           0.0    98027  47.5360 -122.066   \n",
       "\n",
       "       sqft_living15  sqft_lot15  \n",
       "1364            1500        3225  \n",
       "20148           1445        1288  \n",
       "17222           1280        7048  \n",
       "9853            1550        1755  \n",
       "14975           3160        5946  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# View a sample of homes with null value for 'view'\n",
    "\n",
    "data[data['view'].isna()].sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, 'NO', 'YES'], dtype=object)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['waterfront'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date</th>\n",
       "      <th>price</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>sqft_living</th>\n",
       "      <th>sqft_lot</th>\n",
       "      <th>floors</th>\n",
       "      <th>waterfront</th>\n",
       "      <th>view</th>\n",
       "      <th>...</th>\n",
       "      <th>grade</th>\n",
       "      <th>sqft_above</th>\n",
       "      <th>sqft_basement</th>\n",
       "      <th>yr_built</th>\n",
       "      <th>yr_renovated</th>\n",
       "      <th>zipcode</th>\n",
       "      <th>lat</th>\n",
       "      <th>long</th>\n",
       "      <th>sqft_living15</th>\n",
       "      <th>sqft_lot15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6068</th>\n",
       "      <td>3558000170</td>\n",
       "      <td>7/11/2014</td>\n",
       "      <td>329950.0</td>\n",
       "      <td>4</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1920</td>\n",
       "      <td>4600</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NONE</td>\n",
       "      <td>...</td>\n",
       "      <td>7 Average</td>\n",
       "      <td>1920</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2002</td>\n",
       "      <td>0.0</td>\n",
       "      <td>98038</td>\n",
       "      <td>47.3795</td>\n",
       "      <td>-122.023</td>\n",
       "      <td>2200</td>\n",
       "      <td>6600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>580</th>\n",
       "      <td>7853240200</td>\n",
       "      <td>5/16/2014</td>\n",
       "      <td>619000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2720</td>\n",
       "      <td>6439</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NONE</td>\n",
       "      <td>...</td>\n",
       "      <td>9 Better</td>\n",
       "      <td>2720</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2005</td>\n",
       "      <td>0.0</td>\n",
       "      <td>98065</td>\n",
       "      <td>47.5422</td>\n",
       "      <td>-121.860</td>\n",
       "      <td>3180</td>\n",
       "      <td>7320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3317</th>\n",
       "      <td>1523089097</td>\n",
       "      <td>3/4/2015</td>\n",
       "      <td>496000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1.5</td>\n",
       "      <td>2520</td>\n",
       "      <td>37616</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NONE</td>\n",
       "      <td>...</td>\n",
       "      <td>8 Good</td>\n",
       "      <td>2520</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1955</td>\n",
       "      <td>0.0</td>\n",
       "      <td>98045</td>\n",
       "      <td>47.4777</td>\n",
       "      <td>-121.763</td>\n",
       "      <td>1470</td>\n",
       "      <td>33750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2339</th>\n",
       "      <td>2771603610</td>\n",
       "      <td>4/23/2015</td>\n",
       "      <td>545000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1550</td>\n",
       "      <td>4000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NONE</td>\n",
       "      <td>...</td>\n",
       "      <td>7 Average</td>\n",
       "      <td>940</td>\n",
       "      <td>610.0</td>\n",
       "      <td>1955</td>\n",
       "      <td>0.0</td>\n",
       "      <td>98199</td>\n",
       "      <td>47.6379</td>\n",
       "      <td>-122.387</td>\n",
       "      <td>1880</td>\n",
       "      <td>4000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13445</th>\n",
       "      <td>9547200530</td>\n",
       "      <td>1/12/2015</td>\n",
       "      <td>780000.0</td>\n",
       "      <td>6</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3300</td>\n",
       "      <td>5720</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NONE</td>\n",
       "      <td>...</td>\n",
       "      <td>8 Good</td>\n",
       "      <td>1960</td>\n",
       "      <td>1340.0</td>\n",
       "      <td>1963</td>\n",
       "      <td>0.0</td>\n",
       "      <td>98115</td>\n",
       "      <td>47.6760</td>\n",
       "      <td>-122.309</td>\n",
       "      <td>2030</td>\n",
       "      <td>4080</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               id       date     price  bedrooms  bathrooms  sqft_living  \\\n",
       "6068   3558000170  7/11/2014  329950.0         4        2.5         1920   \n",
       "580    7853240200  5/16/2014  619000.0         3        2.5         2720   \n",
       "3317   1523089097   3/4/2015  496000.0         3        1.5         2520   \n",
       "2339   2771603610  4/23/2015  545000.0         3        2.0         1550   \n",
       "13445  9547200530  1/12/2015  780000.0         6        4.0         3300   \n",
       "\n",
       "       sqft_lot  floors waterfront  view  ...      grade sqft_above  \\\n",
       "6068       4600     2.0        NaN  NONE  ...  7 Average       1920   \n",
       "580        6439     2.0        NaN  NONE  ...   9 Better       2720   \n",
       "3317      37616     1.0        NaN  NONE  ...     8 Good       2520   \n",
       "2339       4000     1.0        NaN  NONE  ...  7 Average        940   \n",
       "13445      5720     1.0        NaN  NONE  ...     8 Good       1960   \n",
       "\n",
       "       sqft_basement yr_built  yr_renovated  zipcode      lat     long  \\\n",
       "6068             0.0     2002           0.0    98038  47.3795 -122.023   \n",
       "580              0.0     2005           0.0    98065  47.5422 -121.860   \n",
       "3317             0.0     1955           0.0    98045  47.4777 -121.763   \n",
       "2339           610.0     1955           0.0    98199  47.6379 -122.387   \n",
       "13445         1340.0     1963           0.0    98115  47.6760 -122.309   \n",
       "\n",
       "       sqft_living15  sqft_lot15  \n",
       "6068            2200        6600  \n",
       "580             3180        7320  \n",
       "3317            1470       33750  \n",
       "2339            1880        4000  \n",
       "13445           2030        4080  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# View a sample of homes with null value for 'waterfront'\n",
    "\n",
    "data[data['waterfront'].isna()].sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Addressing null values in `waterfront` and  `view`:\n",
    "Using Google Maps, looked at a small sample of homes that have null values in `waterfront` or `view`. It's apparent that these homes are not on a waterfront or do not have a noteable view, respectively. This suggests that the null values can be filled in accordingly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['waterfront'].fillna('NO', inplace=True)\n",
    "data['view'].fillna('NONE', inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Addresing null values in `yr_renovated`:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3842"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(data['yr_renovated'].isna()).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17011"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(data['yr_renovated'] == 0).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This column has 3,842 null values. Additionally, 17,011 records have a `yr_renovated` of 0, which is asssumed to mean the home wasn't renovated. Together, these amount to over 96% of the data.\n",
    "\n",
    "Rather than impute these values with some number, we can make a new column that tells us whether the home was reonavated in the 10 years prior to its sale."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "yr_renovated = data['yr_renovated']\n",
    "data.drop('yr_renovated', inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date</th>\n",
       "      <th>price</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>sqft_living</th>\n",
       "      <th>sqft_lot</th>\n",
       "      <th>floors</th>\n",
       "      <th>waterfront</th>\n",
       "      <th>view</th>\n",
       "      <th>...</th>\n",
       "      <th>grade</th>\n",
       "      <th>sqft_above</th>\n",
       "      <th>sqft_basement</th>\n",
       "      <th>yr_built</th>\n",
       "      <th>zipcode</th>\n",
       "      <th>lat</th>\n",
       "      <th>long</th>\n",
       "      <th>sqft_living15</th>\n",
       "      <th>sqft_lot15</th>\n",
       "      <th>renovated_last_10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7129300520</td>\n",
       "      <td>10/13/2014</td>\n",
       "      <td>221900.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1180</td>\n",
       "      <td>5650</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>...</td>\n",
       "      <td>7 Average</td>\n",
       "      <td>1180</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1955</td>\n",
       "      <td>98178</td>\n",
       "      <td>47.5112</td>\n",
       "      <td>-122.257</td>\n",
       "      <td>1340</td>\n",
       "      <td>5650</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6414100192</td>\n",
       "      <td>12/9/2014</td>\n",
       "      <td>538000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2.25</td>\n",
       "      <td>2570</td>\n",
       "      <td>7242</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>...</td>\n",
       "      <td>7 Average</td>\n",
       "      <td>2170</td>\n",
       "      <td>400.0</td>\n",
       "      <td>1951</td>\n",
       "      <td>98125</td>\n",
       "      <td>47.7210</td>\n",
       "      <td>-122.319</td>\n",
       "      <td>1690</td>\n",
       "      <td>7639</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5631500400</td>\n",
       "      <td>2/25/2015</td>\n",
       "      <td>180000.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.00</td>\n",
       "      <td>770</td>\n",
       "      <td>10000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>...</td>\n",
       "      <td>6 Low Average</td>\n",
       "      <td>770</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1933</td>\n",
       "      <td>98028</td>\n",
       "      <td>47.7379</td>\n",
       "      <td>-122.233</td>\n",
       "      <td>2720</td>\n",
       "      <td>8062</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id        date     price  bedrooms  bathrooms  sqft_living  \\\n",
       "0  7129300520  10/13/2014  221900.0         3       1.00         1180   \n",
       "1  6414100192   12/9/2014  538000.0         3       2.25         2570   \n",
       "2  5631500400   2/25/2015  180000.0         2       1.00          770   \n",
       "\n",
       "   sqft_lot  floors waterfront  view  ...          grade sqft_above  \\\n",
       "0      5650     1.0         NO  NONE  ...      7 Average       1180   \n",
       "1      7242     2.0         NO  NONE  ...      7 Average       2170   \n",
       "2     10000     1.0         NO  NONE  ...  6 Low Average        770   \n",
       "\n",
       "   sqft_basement yr_built  zipcode      lat     long  sqft_living15  \\\n",
       "0            0.0     1955    98178  47.5112 -122.257           1340   \n",
       "1          400.0     1951    98125  47.7210 -122.319           1690   \n",
       "2            0.0     1933    98028  47.7379 -122.233           2720   \n",
       "\n",
       "   sqft_lot15  renovated_last_10  \n",
       "0        5650                 No  \n",
       "1        7639                 No  \n",
       "2        8062                 No  \n",
       "\n",
       "[3 rows x 21 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yr_sold = data['date'].str[-4:].astype(int)\n",
    "data['renovated_last_10'] = (yr_renovated >= (yr_sold - 10))\n",
    "data['renovated_last_10'] = data['renovated_last_10'].map({True: 'Yes', False: 'No'})\n",
    "# Keeping this as a string for now in order to group it with categorical variables later.\n",
    "data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 21597 entries, 0 to 21596\n",
      "Data columns (total 21 columns):\n",
      " #   Column             Non-Null Count  Dtype  \n",
      "---  ------             --------------  -----  \n",
      " 0   id                 21597 non-null  int64  \n",
      " 1   date               21597 non-null  object \n",
      " 2   price              21597 non-null  float64\n",
      " 3   bedrooms           21597 non-null  int64  \n",
      " 4   bathrooms          21597 non-null  float64\n",
      " 5   sqft_living        21597 non-null  int64  \n",
      " 6   sqft_lot           21597 non-null  int64  \n",
      " 7   floors             21597 non-null  float64\n",
      " 8   waterfront         21597 non-null  object \n",
      " 9   view               21597 non-null  object \n",
      " 10  condition          21597 non-null  object \n",
      " 11  grade              21597 non-null  object \n",
      " 12  sqft_above         21597 non-null  int64  \n",
      " 13  sqft_basement      21597 non-null  object \n",
      " 14  yr_built           21597 non-null  int64  \n",
      " 15  zipcode            21597 non-null  int64  \n",
      " 16  lat                21597 non-null  float64\n",
      " 17  long               21597 non-null  float64\n",
      " 18  sqft_living15      21597 non-null  int64  \n",
      " 19  sqft_lot15         21597 non-null  int64  \n",
      " 20  renovated_last_10  21597 non-null  object \n",
      "dtypes: float64(5), int64(9), object(7)\n",
      "memory usage: 3.5+ MB\n"
     ]
    }
   ],
   "source": [
    "# Making sure there are no more null values\n",
    "# and checking again on the data types\n",
    "\n",
    "data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Why is `sqft_basement` an object type?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['0.0', '400.0', '910.0', '1530.0', '?', '730.0', '1700.0', '300.0',\n",
       "       '970.0', '760.0', '720.0', '700.0', '820.0', '780.0', '790.0',\n",
       "       '330.0', '1620.0', '360.0', '588.0', '1510.0', '410.0', '990.0',\n",
       "       '600.0', '560.0', '550.0', '1000.0', '1600.0', '500.0', '1040.0',\n",
       "       '880.0', '1010.0', '240.0', '265.0', '290.0', '800.0', '540.0',\n",
       "       '710.0', '840.0', '380.0', '770.0', '480.0', '570.0', '1490.0',\n",
       "       '620.0', '1250.0', '1270.0', '120.0', '650.0', '180.0', '1130.0',\n",
       "       '450.0', '1640.0', '1460.0', '1020.0', '1030.0', '750.0', '640.0',\n",
       "       '1070.0', '490.0', '1310.0', '630.0', '2000.0', '390.0', '430.0',\n",
       "       '850.0', '210.0', '1430.0', '1950.0', '440.0', '220.0', '1160.0',\n",
       "       '860.0', '580.0', '2060.0', '1820.0', '1180.0', '200.0', '1150.0',\n",
       "       '1200.0', '680.0', '530.0', '1450.0', '1170.0', '1080.0', '960.0',\n",
       "       '280.0', '870.0', '1100.0', '460.0', '1400.0', '660.0', '1220.0',\n",
       "       '900.0', '420.0', '1580.0', '1380.0', '475.0', '690.0', '270.0',\n",
       "       '350.0', '935.0', '1370.0', '980.0', '1470.0', '160.0', '950.0',\n",
       "       '50.0', '740.0', '1780.0', '1900.0', '340.0', '470.0', '370.0',\n",
       "       '140.0', '1760.0', '130.0', '520.0', '890.0', '1110.0', '150.0',\n",
       "       '1720.0', '810.0', '190.0', '1290.0', '670.0', '1800.0', '1120.0',\n",
       "       '1810.0', '60.0', '1050.0', '940.0', '310.0', '930.0', '1390.0',\n",
       "       '610.0', '1830.0', '1300.0', '510.0', '1330.0', '1590.0', '920.0',\n",
       "       '1320.0', '1420.0', '1240.0', '1960.0', '1560.0', '2020.0',\n",
       "       '1190.0', '2110.0', '1280.0', '250.0', '2390.0', '1230.0', '170.0',\n",
       "       '830.0', '1260.0', '1410.0', '1340.0', '590.0', '1500.0', '1140.0',\n",
       "       '260.0', '100.0', '320.0', '1480.0', '1060.0', '1284.0', '1670.0',\n",
       "       '1350.0', '2570.0', '1090.0', '110.0', '2500.0', '90.0', '1940.0',\n",
       "       '1550.0', '2350.0', '2490.0', '1481.0', '1360.0', '1135.0',\n",
       "       '1520.0', '1850.0', '1660.0', '2130.0', '2600.0', '1690.0',\n",
       "       '243.0', '1210.0', '1024.0', '1798.0', '1610.0', '1440.0',\n",
       "       '1570.0', '1650.0', '704.0', '1910.0', '1630.0', '2360.0',\n",
       "       '1852.0', '2090.0', '2400.0', '1790.0', '2150.0', '230.0', '70.0',\n",
       "       '1680.0', '2100.0', '3000.0', '1870.0', '1710.0', '2030.0',\n",
       "       '875.0', '1540.0', '2850.0', '2170.0', '506.0', '906.0', '145.0',\n",
       "       '2040.0', '784.0', '1750.0', '374.0', '518.0', '2720.0', '2730.0',\n",
       "       '1840.0', '3480.0', '2160.0', '1920.0', '2330.0', '1860.0',\n",
       "       '2050.0', '4820.0', '1913.0', '80.0', '2010.0', '3260.0', '2200.0',\n",
       "       '415.0', '1730.0', '652.0', '2196.0', '1930.0', '515.0', '40.0',\n",
       "       '2080.0', '2580.0', '1548.0', '1740.0', '235.0', '861.0', '1890.0',\n",
       "       '2220.0', '792.0', '2070.0', '4130.0', '2250.0', '2240.0',\n",
       "       '1990.0', '768.0', '2550.0', '435.0', '1008.0', '2300.0', '2610.0',\n",
       "       '666.0', '3500.0', '172.0', '1816.0', '2190.0', '1245.0', '1525.0',\n",
       "       '1880.0', '862.0', '946.0', '1281.0', '414.0', '2180.0', '276.0',\n",
       "       '1248.0', '602.0', '516.0', '176.0', '225.0', '1275.0', '266.0',\n",
       "       '283.0', '65.0', '2310.0', '10.0', '1770.0', '2120.0', '295.0',\n",
       "       '207.0', '915.0', '556.0', '417.0', '143.0', '508.0', '2810.0',\n",
       "       '20.0', '274.0', '248.0'], dtype=object)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['sqft_basement'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0       0.593879\n",
       "?         0.021021\n",
       "600.0     0.010048\n",
       "500.0     0.009677\n",
       "700.0     0.009631\n",
       "            ...   \n",
       "1920.0    0.000046\n",
       "3480.0    0.000046\n",
       "2730.0    0.000046\n",
       "2720.0    0.000046\n",
       "248.0     0.000046\n",
       "Name: sqft_basement, Length: 304, dtype: float64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['sqft_basement'].value_counts(normalize=True, ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The values in `sqft_basement` are strings. I have to deal with values of '0.0' and '?' It is unclear whether these values mean\n",
    "* there is no basement\n",
    "* there is a basement but it's unfinished\n",
    "* something else\n",
    "\n",
    "Rather than make any unsafe assumptions, I will drop this column, as well as `sqft_above`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.drop(['sqft_basement', 'sqft_above'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date</th>\n",
       "      <th>price</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>sqft_living</th>\n",
       "      <th>sqft_lot</th>\n",
       "      <th>floors</th>\n",
       "      <th>waterfront</th>\n",
       "      <th>view</th>\n",
       "      <th>condition</th>\n",
       "      <th>grade</th>\n",
       "      <th>yr_built</th>\n",
       "      <th>zipcode</th>\n",
       "      <th>lat</th>\n",
       "      <th>long</th>\n",
       "      <th>sqft_living15</th>\n",
       "      <th>sqft_lot15</th>\n",
       "      <th>renovated_last_10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7129300520</td>\n",
       "      <td>10/13/2014</td>\n",
       "      <td>221900.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1180</td>\n",
       "      <td>5650</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>Average</td>\n",
       "      <td>7 Average</td>\n",
       "      <td>1955</td>\n",
       "      <td>98178</td>\n",
       "      <td>47.5112</td>\n",
       "      <td>-122.257</td>\n",
       "      <td>1340</td>\n",
       "      <td>5650</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6414100192</td>\n",
       "      <td>12/9/2014</td>\n",
       "      <td>538000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2.25</td>\n",
       "      <td>2570</td>\n",
       "      <td>7242</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>Average</td>\n",
       "      <td>7 Average</td>\n",
       "      <td>1951</td>\n",
       "      <td>98125</td>\n",
       "      <td>47.7210</td>\n",
       "      <td>-122.319</td>\n",
       "      <td>1690</td>\n",
       "      <td>7639</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5631500400</td>\n",
       "      <td>2/25/2015</td>\n",
       "      <td>180000.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.00</td>\n",
       "      <td>770</td>\n",
       "      <td>10000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>Average</td>\n",
       "      <td>6 Low Average</td>\n",
       "      <td>1933</td>\n",
       "      <td>98028</td>\n",
       "      <td>47.7379</td>\n",
       "      <td>-122.233</td>\n",
       "      <td>2720</td>\n",
       "      <td>8062</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id        date     price  bedrooms  bathrooms  sqft_living  \\\n",
       "0  7129300520  10/13/2014  221900.0         3       1.00         1180   \n",
       "1  6414100192   12/9/2014  538000.0         3       2.25         2570   \n",
       "2  5631500400   2/25/2015  180000.0         2       1.00          770   \n",
       "\n",
       "   sqft_lot  floors waterfront  view condition          grade  yr_built  \\\n",
       "0      5650     1.0         NO  NONE   Average      7 Average      1955   \n",
       "1      7242     2.0         NO  NONE   Average      7 Average      1951   \n",
       "2     10000     1.0         NO  NONE   Average  6 Low Average      1933   \n",
       "\n",
       "   zipcode      lat     long  sqft_living15  sqft_lot15 renovated_last_10  \n",
       "0    98178  47.5112 -122.257           1340        5650                No  \n",
       "1    98125  47.7210 -122.319           1690        7639                No  \n",
       "2    98028  47.7379 -122.233           2720        8062                No  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Converting `yr_built` to `age`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "yr_built = data['yr_built']\n",
    "data['age'] = yr_sold - yr_built\n",
    "data.drop('yr_built', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bulk dropping columns:\n",
    "\n",
    "The following variables either can't be changed by someone selling their home or don't provide much useful information. As for `date`, all homes in this dataset were sold between May 2014 and May 2015.\n",
    "\n",
    "* `id`\n",
    "* `date`\n",
    "* `zipcode`\n",
    "* `lat`\n",
    "* `long`\n",
    "* `sqft_living15`\n",
    "* `sqft_lot15`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.drop(\n",
    "    ['id', \n",
    "    'date',\n",
    "    'zipcode',\n",
    "    'lat',\n",
    "    'long',\n",
    "    'sqft_living15',\n",
    "    'sqft_lot15'],\n",
    "    \n",
    "    axis=1,\n",
    "    inplace=True\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dealing with categorical variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>price</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>sqft_living</th>\n",
       "      <th>sqft_lot</th>\n",
       "      <th>floors</th>\n",
       "      <th>waterfront</th>\n",
       "      <th>view</th>\n",
       "      <th>condition</th>\n",
       "      <th>grade</th>\n",
       "      <th>renovated_last_10</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11868</th>\n",
       "      <td>367999.0</td>\n",
       "      <td>4</td>\n",
       "      <td>2.75</td>\n",
       "      <td>3430</td>\n",
       "      <td>8000</td>\n",
       "      <td>1.5</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>Good</td>\n",
       "      <td>8 Good</td>\n",
       "      <td>No</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21057</th>\n",
       "      <td>636230.0</td>\n",
       "      <td>4</td>\n",
       "      <td>2.50</td>\n",
       "      <td>2840</td>\n",
       "      <td>6284</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>Average</td>\n",
       "      <td>9 Better</td>\n",
       "      <td>No</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6270</th>\n",
       "      <td>353500.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2.50</td>\n",
       "      <td>2020</td>\n",
       "      <td>4845</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>Average</td>\n",
       "      <td>7 Average</td>\n",
       "      <td>No</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16846</th>\n",
       "      <td>585000.0</td>\n",
       "      <td>4</td>\n",
       "      <td>2.50</td>\n",
       "      <td>2250</td>\n",
       "      <td>4119</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>Average</td>\n",
       "      <td>8 Good</td>\n",
       "      <td>No</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18870</th>\n",
       "      <td>380000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1.50</td>\n",
       "      <td>1540</td>\n",
       "      <td>8400</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>Very Good</td>\n",
       "      <td>7 Average</td>\n",
       "      <td>No</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267</th>\n",
       "      <td>415000.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1070</td>\n",
       "      <td>4500</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>Average</td>\n",
       "      <td>7 Average</td>\n",
       "      <td>No</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>804</th>\n",
       "      <td>385200.0</td>\n",
       "      <td>4</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1550</td>\n",
       "      <td>7740</td>\n",
       "      <td>1.5</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>Average</td>\n",
       "      <td>6 Low Average</td>\n",
       "      <td>No</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19433</th>\n",
       "      <td>450000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1350</td>\n",
       "      <td>92721</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>Fair</td>\n",
       "      <td>6 Low Average</td>\n",
       "      <td>No</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5484</th>\n",
       "      <td>751305.0</td>\n",
       "      <td>4</td>\n",
       "      <td>2.50</td>\n",
       "      <td>2660</td>\n",
       "      <td>8469</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>Average</td>\n",
       "      <td>10 Very Good</td>\n",
       "      <td>No</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6103</th>\n",
       "      <td>1240000.0</td>\n",
       "      <td>5</td>\n",
       "      <td>3.00</td>\n",
       "      <td>2830</td>\n",
       "      <td>7500</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NO</td>\n",
       "      <td>NONE</td>\n",
       "      <td>Average</td>\n",
       "      <td>9 Better</td>\n",
       "      <td>No</td>\n",
       "      <td>91</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           price  bedrooms  bathrooms  sqft_living  sqft_lot  floors  \\\n",
       "11868   367999.0         4       2.75         3430      8000     1.5   \n",
       "21057   636230.0         4       2.50         2840      6284     2.0   \n",
       "6270    353500.0         3       2.50         2020      4845     2.0   \n",
       "16846   585000.0         4       2.50         2250      4119     2.0   \n",
       "18870   380000.0         3       1.50         1540      8400     1.0   \n",
       "267     415000.0         2       1.00         1070      4500     1.0   \n",
       "804     385200.0         4       1.00         1550      7740     1.5   \n",
       "19433   450000.0         3       1.00         1350     92721     1.0   \n",
       "5484    751305.0         4       2.50         2660      8469     2.0   \n",
       "6103   1240000.0         5       3.00         2830      7500     2.0   \n",
       "\n",
       "      waterfront  view  condition          grade renovated_last_10  age  \n",
       "11868         NO  NONE       Good         8 Good                No   43  \n",
       "21057         NO  NONE    Average       9 Better                No    1  \n",
       "6270          NO  NONE    Average      7 Average                No   14  \n",
       "16846         NO  NONE    Average         8 Good                No   14  \n",
       "18870         NO  NONE  Very Good      7 Average                No   46  \n",
       "267           NO  NONE    Average      7 Average                No   77  \n",
       "804           NO  NONE    Average  6 Low Average                No   60  \n",
       "19433         NO  NONE       Fair  6 Low Average                No   69  \n",
       "5484          NO  NONE    Average   10 Very Good                No   20  \n",
       "6103          NO  NONE    Average       9 Better                No   91  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I plan to treat the following as categorical variables:\n",
    "* `bedrooms`\n",
    "* `bathrooms`\n",
    "* `floors`\n",
    "* `waterfront`\n",
    "* `view`\n",
    "* `condition`\n",
    "* `grade`\n",
    "* `renovated_last_10`\n",
    "\n",
    "I'll have to clean up some of the values for consistency.\n",
    "\n",
    "With some [research online](https://www.badeloftusa.com/buying-guides/75-bathroom/), I found out that a .75 bathroom means it has a sink, toilet, and shower, but not a full tub. That explains why some houses have bathroom counts ending in .75 and .25. \n",
    "\n",
    "**Removing noise**: If a category appears in the dataset fewer than 40 times, I am dropping that category from the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat = ['bedrooms', 'bathrooms', 'floors', 'waterfront', 'view', 'condition', 'grade', 'renovated_last_10']\n",
    "\n",
    "for c in cat:\n",
    "    # get value counts\n",
    "    counts = data[c].value_counts()\n",
    "    # Isolate offending categories for each variable\n",
    "    bad_cats = counts[counts < 40].index\n",
    "    # Isolate indices in the dataset where offending categories are found\n",
    "    to_drop = data[data[c].isin(bad_cats)].index\n",
    "    # Drop\n",
    "    data.drop(to_drop, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### View"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD5CAYAAAAndkJ4AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAlhklEQVR4nO3dfVhU553/8ffMQCvIg/MgEgxJi0ITXRNQTAQbYXU2bYJVk2atSaMNkaibrFm0q03spnkwiWwIEg2wdtnsrlq3G3dX1D5cTZcQZAuxYCxXYmxidTSNF0SEGZVU6KDO7w/H8wsBxBlAwHxef3nuc+459/mKfM65z3GOyefz+RARkS8882APQEREhgYFgoiIAAoEERHxUyCIiAigQBARET8FgoiIABDS2wbNzc0UFxdz6tQpTCYTTqeTu+++m08//ZTCwkJOnjzJ6NGjWbFiBREREQCUlZVRUVGB2WwmOzub5ORkAFwuF8XFxXi9XlJSUsjOzsZkMtHR0UFRUREul4vIyEhyc3OJiYkZ0AMXEZHOTL39PwSPx4PH4yEhIYG2tjaeeOIJVq1aRWVlJREREcybN4+dO3fy6aef8uCDD3L8+HE2bNjAiy++iMfjYe3atWzYsAGz2cyTTz5JdnY2iYmJrFu3jrvuuouUlBTeeOMNPvroI5YsWUJ1dTW1tbWsWLGi18E3NDQEddAOh4Pm5uag+n4RqV6BUb0Cp5oFpi/1iouL63Fdr1NGVquVhIQEAMLCwhg7dixut5u6ujoyMjIAyMjIoK6uDoC6ujrS09MJDQ0lJiaG2NhYDh8+jMfjoa2tjaSkJEwmEzNmzDD67Nu3j8zMTACmTZvGgQMH0P+XExG5unqdMvqspqYmjh49yvjx4zl9+jRWqxW4GBpnzpwBwO12k5iYaPSx2Wy43W4sFgt2u91ot9vtuN1uo8+ldRaLhfDwcFpbW4mKiuq0//LycsrLywHIy8vD4XAEerwAhISEBN33i0j1CozqFTjVLDADVa8rDoT29nYKCgp46KGHCA8P73G7ns7sL3fG3906k8nUpc3pdOJ0Oo3lYC+ZdHkaGNUrMKpX4FSzwAzalBHAuXPnKCgo4I477uD2228HIDo6Go/HA1y8z3DpbN5ut9PS0mL0dbvd2Gy2Lu0tLS3YbLYufc6fP8/Zs2eNG9QiInJ19BoIPp+PTZs2MXbsWGbPnm20p6amsmfPHgD27NnD1KlTjfaamho6OjpoamqisbGR8ePHY7VaCQsL49ChQ/h8PqqqqkhNTQVgypQpVFZWArB3714mTpzY7RWCiIgMnF6fMvrggw/40Y9+xA033GD8kr7//vtJTEyksLCQ5uZmHA4HK1euNM7qd+zYwVtvvYXZbOahhx4iJSUFgCNHjlBSUoLX6yU5OZmHH34Yk8mE1+ulqKiIo0ePEhERQW5uLmPGjOl18HrK6OpQvQKjegVONQvMQE0Z9RoIQ5kC4epQvQKjegVONQvMoN5DEBGRa58CQUREgAD/H8K14sQ96YO2b0vp7kHbt4jI5egKQUREAAWCiIj4KRBERARQIIiIiJ8CQUREAAWCiIj4KRBERARQIIiIiJ8CQUREAAWCiIj4KRBERARQIIiIiJ8CQUREAAWCiIj4KRBERAS4gvchlJSUsH//fqKjoykoKACgsLDQeH3l2bNnCQ8PJz8/n6amJlasWGG8oi0xMZElS5YA4HK5KC4uxuv1kpKSQnZ2NiaTiY6ODoqKinC5XERGRpKbm0tMTMxAHa+IiPSg10DIzMzkm9/8JsXFxUbbihUrjD9v2bKF8PBwYzk2Npb8/Pwun1NaWsrSpUtJTExk3bp11NfXk5KSQkVFBSNHjuTVV1+lurqabdu2dfp8ERG5OnqdMpowYQIRERHdrvP5fLz99ttMnz79sp/h8Xhoa2sjKSkJk8nEjBkzqKurA2Dfvn1kZmYCMG3aNA4cOIDP5wvwMEREpK/69ArN3//+90RHR3PdddcZbU1NTaxevZqwsDAWLFjAzTffjNvtxm63G9vY7XbcbjdAp3UWi4Xw8HBaW1uJiorqsr/y8nLKy8sByMvLw+FwBDXuE0H16h/BjnkwhYSEDMtxDxbVK3CqWWAGql59CoTq6upOVwdWq5WSkhIiIyNxuVzk5+dTUFBw2TP+7taZTKZut3U6nTidTmO5ubm5D6MfHMNxzA6HY1iOe7CoXoFTzQLTl3pdusfbnaCfMjp//jy1tbWkp///F9aHhoYSGRkJQEJCAmPGjKGxsRG73U5LS4uxXUtLCzabDaDTuvPnz3P27Nkep6hERGTgBB0I7733HnFxcZ2mgs6cOcOFCxcAOHHiBI2NjYwZMwar1UpYWBiHDh3C5/NRVVVFamoqAFOmTKGyshKAvXv3MnHixB6vEEREZOD0OmX0yiuvcPDgQVpbW1m2bBnz589n5syZXaaLAA4ePMj27duxWCyYzWYeeeQR42w/JyeHkpISvF4vycnJpKSkADBz5kyKiopYvnw5ERER5Obm9v9RiohIr0y+YfxIz6X/CxGo84/M6eeRXDlL6e5B23ewNL8bGNUrcKpZYIbcPQQREbm2KBBERARQIIiIiJ8CQUREAAWCiIj4KRBERARQIIiIiJ8CQUREAAWCiIj4KRBERARQIIiIiJ8CQUREAAWCiIj4KRBERARQIIiIiJ8CQUREAAWCiIj4KRBERAS4gncql5SUsH//fqKjoykoKABg+/btvPnmm0RFRQFw//33M3nyZADKysqoqKjAbDaTnZ1NcnIyAC6Xi+LiYrxeLykpKWRnZ2Mymejo6KCoqAiXy0VkZCS5ubnExMQM0OGKiEhPer1CyMzMZM2aNV3as7KyyM/PJz8/3wiD48ePU1NTw/r16/nhD3/Ia6+9xoULFwAoLS1l6dKlbNy4kU8++YT6+noAKioqGDlyJK+++ipZWVls27atHw9PRESuVK+BMGHCBCIiIq7ow+rq6khPTyc0NJSYmBhiY2M5fPgwHo+HtrY2kpKSMJlMzJgxg7q6OgD27dtHZmYmANOmTePAgQP4fL7gj0hERILS65RRT9544w2qqqpISEhg0aJFRERE4Ha7SUxMNLax2Wy43W4sFgt2u91ot9vtuN1uANxut7HOYrEQHh5Oa2urMR31WeXl5ZSXlwOQl5eHw+EIauwngurVP4Id82AKCQkZluMeLKpX4FSzwAxUvYIKhDvvvJP77rsPgNdff50tW7bw6KOP9nhmf7kz/u7WmUymbrd1Op04nU5jubm5OZBhDwnDccwOh2NYjnuwqF6BU80C05d6xcXF9bguqKeMRo0ahdlsxmw2M2vWLI4cOQJcPPNvaWkxtnO73dhsti7tLS0t2Gy2Ln3Onz/P2bNnr3iKSkRE+k9QgeDxeIw/19bWEh8fD0Bqaio1NTV0dHTQ1NREY2Mj48ePx2q1EhYWxqFDh/D5fFRVVZGamgrAlClTqKysBGDv3r1MnDixxysEEREZOL1OGb3yyiscPHiQ1tZWli1bxvz583n//fc5duwYJpOJ0aNHs2TJEgDi4+NJS0tj5cqVmM1mFi9ejNl8MXNycnIoKSnB6/WSnJxMSkoKADNnzqSoqIjly5cTERFBbm7uwB2tiIj0yOQbxo/0NDQ0BNXv/CNz+nkkV85SunvQ9h0sze8GRvUKnGoWmCF1D0FERK49CgQREQEUCCIi4qdAEBERQIEgIiJ+CgQREQEUCCIi4qdAEBERQIEgIiJ+CgQREQEUCCIi4qdAEBERQIEgIiJ+CgQREQEUCCIi4qdAEBERQIEgIiJ+CgQREQGu4J3KJSUl7N+/n+joaAoKCgDYunUr77zzDiEhIYwZM4ZHH32UkSNH0tTUxIoVK4xXtCUmJhrvW3a5XBQXF+P1eklJSSE7OxuTyURHRwdFRUW4XC4iIyPJzc0lJiZmAA9ZRES60+sVQmZmJmvWrOnUdsstt1BQUMDLL7/MddddR1lZmbEuNjaW/Px88vPzjTAAKC0tZenSpWzcuJFPPvmE+vp6ACoqKhg5ciSvvvoqWVlZbNu2rZ8OTUREAtFrIEyYMIGIiIhObbfeeisWiwWApKQk3G73ZT/D4/HQ1tZGUlISJpOJGTNmUFdXB8C+ffvIzMwEYNq0aRw4cACfzxfMsYiISB/0OmXUm4qKCtLT043lpqYmVq9eTVhYGAsWLODmm2/G7XZjt9uNbex2uxEin11nsVgIDw+ntbWVqKioLvsqLy+nvLwcgLy8PBwOR1BjPhFUr/4R7JgHU0hIyLAc92BRvQKnmgVmoOrVp0DYsWMHFouFO+64AwCr1UpJSQmRkZG4XC7y8/MpKCi47Bl/d+tMJlO32zqdTpxOp7Hc3Nzcl+EPiuE4ZofDMSzHPVhUr8CpZoHpS70u3ePtTtBPGVVWVvLOO+/w+OOPG7/AQ0NDiYyMBCAhIYExY8bQ2NiI3W6npaXF6NvS0oLNZgPotO78+fOcPXu2yxSViIgMvKACob6+nl27dvGDH/yAL3/5y0b7mTNnuHDhAgAnTpygsbGRMWPGYLVaCQsL49ChQ/h8PqqqqkhNTQVgypQpVFZWArB3714mTpzY4xWCiIgMnF6njF555RUOHjxIa2sry5YtY/78+ZSVlXHu3DnWrl0L/P/HSw8ePMj27duxWCyYzWYeeeQR42w/JyeHkpISvF4vycnJpKSkADBz5kyKiopYvnw5ERER5ObmDtzRiohIj0y+YfxIT0NDQ1D9zj8yp59HcuUspbsHbd/B0vxuYFSvwKlmgRly9xBEROTaokAQERFAgSAiIn4KBBERARQIIiLip0AQERFAgSAiIn4KBBERARQIIiLip0AQERFAgSAiIn4KBBERARQIIiLip0AQERFAgSAiIn4KBBERARQIIiLi1+srNEtKSti/fz/R0dEUFBQA8Omnn1JYWMjJkycZPXo0K1asMF6VWVZWRkVFBWazmezsbJKTkwFwuVwUFxfj9XpJSUkhOzsbk8lER0cHRUVFuFwuIiMjyc3NJSYmZuCOWEREutXrFUJmZiZr1qzp1LZz504mTZrExo0bmTRpEjt37gTg+PHj1NTUsH79en74wx/y2muvceHCBQBKS0tZunQpGzdu5JNPPqG+vh6AiooKRo4cyauvvkpWVhbbtm3r3yMUEZEr0msgTJgwwTj7v6Suro6MjAwAMjIyqKurM9rT09MJDQ0lJiaG2NhYDh8+jMfjoa2tjaSkJEwmEzNmzDD67Nu3j8zMTACmTZvGgQMHGMaveRYRGbaCuodw+vRprFYrAFarlTNnzgDgdrux2+3GdjabDbfb3aXdbrfjdru79LFYLISHh9Pa2hrc0YiISNB6vYcQiJ7O7C93xt/dOpPJ1O225eXllJeXA5CXl4fD4QhilHAiqF79I9gxD6aQkJBhOe7BonoFTjULzEDVK6hAiI6OxuPxYLVa8Xg8REVFARfP/FtaWozt3G43NputS3tLSws2m61TH7vdzvnz5zl79myXKapLnE4nTqfTWG5ubg5m+INqOI7Z4XAMy3EPFtUrcKpZYPpSr7i4uB7XBTVllJqayp49ewDYs2cPU6dONdpramro6OigqamJxsZGxo8fj9VqJSwsjEOHDuHz+aiqqiI1NRWAKVOmUFlZCcDevXuZOHFij1cIIiIycHq9QnjllVc4ePAgra2tLFu2jPnz5zNv3jwKCwupqKjA4XCwcuVKAOLj40lLS2PlypWYzWYWL16M2Xwxc3JycigpKcHr9ZKcnExKSgoAM2fOpKioiOXLlxMREUFubu7AHa2IiPTI5BvGj/Q0NDQE1e/8I3P6eSRXzlK6e9D2HSxdzgdG9QqcahaYITVlJCIi1x4FgoiIAAoEERHxUyCIiAigQBARET8FgoiIAAoEERHxUyCIiAigQBARET8FgoiIAAoEERHxUyCIiAigQBARET8FgoiIAAoEERHxUyCIiAigQBARET8FgoiIAFfwTuWeNDQ0UFhYaCw3NTUxf/58/vSnP/Hmm28SFRUFwP3338/kyZMBKCsro6KiArPZTHZ2NsnJyQC4XC6Ki4vxer2kpKSQnZ2NyWTqw2GJiEiggg6EuLg48vPzAbhw4QJLly7ltttu46233iIrK4s5czq/t/j48ePU1NSwfv16PB4Pa9euZcOGDZjNZkpLS1m6dCmJiYmsW7eO+vp6UlJS+nZkIiISkH6ZMnrvvfeIjY1l9OjRPW5TV1dHeno6oaGhxMTEEBsby+HDh/F4PLS1tZGUlITJZGLGjBnU1dX1x7BERCQAQV8hfFZ1dTXTp083lt944w2qqqpISEhg0aJFRERE4Ha7SUxMNLax2Wy43W4sFgt2u91ot9vtuN3ubvdTXl5OeXk5AHl5eTgcjqDGeyKoXv0j2DEPppCQkGE57sGiegVONQvMQNWrz4Fw7tw53nnnHR544AEA7rzzTu677z4AXn/9dbZs2cKjjz6Kz+frtn9P7d1xOp04nU5jubm5uQ8jHxzDccwOh2NYjnuwqF6BU80C05d6xcXF9biuz1NGv/vd7/jqV7/KqFGjABg1ahRmsxmz2cysWbM4cuQIcPHMv6Wlxejndrux2Wxd2ltaWrDZbH0dloiIBKjPgfD56SKPx2P8uba2lvj4eABSU1Opqamho6ODpqYmGhsbGT9+PFarlbCwMA4dOoTP56OqqorU1NS+DktERALUpymjP//5z7z77rssWbLEaPvJT37CsWPHMJlMjB492lgXHx9PWloaK1euxGw2s3jxYszmi3mUk5NDSUkJXq+X5ORkPWEkIjIITL5AJvGHmIaGhqD6nX9kTu8bDRBL6e5B23ewNL8bGNUrcKpZYIbsPQQREbk2KBBERARQIIiIiJ8CQUREAAWCiIj4KRBERARQIIiIiJ8CQUREAAWCiIj4KRBERARQIIiIiJ8CQUREAAWCiIj4KRBERARQIIiIiJ8CQUREAAWCiIj4KRBERATo4zuVH3vsMUaMGIHZbMZisZCXl8enn35KYWEhJ0+eZPTo0axYsYKIiAgAysrKqKiowGw2k52dTXJyMgAul4vi4mK8Xi8pKSlkZ2djMpn6fHAiInLl+hQIAE8//TRRUVHG8s6dO5k0aRLz5s1j586d7Ny5kwcffJDjx49TU1PD+vXr8Xg8rF27lg0bNmA2myktLWXp0qUkJiaybt066uvrSUlJ6evQREQkAP0+ZVRXV0dGRgYAGRkZ1NXVGe3p6emEhoYSExNDbGwshw8fxuPx0NbWRlJSEiaTiRkzZhh9RETk6unzFcILL7wAwF/91V/hdDo5ffo0VqsVAKvVypkzZwBwu90kJiYa/Ww2G263G4vFgt1uN9rtdjtut7vbfZWXl1NeXg5AXl4eDocjqDGfCKpX/wh2zIMpJCRkWI57sKhegVPNAjNQ9epTIKxduxabzcbp06d5/vnniYuL63Fbn88XUHt3nE4nTqfTWG5ubr7ywQ4Rw3HMDodjWI57sKhegVPNAtOXel3u93SfpoxsNhsA0dHRTJ06lcOHDxMdHY3H4wHA4/EY9xfsdjstLS1GX7fbjc1m69Le0tJifK6IiFw9QQdCe3s7bW1txp/fffddbrjhBlJTU9mzZw8Ae/bsYerUqQCkpqZSU1NDR0cHTU1NNDY2Mn78eKxWK2FhYRw6dAifz0dVVRWpqan9cGgiIhKIoKeMTp8+zcsvvwzA+fPn+frXv05ycjLjxo2jsLCQiooKHA4HK1euBCA+Pp60tDRWrlyJ2Wxm8eLFmM0X8ygnJ4eSkhK8Xi/Jycl6wkhEZBCYfIFM4g8xDQ0NQfU7/8icfh7JlbOU7h60fQdL87uBUb0Cp5oFZkjeQxARkWuHAkFERAAFgoiI+CkQREQEUCCIiIifAkFERAAFgoiI+CkQREQEUCCIiIifAkFERAAFgoiI+CkQREQEUCCIiIifAkFERAAFgoiI+CkQREQEUCCIiIifAkFERIA+vFO5ubmZ4uJiTp06hclkwul0cvfdd7N9+3befPNNoqKiALj//vuZPHkyAGVlZVRUVGA2m8nOziY5ORkAl8tFcXExXq+XlJQUsrOzMZlMfT86ERG5YkEHgsViYeHChSQkJNDW1sYTTzzBLbfcAkBWVhZz5nR+b/Hx48epqalh/fr1eDwe1q5dy4YNGzCbzZSWlrJ06VISExNZt24d9fX1pKSk9O3IREQkIEFPGVmtVhISEgAICwtj7NixuN3uHrevq6sjPT2d0NBQYmJiiI2N5fDhw3g8Htra2khKSsJkMjFjxgzq6uqCHZaIiAQp6CuEz2pqauLo0aOMHz+eDz74gDfeeIOqqioSEhJYtGgRERERuN1uEhMTjT42mw23243FYsFutxvtdru9x2ApLy+nvLwcgLy8PBwOR1DjPRFUr/4R7JgHU0hIyLAc92BRvQKnmgVmoOrV50Bob2+noKCAhx56iPDwcO68807uu+8+AF5//XW2bNnCo48+is/n67Z/T+3dcTqdOJ1OY7m5ublvgx8Ew3HMDodjWI57sKhegVPNAtOXesXFxfW4rk9PGZ07d46CggLuuOMObr/9dgBGjRqF2WzGbDYza9Ysjhw5Alw8829paTH6ut1ubDZbl/aWlhZsNltfhiUiIkEIOhB8Ph+bNm1i7NixzJ4922j3eDzGn2tra4mPjwcgNTWVmpoaOjo6aGpqorGxkfHjx2O1WgkLC+PQoUP4fD6qqqpITU3twyGJiEgwgp4y+vDDD6mqquKGG25g1apVwMVHTKurqzl27Bgmk4nRo0ezZMkSAOLj40lLS2PlypWYzWYWL16M2Xwxj3JycigpKcHr9ZKcnKwnjEREBoHJF8gk/hDT0NAQVL/zj8zpfaMBYindPWj7DpbmdwOjegVONQvMkLyHICIi1w4FgoiIAAoEERHxUyCIiAigQBARET8FgoiIAAoEERHxUyCIiAigQBARET8FgoiIAAoEERHx65cX5IjIF1tfvx+sLy+tGo7fDzZU6QpBREQABYKIiPgpEEREBNA9hC+Mvszx9mV+FzTHKzJc6ApBREQAXSGIiARlMN+8SFnNgHzskAmE+vp6/u3f/o0LFy4wa9Ys5s2bN9hDEgnKiXvSB23fmp6TvhgSU0YXLlzgtddeY82aNRQWFlJdXc3x48cHe1giIl8oQyIQDh8+TGxsLGPGjCEkJIT09HTq6uoGe1giIl8oQ2LKyO12Y7fbjWW73c4f/vCHLtuVl5dTXl4OQF5eHnFxccHt8Bf7gus3nH0Rj3mwfBFrrWO+6oL+/XcZQ+IKwefzdWkzmUxd2pxOJ3l5eeTl5fVpf0888USf+n/RqF6BUb0Cp5oFZqDqNSQCwW6309LSYiy3tLRgtVoHcUQiIl88QyIQxo0bR2NjI01NTZw7d46amhpSU1MHe1giIl8oQ+IegsVi4eGHH+aFF17gwoUL/OVf/iXx8fEDtj+n0zlgn30tUr0Co3oFTjULzEDVy+TrbgJfRES+cIbElJGIiAw+BYKIiABD5B5CX8yfP5/Zs2ezaNEiAHbv3k17ezvz588HLv7fhZ///OcAhIWF8b3vfY+bbroJgGeeeYb29nbjMdYjR46wdetWnnnmGd5//31eeuklYmJijH0tXLiQW2655WoeXr86deoUmzdv5g9/+AMjR44kJCSEuXPnctttt/HBBx+wefNm2traAJg9e3anecre6ujxeAgNDeXcuXNMmjSJBQsWMHLkyKt/kD34zne+ww033GAsT58+nTlz5vDkk0/yve99jwkTJgDw/PPPM2vWLNLS0ti9ezcVFRVYLBbMZjOzZ88mIyPDON4vfelLAMTGxvL973+f7du3M2LECObM6fwdNwsXLmTr1q2d2rZv386bb75JVFSU0fbMM89w7Ngxnn32WVavXm08WJGXl8e3vvUtfvnLX9LU1ER7eztnzpwxfjZzcnL42te+1v9F60ZtbS0vv/yy8Y0C586d44EHHjDWHzt2jA0bNlBYWMhjjz3GiBEjMJsvnnfefPPNPPzwwxQXF3Pw4EHCw8MBWLRoEZMmTTI+4+c//zk//elPKS0tNbaBi19v8/rrr9PW1kZoaChxcXEsXLgQh8PR5TO//OUv8/zzz1+NkvSrz/+crlq1ipiYmG5r8v777/Ozn/2MJ554gsrKSrZu3YrNZqOjowOn08ns2bMD3v+wD4TQ0FB++9vfMm/evE7/uADeeecd/vd//5fnnnuOqKgoXC4X+fn5rFu3jlGjRgFw+vRpfve735GSktLls2+++eZr5vlon89Hfn4+GRkZ/N3f/R0AJ0+eZN++fZw6dYoNGzawatUqEhISOHPmDC+88AI2m43JkydfUR0ff/xxxo0bx7lz5/iP//gPXnrpJZ599tlBPOLOvvSlL5Gfn9+lPScnh02bNvGP//iP7N27F5PJRFpaGr/+9a957733ePHFFwkPD+fs2bPU1tYa/S4db19kZWV1CQ+4+Bh2WVlZlyftVq1aBXT+RXC1/eY3v+Gmm26iurqar3/967z44oudAqG6uprp06cby08//XSXf5dwMSSnTZvGgQMH+Od//mc2btzY6TPGjRtHbW0tmZmZAPzxj3/kX//1X1m9ejXXX389APv27aOpqQmHw9HpM4eznn5Ou6vJ56Wnp7N48WJaW1vJzc1l2rRpRm2u1LCfMjKbzTidTn7xi190Wbdr1y4WLlxo/EAmJCSQkZHBr371K2ObOXPmsGPHjqs23sFy4MABQkJCuPPOO4220aNHc9ddd/GrX/2KzMxMEhISAIiKiuLBBx9k586dwJXV8ZKQkBAefPBBmpubOXbs2IAfV18lJibyta99jf/6r//ipz/9KYsXLwagrKyMxYsXG2dj4eHhPf5D7G833ngj4eHhvPvuu1dlf1eqvb2dDz/8kGXLllFTU0NcXBwjR47s9K0Cb7/9dqdA6E1SUhJut9tY/uSTT2hvb2fBggVUV1cb7bt27eKee+4xwgAgNTXVuLK7lvVUk55ERkYSGxvLqVOnAt7XsL9CAPjGN77BqlWrmDt3bqf2jz/+2Pgld8m4cePYs2ePsZyUlERtbS0HDhwgLCys07a///3vjbMygO9///vExsYOwBEMvI8//pivfvWr3a47fvw4GRkZndrGjRvHxx9/bPTtrY6fZTabufHGG2loaOArX/lK3wffD7xeb6e/y3vuuYf09IvfSvrAAw/wN3/zN9x9993ExsbS1tZGe3v7Zf+uN27caEwZ3XLLLSxcuDDgMf3iF7/g//7v/wCIiIjg6aefNtbde++9/Od//ueQmqKsra0lOTmZuLg4IiIicLlcTJ8+nerqahITEzl06BCRkZFcd911Rp9nn33WmDLKyMjoMo1RX1/P1KlTjeVLVxg33XQTDQ0NnD59mujoaI4fP863vvWty45v69at/M///A8A8fHxPP744/116FfNZ39OY2JiWLVqVY816UlzczNer7fT1NOVuiYCITw8nBkzZvDLX/7S+Efak+6esv32t7/Njh07+O53v9up/VqaMvq8f/mXf+HDDz8kJCQEm83W7VeFdNd2yXB7WrmnS3HAmHu+FIBXcmwDOWUEF3/24OJJyVBRXV1NVlYWcHF6orq6mrvuuounnnqKRYsWUVNT0+XqoKcpo61bt/KTn/yE06dP88ILLxjtNTU1/P3f/z1ms5nbbruNt99+m29+85ud+ra2tvLcc8/h9XqZNWuWUcNrdcroSmpyabv333+fhoYGli5d2uvvwu4M+ymjS7Kysnjrrbf485//bLRdf/31uFyuTtsdPXq002UnwF/8xV/Q0dHR7RfqXSvi4+M5evSosZyTk8NTTz3FmTNniI+P58iRI522d7lcRp2utI6XXLhwgT/+8Y+MHTu2n4+i/7W3t7Nt2zZ+9KMfcebMGfbv3094eDgjRozgxIm+vjy0b+69994hM53Z2trKgQMH2LRpE4899hg/+9nPqKmpwW63M3r0aA4ePMhvf/tb0tLSrujzFi5cyMaNG1mwYAHFxcUAfPTRRzQ2NvL888/z2GOPUVNTY0yRXH/99cbPb2RkJPn5+cyaNYv29vaBOeAh4nI1+bz09HTWr1/Pc889x5YtW4KaMrpmAiEiIoK0tDQqKiqMtrlz57Jt2zZaW1uBi09AVFZW8o1vfKNL/3vuuYddu3ZdtfFebZdC79e//rXR5vV6gYtTbpWVlcacf2trK9u2bTPOvAKp46Wbyna7nRtvvHGAj6rv/vu//5u0tDTGjh1LTk4Omzdvxuv1Mm/ePF577TXOnj0LwNmzZ41v2r1abr31Vv70pz/x0UcfXdX9dmfv3r1kZGRQUlJCcXEx//RP/0RMTAwffPAB06dPZ/PmzYwZM6bTtxb3xmw2c/fdd+Pz+aivr6e6upq//uu/pri4mOLiYn784x/jdrs5efIkc+fOZceOHZ3ek3Lp5/dadrma9CQpKcmYMQnUNTFldMns2bM73ehMTU3F7XbzD//wD5hMJsLCwli+fHm3X5w3efLkLpe2n7+H8O1vf3vYXpKaTCZWrVrF5s2b2bVrF1FRUYwYMYLvfve7WK1Wli9fzo9//GPa2trw+XxkZWUZT7lcSR03btxIaGgoHR0dTJo0idWrVw/WoXbr8/cQkpOTycjIoK6uzrhE/8pXvsKtt97Krl27uO+++2hvb+fJJ58kJCQEi8XSaf77s/cQoqKieOqppwDYsWNHp3+ImzZtwuv1smzZMqPt0ud89h4C0Gl8l9x777289NJL/VGCPqmuru7yFsPbb7+d3/zmN3znO9/h3//938nOzu7S77P3EG688Ub+9m//ttN6k8nEvffey+7duzlx4gRr1qzptP62224z9p2dnU1xcTFtbW1ERkZit9uNx8uh8z0EgHXr1hESMrx/xVVXV/dYk8TExB77zZ07lx/84Afcc889Xe6NXo6+ukJERIBraMpIRET6RoEgIiKAAkFERPwUCCIiAigQRETET4EgIiKAAkFERPz+H7CpNNFIcBjYAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Histogram of the values in 'view'\n",
    "plt.hist(data['view']);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**`view` can be made into a binary variable where 1 means there is a view and 0 means there is not.**\n",
    "\n",
    "**While we're at, we'll do this for `renovated_last_10` and `waterfront`.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Binarizing view\n",
    "view_dict = {\n",
    "    'FAIR': 1,\n",
    "    'AVERAGE': 1,\n",
    "    'GOOD': 1,\n",
    "    'EXCELLENT': 1,\n",
    "    'NONE': 0\n",
    "}\n",
    "\n",
    "data['view'] = data['view'].map(view_dict)\n",
    "\n",
    "# Binarizing waterfront\n",
    "waterfront_dict = {\n",
    "    'YES': 1,\n",
    "    'NO': 0\n",
    "}\n",
    "\n",
    "data['waterfront'] = data['waterfront'].map(waterfront_dict)\n",
    "\n",
    "# Binarizing renovated_last_10\n",
    "renovated_dict = {\n",
    "    'Yes': 1,\n",
    "    'No': 0\n",
    "}\n",
    "\n",
    "data['renovated_last_10'] = data['renovated_last_10'].map(renovated_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**I want to standardize (stylistically) what `grade` and `condition` look like.** "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7 Average        8934\n",
       "8 Good           6035\n",
       "9 Better         2605\n",
       "6 Low Average    2023\n",
       "10 Very Good     1118\n",
       "11 Excellent      363\n",
       "5 Fair            232\n",
       "12 Luxury          68\n",
       "Name: grade, dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['grade'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Average      13886\n",
       "Good          5636\n",
       "Very Good     1690\n",
       "Fair           166\n",
       "Name: condition, dtype: int64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['condition'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace spaces in grade with undescores\n",
    "\n",
    "data['grade'] = data['grade'].str.replace(' ', '_')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Give condition the same syntax as grade\n",
    "\n",
    "condition_dict = {\n",
    "    'Poor': '1_Poor',\n",
    "    'Fair': '2_Fair',\n",
    "    'Average': '3_Average',\n",
    "    'Good': '4_Good',\n",
    "    'Very Good': '5_Very_Good'\n",
    "}\n",
    "\n",
    "data['condition'] = data['condition'].map(condition_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Getting dummies\n",
    "In order for the model to process the categorical variables, they need to made into dummy variables.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving a copy of the dataset in its current state just in case\n",
    "data_before_dummies = data.copy()\n",
    "# data_before_dummies.to_csv('./data/data_before_dummies.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Floors, Bathrooms, and Bedrooms\n",
    "# Turning these into strings so that Pandas can turn them into dummies.\n",
    "\n",
    "data[['floors', 'bedrooms', 'bathrooms']] = data[['floors', 'bedrooms', 'bathrooms']].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The three types of variables my cleaned up dataset will have\n",
    "\n",
    "data_binary = data[['waterfront', 'view', 'renovated_last_10']]\n",
    "data_num = data[['price', 'sqft_living', 'sqft_lot', 'age']]\n",
    "data_cat = data[['floors', 'bedrooms', 'bathrooms', 'condition', 'grade']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating dummies\n",
    "# I am NOT dropping the first dummy arbitrarily.\n",
    "# Instead, I will choose which specific dummy I want to drop \n",
    "# in order to make the model coefficients more interpretable.\n",
    "\n",
    "data_cat_dummies = pd.get_dummies(data_cat, dtype=int)\n",
    "\n",
    "# These are the lowest order from each set of categories (that haven't already dropped)\n",
    "# Dummy variable coefficients will be interpreted in reference to these.\n",
    "\n",
    "dummies_to_drop = [\n",
    "    'floors_1.0', \n",
    "    'bedrooms_1', \n",
    "    'bathrooms_0.75', \n",
    "    'condition_2_Fair', \n",
    "    'grade_5_Fair'\n",
    "    ]\n",
    "\n",
    "data_cat_dummies.drop(\n",
    "    dummies_to_drop,\n",
    "     axis = 1,\n",
    "     inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>price</th>\n",
       "      <th>sqft_living</th>\n",
       "      <th>sqft_lot</th>\n",
       "      <th>age</th>\n",
       "      <th>waterfront</th>\n",
       "      <th>view</th>\n",
       "      <th>renovated_last_10</th>\n",
       "      <th>floors_1.5</th>\n",
       "      <th>floors_2.0</th>\n",
       "      <th>floors_2.5</th>\n",
       "      <th>...</th>\n",
       "      <th>condition_3_Average</th>\n",
       "      <th>condition_4_Good</th>\n",
       "      <th>condition_5_Very_Good</th>\n",
       "      <th>grade_10_Very_Good</th>\n",
       "      <th>grade_11_Excellent</th>\n",
       "      <th>grade_12_Luxury</th>\n",
       "      <th>grade_6_Low_Average</th>\n",
       "      <th>grade_7_Average</th>\n",
       "      <th>grade_8_Good</th>\n",
       "      <th>grade_9_Better</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>221900.0</td>\n",
       "      <td>1180</td>\n",
       "      <td>5650</td>\n",
       "      <td>59</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>538000.0</td>\n",
       "      <td>2570</td>\n",
       "      <td>7242</td>\n",
       "      <td>63</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>180000.0</td>\n",
       "      <td>770</td>\n",
       "      <td>10000</td>\n",
       "      <td>82</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>604000.0</td>\n",
       "      <td>1960</td>\n",
       "      <td>5000</td>\n",
       "      <td>49</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>510000.0</td>\n",
       "      <td>1680</td>\n",
       "      <td>8080</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      price  sqft_living  sqft_lot  age  waterfront  view  renovated_last_10  \\\n",
       "0  221900.0         1180      5650   59           0     0                  0   \n",
       "1  538000.0         2570      7242   63           0     0                  0   \n",
       "2  180000.0          770     10000   82           0     0                  0   \n",
       "3  604000.0         1960      5000   49           0     0                  0   \n",
       "4  510000.0         1680      8080   28           0     0                  0   \n",
       "\n",
       "   floors_1.5  floors_2.0  floors_2.5  ...  condition_3_Average  \\\n",
       "0           0           0           0  ...                    1   \n",
       "1           0           1           0  ...                    1   \n",
       "2           0           0           0  ...                    1   \n",
       "3           0           0           0  ...                    0   \n",
       "4           0           0           0  ...                    1   \n",
       "\n",
       "   condition_4_Good  condition_5_Very_Good  grade_10_Very_Good  \\\n",
       "0                 0                      0                   0   \n",
       "1                 0                      0                   0   \n",
       "2                 0                      0                   0   \n",
       "3                 0                      1                   0   \n",
       "4                 0                      0                   0   \n",
       "\n",
       "   grade_11_Excellent  grade_12_Luxury  grade_6_Low_Average  grade_7_Average  \\\n",
       "0                   0                0                    0                1   \n",
       "1                   0                0                    0                1   \n",
       "2                   0                0                    1                0   \n",
       "3                   0                0                    0                1   \n",
       "4                   0                0                    0                0   \n",
       "\n",
       "   grade_8_Good  grade_9_Better  \n",
       "0             0               0  \n",
       "1             0               0  \n",
       "2             0               0  \n",
       "3             0               0  \n",
       "4             1               0  \n",
       "\n",
       "[5 rows x 40 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Merge all the variables back together\n",
    "\n",
    "data = pd.concat([data_num, data_binary, data_cat_dummies], axis=1)\n",
    "\n",
    "data.head() # preview\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['price', 'sqft_living', 'sqft_lot', 'age', 'waterfront', 'view',\n",
       "       'renovated_last_10', 'floors_1.5', 'floors_2.0', 'floors_2.5',\n",
       "       'floors_3.0', 'bedrooms_2', 'bedrooms_3', 'bedrooms_4', 'bedrooms_5',\n",
       "       'bedrooms_6', 'bathrooms_1.0', 'bathrooms_1.5', 'bathrooms_1.75',\n",
       "       'bathrooms_2.0', 'bathrooms_2.25', 'bathrooms_2.5', 'bathrooms_2.75',\n",
       "       'bathrooms_3.0', 'bathrooms_3.25', 'bathrooms_3.5', 'bathrooms_3.75',\n",
       "       'bathrooms_4.0', 'bathrooms_4.25', 'bathrooms_4.5',\n",
       "       'condition_3_Average', 'condition_4_Good', 'condition_5_Very_Good',\n",
       "       'grade_10_Very_Good', 'grade_11_Excellent', 'grade_12_Luxury',\n",
       "       'grade_6_Low_Average', 'grade_7_Average', 'grade_8_Good',\n",
       "       'grade_9_Better'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Columns for `grade` are out of order. I will reposition them for readability."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['price',\n",
       " 'sqft_living',\n",
       " 'sqft_lot',\n",
       " 'age',\n",
       " 'waterfront',\n",
       " 'view',\n",
       " 'renovated_last_10',\n",
       " 'floors_1.5',\n",
       " 'floors_2.0',\n",
       " 'floors_2.5',\n",
       " 'floors_3.0',\n",
       " 'bedrooms_2',\n",
       " 'bedrooms_3',\n",
       " 'bedrooms_4',\n",
       " 'bedrooms_5',\n",
       " 'bedrooms_6',\n",
       " 'bathrooms_1.0',\n",
       " 'bathrooms_1.5',\n",
       " 'bathrooms_1.75',\n",
       " 'bathrooms_2.0',\n",
       " 'bathrooms_2.25',\n",
       " 'bathrooms_2.5',\n",
       " 'bathrooms_2.75',\n",
       " 'bathrooms_3.0',\n",
       " 'bathrooms_3.25',\n",
       " 'bathrooms_3.5',\n",
       " 'bathrooms_3.75',\n",
       " 'bathrooms_4.0',\n",
       " 'bathrooms_4.25',\n",
       " 'bathrooms_4.5',\n",
       " 'condition_3_Average',\n",
       " 'condition_4_Good',\n",
       " 'condition_5_Very_Good',\n",
       " 'grade_10_Very_Good',\n",
       " 'grade_11_Excellent',\n",
       " 'grade_12_Luxury',\n",
       " 'grade_6_Low_Average',\n",
       " 'grade_7_Average',\n",
       " 'grade_8_Good',\n",
       " 'grade_9_Better']"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_columns = data.columns.to_list()\n",
    "df_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_columns_reordered = [\n",
    "'price',\n",
    "'sqft_living',\n",
    "'sqft_lot',\n",
    "'age',\n",
    "'waterfront',\n",
    "'view',\n",
    "'renovated_last_10',\n",
    "'floors_1.5',\n",
    "'floors_2.0',\n",
    "'floors_2.5',\n",
    "'floors_3.0',\n",
    "'bedrooms_2',\n",
    "'bedrooms_3',\n",
    "'bedrooms_4',\n",
    "'bedrooms_5',\n",
    "'bedrooms_6',\n",
    "'bathrooms_1.0',\n",
    "'bathrooms_1.5',\n",
    "'bathrooms_1.75',\n",
    "'bathrooms_2.0',\n",
    "'bathrooms_2.25',\n",
    "'bathrooms_2.5',\n",
    "'bathrooms_2.75',\n",
    "'bathrooms_3.0',\n",
    "'bathrooms_3.25',\n",
    "'bathrooms_3.5',\n",
    "'bathrooms_3.75',\n",
    "'bathrooms_4.0',\n",
    "'bathrooms_4.25',\n",
    "'bathrooms_4.5',\n",
    "'condition_3_Average',\n",
    "'condition_4_Good',\n",
    "'condition_5_Very_Good',\n",
    "'grade_6_Low_Average',\n",
    "'grade_7_Average',\n",
    "'grade_8_Good',\n",
    "'grade_9_Better',\n",
    "'grade_10_Very_Good',\n",
    "'grade_11_Excellent',\n",
    "'grade_12_Luxury',\n",
    "]\n",
    "\n",
    "data = data[df_columns_reordered]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>price</th>\n",
       "      <th>sqft_living</th>\n",
       "      <th>sqft_lot</th>\n",
       "      <th>age</th>\n",
       "      <th>waterfront</th>\n",
       "      <th>view</th>\n",
       "      <th>renovated_last_10</th>\n",
       "      <th>floors_1.5</th>\n",
       "      <th>floors_2.0</th>\n",
       "      <th>floors_2.5</th>\n",
       "      <th>...</th>\n",
       "      <th>condition_3_Average</th>\n",
       "      <th>condition_4_Good</th>\n",
       "      <th>condition_5_Very_Good</th>\n",
       "      <th>grade_6_Low_Average</th>\n",
       "      <th>grade_7_Average</th>\n",
       "      <th>grade_8_Good</th>\n",
       "      <th>grade_9_Better</th>\n",
       "      <th>grade_10_Very_Good</th>\n",
       "      <th>grade_11_Excellent</th>\n",
       "      <th>grade_12_Luxury</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>221900.0</td>\n",
       "      <td>1180</td>\n",
       "      <td>5650</td>\n",
       "      <td>59</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>538000.0</td>\n",
       "      <td>2570</td>\n",
       "      <td>7242</td>\n",
       "      <td>63</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>180000.0</td>\n",
       "      <td>770</td>\n",
       "      <td>10000</td>\n",
       "      <td>82</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>604000.0</td>\n",
       "      <td>1960</td>\n",
       "      <td>5000</td>\n",
       "      <td>49</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>510000.0</td>\n",
       "      <td>1680</td>\n",
       "      <td>8080</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      price  sqft_living  sqft_lot  age  waterfront  view  renovated_last_10  \\\n",
       "0  221900.0         1180      5650   59           0     0                  0   \n",
       "1  538000.0         2570      7242   63           0     0                  0   \n",
       "2  180000.0          770     10000   82           0     0                  0   \n",
       "3  604000.0         1960      5000   49           0     0                  0   \n",
       "4  510000.0         1680      8080   28           0     0                  0   \n",
       "\n",
       "   floors_1.5  floors_2.0  floors_2.5  ...  condition_3_Average  \\\n",
       "0           0           0           0  ...                    1   \n",
       "1           0           1           0  ...                    1   \n",
       "2           0           0           0  ...                    1   \n",
       "3           0           0           0  ...                    0   \n",
       "4           0           0           0  ...                    1   \n",
       "\n",
       "   condition_4_Good  condition_5_Very_Good  grade_6_Low_Average  \\\n",
       "0                 0                      0                    0   \n",
       "1                 0                      0                    0   \n",
       "2                 0                      0                    1   \n",
       "3                 0                      1                    0   \n",
       "4                 0                      0                    0   \n",
       "\n",
       "   grade_7_Average  grade_8_Good  grade_9_Better  grade_10_Very_Good  \\\n",
       "0                1             0               0                   0   \n",
       "1                1             0               0                   0   \n",
       "2                0             0               0                   0   \n",
       "3                1             0               0                   0   \n",
       "4                0             1               0                   0   \n",
       "\n",
       "   grade_11_Excellent  grade_12_Luxury  \n",
       "0                   0                0  \n",
       "1                   0                0  \n",
       "2                   0                0  \n",
       "3                   0                0  \n",
       "4                   0                0  \n",
       "\n",
       "[5 rows x 40 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['price', 'sqft_living', 'sqft_lot', 'age', 'waterfront', 'view',\n",
       "       'renovated_last_10', 'floors_1.5', 'floors_2.0', 'floors_2.5',\n",
       "       'floors_3.0', 'bedrooms_2', 'bedrooms_3', 'bedrooms_4', 'bedrooms_5',\n",
       "       'bedrooms_6', 'bathrooms_1.0', 'bathrooms_1.5', 'bathrooms_1.75',\n",
       "       'bathrooms_2.0', 'bathrooms_2.25', 'bathrooms_2.5', 'bathrooms_2.75',\n",
       "       'bathrooms_3.0', 'bathrooms_3.25', 'bathrooms_3.5', 'bathrooms_3.75',\n",
       "       'bathrooms_4.0', 'bathrooms_4.25', 'bathrooms_4.5',\n",
       "       'condition_3_Average', 'condition_4_Good', 'condition_5_Very_Good',\n",
       "       'grade_6_Low_Average', 'grade_7_Average', 'grade_8_Good',\n",
       "       'grade_9_Better', 'grade_10_Very_Good', 'grade_11_Excellent',\n",
       "       'grade_12_Luxury'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save this version of the dataset\n",
    "# data.to_csv('./data/data_post_prep.csv')"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "9066515e2c74396c3dc22ac47132aa38294e007e809df6151bffa6da73189c66"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}