Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### We will start by making dataset suitable for further processing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>MSZoning</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>Street</th>\n",
       "      <th>Alley</th>\n",
       "      <th>LotShape</th>\n",
       "      <th>LandContour</th>\n",
       "      <th>Utilities</th>\n",
       "      <th>LotConfig</th>\n",
       "      <th>LandSlope</th>\n",
       "      <th>Neighborhood</th>\n",
       "      <th>Condition1</th>\n",
       "      <th>Condition2</th>\n",
       "      <th>BldgType</th>\n",
       "      <th>HouseStyle</th>\n",
       "      <th>OverallQual</th>\n",
       "      <th>OverallCond</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>YearRemodAdd</th>\n",
       "      <th>RoofStyle</th>\n",
       "      <th>RoofMatl</th>\n",
       "      <th>Exterior1st</th>\n",
       "      <th>Exterior2nd</th>\n",
       "      <th>MasVnrType</th>\n",
       "      <th>MasVnrArea</th>\n",
       "      <th>ExterQual</th>\n",
       "      <th>ExterCond</th>\n",
       "      <th>Foundation</th>\n",
       "      <th>BsmtQual</th>\n",
       "      <th>BsmtCond</th>\n",
       "      <th>BsmtExposure</th>\n",
       "      <th>BsmtFinType1</th>\n",
       "      <th>BsmtFinSF1</th>\n",
       "      <th>BsmtFinType2</th>\n",
       "      <th>BsmtFinSF2</th>\n",
       "      <th>BsmtUnfSF</th>\n",
       "      <th>TotalBsmtSF</th>\n",
       "      <th>Heating</th>\n",
       "      <th>HeatingQC</th>\n",
       "      <th>CentralAir</th>\n",
       "      <th>Electrical</th>\n",
       "      <th>1stFlrSF</th>\n",
       "      <th>2ndFlrSF</th>\n",
       "      <th>LowQualFinSF</th>\n",
       "      <th>GrLivArea</th>\n",
       "      <th>BsmtFullBath</th>\n",
       "      <th>BsmtHalfBath</th>\n",
       "      <th>FullBath</th>\n",
       "      <th>HalfBath</th>\n",
       "      <th>BedroomAbvGr</th>\n",
       "      <th>KitchenAbvGr</th>\n",
       "      <th>KitchenQual</th>\n",
       "      <th>TotRmsAbvGrd</th>\n",
       "      <th>Functional</th>\n",
       "      <th>Fireplaces</th>\n",
       "      <th>FireplaceQu</th>\n",
       "      <th>GarageType</th>\n",
       "      <th>GarageYrBlt</th>\n",
       "      <th>GarageFinish</th>\n",
       "      <th>GarageCars</th>\n",
       "      <th>GarageArea</th>\n",
       "      <th>GarageQual</th>\n",
       "      <th>GarageCond</th>\n",
       "      <th>PavedDrive</th>\n",
       "      <th>WoodDeckSF</th>\n",
       "      <th>OpenPorchSF</th>\n",
       "      <th>EnclosedPorch</th>\n",
       "      <th>3SsnPorch</th>\n",
       "      <th>ScreenPorch</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>PoolQC</th>\n",
       "      <th>Fence</th>\n",
       "      <th>MiscFeature</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SaleType</th>\n",
       "      <th>SaleCondition</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1246</th>\n",
       "      <td>60</td>\n",
       "      <td>FV</td>\n",
       "      <td>65.0</td>\n",
       "      <td>8125</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>Somerst</td>\n",
       "      <td>Norm</td>\n",
       "      <td>Norm</td>\n",
       "      <td>1Fam</td>\n",
       "      <td>2Story</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>2005</td>\n",
       "      <td>2006</td>\n",
       "      <td>Gable</td>\n",
       "      <td>CompShg</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>None</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>PConc</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>No</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>756</td>\n",
       "      <td>756</td>\n",
       "      <td>GasA</td>\n",
       "      <td>Ex</td>\n",
       "      <td>Y</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>756</td>\n",
       "      <td>797</td>\n",
       "      <td>0</td>\n",
       "      <td>1553</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>Gd</td>\n",
       "      <td>6</td>\n",
       "      <td>Typ</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>2005.0</td>\n",
       "      <td>RFn</td>\n",
       "      <td>2</td>\n",
       "      <td>615</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>45</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2006</td>\n",
       "      <td>New</td>\n",
       "      <td>Partial</td>\n",
       "      <td>186500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>420</th>\n",
       "      <td>90</td>\n",
       "      <td>RM</td>\n",
       "      <td>78.0</td>\n",
       "      <td>7060</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>Mitchel</td>\n",
       "      <td>Norm</td>\n",
       "      <td>Norm</td>\n",
       "      <td>Duplex</td>\n",
       "      <td>SFoyer</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>1997</td>\n",
       "      <td>1998</td>\n",
       "      <td>Gable</td>\n",
       "      <td>CompShg</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>BrkFace</td>\n",
       "      <td>200.0</td>\n",
       "      <td>TA</td>\n",
       "      <td>Gd</td>\n",
       "      <td>PConc</td>\n",
       "      <td>Gd</td>\n",
       "      <td>Gd</td>\n",
       "      <td>Gd</td>\n",
       "      <td>GLQ</td>\n",
       "      <td>1309</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>1344</td>\n",
       "      <td>GasA</td>\n",
       "      <td>Ex</td>\n",
       "      <td>Y</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>1344</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1344</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>TA</td>\n",
       "      <td>8</td>\n",
       "      <td>Typ</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>1997.0</td>\n",
       "      <td>Fin</td>\n",
       "      <td>4</td>\n",
       "      <td>784</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Alloca</td>\n",
       "      <td>206300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>672</th>\n",
       "      <td>20</td>\n",
       "      <td>RL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11250</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>Veenker</td>\n",
       "      <td>Norm</td>\n",
       "      <td>Norm</td>\n",
       "      <td>1Fam</td>\n",
       "      <td>1Story</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>1977</td>\n",
       "      <td>1977</td>\n",
       "      <td>Gable</td>\n",
       "      <td>CompShg</td>\n",
       "      <td>Plywood</td>\n",
       "      <td>Plywood</td>\n",
       "      <td>None</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>CBlock</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>No</td>\n",
       "      <td>ALQ</td>\n",
       "      <td>767</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>441</td>\n",
       "      <td>1208</td>\n",
       "      <td>GasA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>1208</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1208</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>TA</td>\n",
       "      <td>6</td>\n",
       "      <td>Typ</td>\n",
       "      <td>1</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>1977.0</td>\n",
       "      <td>RFn</td>\n",
       "      <td>2</td>\n",
       "      <td>546</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>198</td>\n",
       "      <td>42</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2006</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>165000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>101.0</td>\n",
       "      <td>14215</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Corner</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>NridgHt</td>\n",
       "      <td>Norm</td>\n",
       "      <td>Norm</td>\n",
       "      <td>1Fam</td>\n",
       "      <td>2Story</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>2005</td>\n",
       "      <td>2006</td>\n",
       "      <td>Gable</td>\n",
       "      <td>CompShg</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>BrkFace</td>\n",
       "      <td>380.0</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>PConc</td>\n",
       "      <td>Ex</td>\n",
       "      <td>TA</td>\n",
       "      <td>Av</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>1158</td>\n",
       "      <td>1158</td>\n",
       "      <td>GasA</td>\n",
       "      <td>Ex</td>\n",
       "      <td>Y</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>1158</td>\n",
       "      <td>1218</td>\n",
       "      <td>0</td>\n",
       "      <td>2376</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>Gd</td>\n",
       "      <td>9</td>\n",
       "      <td>Typ</td>\n",
       "      <td>1</td>\n",
       "      <td>Gd</td>\n",
       "      <td>BuiltIn</td>\n",
       "      <td>2005.0</td>\n",
       "      <td>RFn</td>\n",
       "      <td>3</td>\n",
       "      <td>853</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>240</td>\n",
       "      <td>154</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>2006</td>\n",
       "      <td>New</td>\n",
       "      <td>Partial</td>\n",
       "      <td>325300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>406</th>\n",
       "      <td>50</td>\n",
       "      <td>RL</td>\n",
       "      <td>51.0</td>\n",
       "      <td>10480</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>SWISU</td>\n",
       "      <td>Norm</td>\n",
       "      <td>Norm</td>\n",
       "      <td>1Fam</td>\n",
       "      <td>1.5Fin</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>1936</td>\n",
       "      <td>1950</td>\n",
       "      <td>Gable</td>\n",
       "      <td>CompShg</td>\n",
       "      <td>MetalSd</td>\n",
       "      <td>MetalSd</td>\n",
       "      <td>None</td>\n",
       "      <td>0.0</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>BrkTil</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>No</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>1064</td>\n",
       "      <td>1064</td>\n",
       "      <td>GasA</td>\n",
       "      <td>Ex</td>\n",
       "      <td>Y</td>\n",
       "      <td>FuseA</td>\n",
       "      <td>1166</td>\n",
       "      <td>0</td>\n",
       "      <td>473</td>\n",
       "      <td>1639</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>TA</td>\n",
       "      <td>6</td>\n",
       "      <td>Maj2</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Detchd</td>\n",
       "      <td>1936.0</td>\n",
       "      <td>Unf</td>\n",
       "      <td>1</td>\n",
       "      <td>240</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>115000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \\\n",
       "1246          60       FV         65.0     8125   Pave   NaN      Reg   \n",
       "420           90       RM         78.0     7060   Pave   NaN      Reg   \n",
       "672           20       RL          NaN    11250   Pave   NaN      IR1   \n",
       "20            60       RL        101.0    14215   Pave   NaN      IR1   \n",
       "406           50       RL         51.0    10480   Pave   NaN      Reg   \n",
       "\n",
       "     LandContour Utilities LotConfig LandSlope Neighborhood Condition1  \\\n",
       "1246         Lvl    AllPub    Inside       Gtl      Somerst       Norm   \n",
       "420          Lvl    AllPub    Inside       Gtl      Mitchel       Norm   \n",
       "672          Lvl    AllPub    Inside       Gtl      Veenker       Norm   \n",
       "20           Lvl    AllPub    Corner       Gtl      NridgHt       Norm   \n",
       "406          Lvl    AllPub    Inside       Gtl        SWISU       Norm   \n",
       "\n",
       "     Condition2 BldgType HouseStyle  OverallQual  OverallCond  YearBuilt  \\\n",
       "1246       Norm     1Fam     2Story            7            5       2005   \n",
       "420        Norm   Duplex     SFoyer            7            5       1997   \n",
       "672        Norm     1Fam     1Story            6            6       1977   \n",
       "20         Norm     1Fam     2Story            8            5       2005   \n",
       "406        Norm     1Fam     1.5Fin            6            5       1936   \n",
       "\n",
       "      YearRemodAdd RoofStyle RoofMatl Exterior1st Exterior2nd MasVnrType  \\\n",
       "1246          2006     Gable  CompShg     VinylSd     VinylSd       None   \n",
       "420           1998     Gable  CompShg     VinylSd     VinylSd    BrkFace   \n",
       "672           1977     Gable  CompShg     Plywood     Plywood       None   \n",
       "20            2006     Gable  CompShg     VinylSd     VinylSd    BrkFace   \n",
       "406           1950     Gable  CompShg     MetalSd     MetalSd       None   \n",
       "\n",
       "      MasVnrArea ExterQual ExterCond Foundation BsmtQual BsmtCond  \\\n",
       "1246         0.0        Gd        TA      PConc       Gd       TA   \n",
       "420        200.0        TA        Gd      PConc       Gd       Gd   \n",
       "672          0.0        Gd        TA     CBlock       Gd       TA   \n",
       "20         380.0        Gd        TA      PConc       Ex       TA   \n",
       "406          0.0        TA        TA     BrkTil       TA       TA   \n",
       "\n",
       "     BsmtExposure BsmtFinType1  BsmtFinSF1 BsmtFinType2  BsmtFinSF2  \\\n",
       "1246           No          Unf           0          Unf           0   \n",
       "420            Gd          GLQ        1309          Unf           0   \n",
       "672            No          ALQ         767          Unf           0   \n",
       "20             Av          Unf           0          Unf           0   \n",
       "406            No          Unf           0          Unf           0   \n",
       "\n",
       "      BsmtUnfSF  TotalBsmtSF Heating HeatingQC CentralAir Electrical  \\\n",
       "1246        756          756    GasA        Ex          Y      SBrkr   \n",
       "420          35         1344    GasA        Ex          Y      SBrkr   \n",
       "672         441         1208    GasA        TA          Y      SBrkr   \n",
       "20         1158         1158    GasA        Ex          Y      SBrkr   \n",
       "406        1064         1064    GasA        Ex          Y      FuseA   \n",
       "\n",
       "      1stFlrSF  2ndFlrSF  LowQualFinSF  GrLivArea  BsmtFullBath  BsmtHalfBath  \\\n",
       "1246       756       797             0       1553             0             0   \n",
       "420       1344         0             0       1344             2             0   \n",
       "672       1208         0             0       1208             1             0   \n",
       "20        1158      1218             0       2376             0             0   \n",
       "406       1166         0           473       1639             0             0   \n",
       "\n",
       "      FullBath  HalfBath  BedroomAbvGr  KitchenAbvGr KitchenQual  \\\n",
       "1246         2         1             3             1          Gd   \n",
       "420          2         0             2             2          TA   \n",
       "672          1         1             3             1          TA   \n",
       "20           3         1             4             1          Gd   \n",
       "406          1         0             3             1          TA   \n",
       "\n",
       "      TotRmsAbvGrd Functional  Fireplaces FireplaceQu GarageType  GarageYrBlt  \\\n",
       "1246             6        Typ           0         NaN     Attchd       2005.0   \n",
       "420              8        Typ           0         NaN     Attchd       1997.0   \n",
       "672              6        Typ           1          TA     Attchd       1977.0   \n",
       "20               9        Typ           1          Gd    BuiltIn       2005.0   \n",
       "406              6       Maj2           0         NaN     Detchd       1936.0   \n",
       "\n",
       "     GarageFinish  GarageCars  GarageArea GarageQual GarageCond PavedDrive  \\\n",
       "1246          RFn           2         615         TA         TA          Y   \n",
       "420           Fin           4         784         TA         TA          Y   \n",
       "672           RFn           2         546         TA         TA          Y   \n",
       "20            RFn           3         853         TA         TA          Y   \n",
       "406           Unf           1         240         TA         TA          Y   \n",
       "\n",
       "      WoodDeckSF  OpenPorchSF  EnclosedPorch  3SsnPorch  ScreenPorch  \\\n",
       "1246           0           45              0          0            0   \n",
       "420            0            0              0          0            0   \n",
       "672          198           42              0          0            0   \n",
       "20           240          154              0          0            0   \n",
       "406            0            0              0          0            0   \n",
       "\n",
       "      PoolArea PoolQC Fence MiscFeature  MiscVal  MoSold  YrSold SaleType  \\\n",
       "1246         0    NaN   NaN         NaN        0       3    2006      New   \n",
       "420          0    NaN   NaN         NaN        0      11    2008       WD   \n",
       "672          0    NaN   NaN         NaN        0       6    2006       WD   \n",
       "20           0    NaN   NaN         NaN        0      11    2006      New   \n",
       "406          0    NaN   NaN         NaN        0       3    2008       WD   \n",
       "\n",
       "     SaleCondition  SalePrice  \n",
       "1246       Partial     186500  \n",
       "420         Alloca     206300  \n",
       "672         Normal     165000  \n",
       "20         Partial     325300  \n",
       "406         Normal     115000  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#importing necessary libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "#importing the dataset \n",
    "ah_data=pd.read_csv('C:/Users/admin/Desktop/EeD/Python/Project/AmesHousingData.csv')\n",
    "        \n",
    "#dropping Id column\n",
    "ah_data = ah_data.drop(['Id'], axis = 1)\n",
    "\n",
    "#changing the default column and row display settings to display all cloumns\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.max_rows', 1000)\n",
    "ah_data.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1460, 80)\n"
     ]
    }
   ],
   "source": [
    "#getting to know the dimensions of the dataset\n",
    "print(ah_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#splitting the variables in dataset into predictors and target variable\n",
    "X = ah_data.drop(['SalePrice'],axis = 1)\n",
    "y = ah_data.SalePrice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1168, 79)\n",
      "(292, 79)\n",
      "(1168,)\n",
      "(292,)\n"
     ]
    }
   ],
   "source": [
    "#splitting the dataset into train and test sets\n",
    "from sklearn.model_selection import train_test_split\n",
    "train_X,val_X,train_y,val_y = train_test_split(X,y,test_size = 0.2,random_state = 9)\n",
    "\n",
    "#printing the dimensions of train and test set of predictors and target variable\n",
    "for item in [train_X,val_X,train_y,val_y]:\n",
    "    print(item.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Creating a custom function for imputation with different strategies like \n",
    "imputing mean or median or mode''' \n",
    "\n",
    "from sklearn.base import TransformerMixin\n",
    "from sklearn.base import BaseEstimator\n",
    "\n",
    "class CustomImputer(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, strategy='mean',filler='NA'):\n",
    "       self.strategy = strategy\n",
    "       self.fill = filler\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "       if self.strategy in ['mean','median']:\n",
    "           if not all(X.dtypes == np.number):\n",
    "               raise ValueError('dtypes mismatch np.number dtype is \\\n",
    "                                 required for '+ self.strategy)\n",
    "       if self.strategy == 'mean':\n",
    "           self.fill = X.mean()\n",
    "       elif self.strategy == 'median':\n",
    "           self.fill = X.median()\n",
    "       elif self.strategy == 'mode':\n",
    "           self.fill = X.mode().iloc[0]\n",
    "       elif self.strategy == 'fill':\n",
    "           if type(self.fill) is list and type(X) is pd.DataFrame:\n",
    "               self.fill = dict([(cname, v) for cname,v in zip(X.columns, self.fill)])\n",
    "       return self\n",
    "\n",
    "    def transform(self, X, y=None):\n",
    "       return X.fillna(self.fill)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#using the custom function created to impute mode of the columns  \n",
    "train_X = CustomImputer(strategy = 'mode').fit_transform(train_X)\n",
    "val_X = CustomImputer(strategy = 'mode').fit_transform(val_X)\n",
    "train_y = CustomImputer(strategy = 'mode').fit_transform(train_y)\n",
    "val_y = CustomImputer(strategy = 'mode').fit_transform(val_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "79\n"
     ]
    }
   ],
   "source": [
    "#seggregating columns as per data type \n",
    "integer_columns = train_X.select_dtypes(include=['int64']).columns.tolist()\n",
    "#integer_columns = integer_columns[1:65]\n",
    "char_text_columns = train_X.select_dtypes(include=['object']).columns.tolist()\n",
    "float_columns = train_X.select_dtypes(include = ['float64']).columns.tolist()\n",
    "\n",
    "#we are picking out only integer and float columns and clubbing them together as numbers column\n",
    "numbers_columns = float_columns + integer_columns\n",
    "\n",
    "#confirming that columns add up and nothing is missed\n",
    "all_columns = float_columns + integer_columns + char_text_columns\n",
    "print(len(all_columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = train_X[all_columns]\n",
    "val_X=val_X[all_columns]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}