Data Cleaning Handling Missing Values-NLP.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import the required libraries: Pandas, Numpy, Matplotlib and   Seaborn\n",
    "\n",
    "import pandas as pd          \n",
    "import numpy as np \n",
    "import seaborn as sns # For mathematical calculations\n",
    "import matplotlib.pyplot as plt  # For plotting graphs\n",
    "from datetime import datetime    # To access datetime\n",
    "from pandas import Series # To work on series\n",
    "%matplotlib inline\n",
    "import warnings                   # To ignore the warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Loan_ID', 'Gender', 'Married', 'Dependents', 'Education',\n",
       "       'Self_Employed', 'ApplicantIncome', 'CoapplicantIncome', 'LoanAmount',\n",
       "       'Loan_Amount_Term', 'Credit_History', 'Property_Area', 'Loan_Status'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let usload and read the data from the csv file \n",
    "\n",
    "df=pd.read_csv(\"train.csv\",encoding = \"ISO-8859-1\")\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Loan_ID</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Married</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>Education</th>\n",
       "      <th>Self_Employed</th>\n",
       "      <th>ApplicantIncome</th>\n",
       "      <th>CoapplicantIncome</th>\n",
       "      <th>LoanAmount</th>\n",
       "      <th>Loan_Amount_Term</th>\n",
       "      <th>Credit_History</th>\n",
       "      <th>Property_Area</th>\n",
       "      <th>Loan_Status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>LP001465</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>6080</td>\n",
       "      <td>2569.0</td>\n",
       "      <td>182.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Rural</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>413</th>\n",
       "      <td>LP002332</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0</td>\n",
       "      <td>Not Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>2253</td>\n",
       "      <td>2033.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Rural</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352</th>\n",
       "      <td>LP002141</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>3+</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>2666</td>\n",
       "      <td>2083.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Rural</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>LP001186</td>\n",
       "      <td>Female</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>Yes</td>\n",
       "      <td>11500</td>\n",
       "      <td>0.0</td>\n",
       "      <td>286.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Loan_ID  Gender Married Dependents     Education Self_Employed  \\\n",
       "129  LP001465    Male     Yes          0      Graduate            No   \n",
       "413  LP002332    Male     Yes          0  Not Graduate            No   \n",
       "352  LP002141    Male     Yes         3+      Graduate            No   \n",
       "54   LP001186  Female     Yes          1      Graduate           Yes   \n",
       "\n",
       "     ApplicantIncome  CoapplicantIncome  LoanAmount  Loan_Amount_Term  \\\n",
       "129             6080             2569.0       182.0             360.0   \n",
       "413             2253             2033.0       110.0             360.0   \n",
       "352             2666             2083.0        95.0             360.0   \n",
       "54             11500                0.0       286.0             360.0   \n",
       "\n",
       "     Credit_History Property_Area Loan_Status  \n",
       "129             NaN         Rural           N  \n",
       "413             1.0         Rural           Y  \n",
       "352             1.0         Rural           Y  \n",
       "54              0.0         Urban           N  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(614, 13)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the instances(rows) and attributes(columns) in the dataset\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 614 entries, 0 to 613\n",
      "Data columns (total 13 columns):\n",
      "Loan_ID              614 non-null object\n",
      "Gender               601 non-null object\n",
      "Married              611 non-null object\n",
      "Dependents           599 non-null object\n",
      "Education            614 non-null object\n",
      "Self_Employed        582 non-null object\n",
      "ApplicantIncome      614 non-null int64\n",
      "CoapplicantIncome    614 non-null float64\n",
      "LoanAmount           592 non-null float64\n",
      "Loan_Amount_Term     600 non-null float64\n",
      "Credit_History       564 non-null float64\n",
      "Property_Area        614 non-null object\n",
      "Loan_Status          614 non-null object\n",
      "dtypes: float64(4), int64(1), object(8)\n",
      "memory usage: 62.4+ KB\n"
     ]
    }
   ],
   "source": [
    "# Checking the Column types\n",
    "\n",
    "ctype = df.dtypes.reset_index()\n",
    "ctype.columns = [\"Count\", \"Column Type\"]\n",
    "ctype.groupby(\"Column Type\").aggregate('count').reset_index()\n",
    "\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ApplicantIncome</th>\n",
       "      <th>CoapplicantIncome</th>\n",
       "      <th>LoanAmount</th>\n",
       "      <th>Loan_Amount_Term</th>\n",
       "      <th>Credit_History</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>614.000000</td>\n",
       "      <td>614.000000</td>\n",
       "      <td>592.000000</td>\n",
       "      <td>600.00000</td>\n",
       "      <td>564.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>5403.459283</td>\n",
       "      <td>1621.245798</td>\n",
       "      <td>146.412162</td>\n",
       "      <td>342.00000</td>\n",
       "      <td>0.842199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>6109.041673</td>\n",
       "      <td>2926.248369</td>\n",
       "      <td>85.587325</td>\n",
       "      <td>65.12041</td>\n",
       "      <td>0.364878</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>150.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>12.00000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2877.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>360.00000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3812.500000</td>\n",
       "      <td>1188.500000</td>\n",
       "      <td>128.000000</td>\n",
       "      <td>360.00000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>5795.000000</td>\n",
       "      <td>2297.250000</td>\n",
       "      <td>168.000000</td>\n",
       "      <td>360.00000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>81000.000000</td>\n",
       "      <td>41667.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>480.00000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       ApplicantIncome  CoapplicantIncome  LoanAmount  Loan_Amount_Term  \\\n",
       "count       614.000000         614.000000  592.000000         600.00000   \n",
       "mean       5403.459283        1621.245798  146.412162         342.00000   \n",
       "std        6109.041673        2926.248369   85.587325          65.12041   \n",
       "min         150.000000           0.000000    9.000000          12.00000   \n",
       "25%        2877.500000           0.000000  100.000000         360.00000   \n",
       "50%        3812.500000        1188.500000  128.000000         360.00000   \n",
       "75%        5795.000000        2297.250000  168.000000         360.00000   \n",
       "max       81000.000000       41667.000000  700.000000         480.00000   \n",
       "\n",
       "       Credit_History  \n",
       "count      564.000000  \n",
       "mean         0.842199  \n",
       "std          0.364878  \n",
       "min          0.000000  \n",
       "25%          1.000000  \n",
       "50%          1.000000  \n",
       "75%          1.000000  \n",
       "max          1.000000  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the  Statistical Numerical Data Distribution Summary\n",
    "\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Loan_ID</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Married</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>Education</th>\n",
       "      <th>Self_Employed</th>\n",
       "      <th>Property_Area</th>\n",
       "      <th>Loan_Status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>614</td>\n",
       "      <td>601</td>\n",
       "      <td>611</td>\n",
       "      <td>599</td>\n",
       "      <td>614</td>\n",
       "      <td>582</td>\n",
       "      <td>614</td>\n",
       "      <td>614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>614</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>LP002137</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>Semiurban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>1</td>\n",
       "      <td>489</td>\n",
       "      <td>398</td>\n",
       "      <td>345</td>\n",
       "      <td>480</td>\n",
       "      <td>500</td>\n",
       "      <td>233</td>\n",
       "      <td>422</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Loan_ID Gender Married Dependents Education Self_Employed  \\\n",
       "count        614    601     611        599       614           582   \n",
       "unique       614      2       2          4         2             2   \n",
       "top     LP002137   Male     Yes          0  Graduate            No   \n",
       "freq           1    489     398        345       480           500   \n",
       "\n",
       "       Property_Area Loan_Status  \n",
       "count            614         614  \n",
       "unique             3           2  \n",
       "top        Semiurban           Y  \n",
       "freq             233         422  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the Categorical Data Distribution\n",
    "\n",
    "df.describe(include=['O'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Column Name</th>\n",
       "      <th>Missing Values Count</th>\n",
       "      <th>Filling Factor (%)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Credit_History</td>\n",
       "      <td>50</td>\n",
       "      <td>91.856678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Self_Employed</td>\n",
       "      <td>32</td>\n",
       "      <td>94.788274</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LoanAmount</td>\n",
       "      <td>22</td>\n",
       "      <td>96.416938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Dependents</td>\n",
       "      <td>15</td>\n",
       "      <td>97.557003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Loan_Amount_Term</td>\n",
       "      <td>14</td>\n",
       "      <td>97.719870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Gender</td>\n",
       "      <td>13</td>\n",
       "      <td>97.882736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Married</td>\n",
       "      <td>3</td>\n",
       "      <td>99.511401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Loan_ID</td>\n",
       "      <td>0</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Education</td>\n",
       "      <td>0</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ApplicantIncome</td>\n",
       "      <td>0</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>CoapplicantIncome</td>\n",
       "      <td>0</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Property_Area</td>\n",
       "      <td>0</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Loan_Status</td>\n",
       "      <td>0</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Column Name  Missing Values Count  Filling Factor (%)\n",
       "0      Credit_History                    50           91.856678\n",
       "1       Self_Employed                    32           94.788274\n",
       "2          LoanAmount                    22           96.416938\n",
       "3          Dependents                    15           97.557003\n",
       "4    Loan_Amount_Term                    14           97.719870\n",
       "5              Gender                    13           97.882736\n",
       "6             Married                     3           99.511401\n",
       "7             Loan_ID                     0          100.000000\n",
       "8           Education                     0          100.000000\n",
       "9     ApplicantIncome                     0          100.000000\n",
       "10  CoapplicantIncome                     0          100.000000\n",
       "11      Property_Area                     0          100.000000\n",
       "12        Loan_Status                     0          100.000000"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Checking the Missing Values in the dataset\n",
    "\n",
    "missing_df =df.isnull().sum(axis=0).reset_index()\n",
    "missing_df.columns = ['Column Name', 'Missing Values Count']\n",
    "missing_df['Filling Factor (%)']=(df.shape[0]-missing_df['Missing Values Count'])/df.shape[0]*100\n",
    "missing_df.sort_values('Filling Factor (%)').reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 480 entries, 1 to 613\n",
      "Data columns (total 13 columns):\n",
      "Loan_ID              480 non-null object\n",
      "Gender               480 non-null object\n",
      "Married              480 non-null object\n",
      "Dependents           480 non-null object\n",
      "Education            480 non-null object\n",
      "Self_Employed        480 non-null object\n",
      "ApplicantIncome      480 non-null int64\n",
      "CoapplicantIncome    480 non-null float64\n",
      "LoanAmount           480 non-null float64\n",
      "Loan_Amount_Term     480 non-null float64\n",
      "Credit_History       480 non-null float64\n",
      "Property_Area        480 non-null object\n",
      "Loan_Status          480 non-null object\n",
      "dtypes: float64(4), int64(1), object(8)\n",
      "memory usage: 52.5+ KB\n"
     ]
    }
   ],
   "source": [
    "# Remove All the rows that contain a Missing Value\n",
    "\n",
    "a=df.dropna()\n",
    "a.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pick Subset of the first 10  rows in the data set\n",
    "\n",
    "\n",
    "subset_a=df.iloc[:10:]\n",
    "subset_a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Column Name</th>\n",
       "      <th>Missing Values Count</th>\n",
       "      <th>Filling Factor (%)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Loan_ID</td>\n",
       "      <td>0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Gender</td>\n",
       "      <td>0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Married</td>\n",
       "      <td>0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Dependents</td>\n",
       "      <td>0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Education</td>\n",
       "      <td>0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Self_Employed</td>\n",
       "      <td>0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ApplicantIncome</td>\n",
       "      <td>0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>CoapplicantIncome</td>\n",
       "      <td>0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>LoanAmount</td>\n",
       "      <td>0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Loan_Amount_Term</td>\n",
       "      <td>0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Credit_History</td>\n",
       "      <td>0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Property_Area</td>\n",
       "      <td>0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Loan_Status</td>\n",
       "      <td>0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Column Name  Missing Values Count  Filling Factor (%)\n",
       "0             Loan_ID                     0               100.0\n",
       "1              Gender                     0               100.0\n",
       "2             Married                     0               100.0\n",
       "3          Dependents                     0               100.0\n",
       "4           Education                     0               100.0\n",
       "5       Self_Employed                     0               100.0\n",
       "6     ApplicantIncome                     0               100.0\n",
       "7   CoapplicantIncome                     0               100.0\n",
       "8          LoanAmount                     0               100.0\n",
       "9    Loan_Amount_Term                     0               100.0\n",
       "10     Credit_History                     0               100.0\n",
       "11      Property_Area                     0               100.0\n",
       "12        Loan_Status                     0               100.0"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check any missing value after removing the missing values\n",
    "\n",
    "missing_a =a.isnull().sum(axis=0).reset_index()\n",
    "missing_a.columns = ['Column Name', 'Missing Values Count']\n",
    "missing_a['Filling Factor (%)']=(a.shape[0]-missing_a['Missing Values Count'])/a.shape[0]*100\n",
    "missing_a.sort_values('Filling Factor (%)').reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'column_names' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-9d943fbeb45d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# Also check if the column is unique\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mcolumn_names\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m   \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'{} is unique: {}'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_unique\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'column_names' is not defined"
     ]
    }
   ],
   "source": [
    "# Also check if the column is unique\n",
    "for i in column_names:\n",
    "  print('{} is unique: {}'.format(i, df[i].is_unique))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check the index values\n",
    "df.index.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check if a certain index exists\n",
    "'foo' in df.index.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If index does not exist\n",
    "df.set_index('column_name_to_use', inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Good. Our data has been quickly checked, we know the data types, if columns are unique, and we know \n",
    "\n",
    "it has an index so we can do joins and merges later on. Let’s figure out which columns you want to \n",
    "\n",
    "keep or remove. In this example, we want to get rid of the columns in indexes 1, 3, and 5, so I’ve \n",
    "\n",
    "just added the string values to a list, which will be used to drop the columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create list comprehension function of the columns you want to lose\n",
    "columns_to_drop = [column_names[i] for i in [1, 3, 5]]\n",
    "# Drop unwanted columns\n",
    "df.drop(columns_to_drop, inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Loan_ID</th>\n",
       "      <th>Education</th>\n",
       "      <th>ApplicantIncome</th>\n",
       "      <th>CoapplicantIncome</th>\n",
       "      <th>Property_Area</th>\n",
       "      <th>Loan_Status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LP001002</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>5849</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LP001003</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>4583</td>\n",
       "      <td>1508.0</td>\n",
       "      <td>Rural</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LP001005</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>3000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LP001006</td>\n",
       "      <td>Not Graduate</td>\n",
       "      <td>2583</td>\n",
       "      <td>2358.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LP001008</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>6000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Loan_ID     Education  ApplicantIncome  CoapplicantIncome Property_Area  \\\n",
       "0  LP001002      Graduate             5849                0.0         Urban   \n",
       "1  LP001003      Graduate             4583             1508.0         Rural   \n",
       "2  LP001005      Graduate             3000                0.0         Urban   \n",
       "3  LP001006  Not Graduate             2583             2358.0         Urban   \n",
       "4  LP001008      Graduate             6000                0.0         Urban   \n",
       "\n",
       "  Loan_Status  \n",
       "0           Y  \n",
       "1           N  \n",
       "2           Y  \n",
       "3           Y  \n",
       "4           Y  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove All Columns with at least one missing value\n",
    "\n",
    "columns_with_na_dropped = df.dropna(axis=1)\n",
    "columns_with_na_dropped.head()\n",
    "\n",
    "# Remove date_first_booking column\n",
    "# df_all.drop('date_first_booking', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Columns in original dataset: 13 \n",
      "\n",
      "Columns with na's dropped: 6\n"
     ]
    }
   ],
   "source": [
    "# just how much data did we lose?\n",
    "\n",
    "print(\"Columns in original dataset: %d \\n\" % df.shape[1])\n",
    "print(\"Columns with na's dropped: %d\" % columns_with_na_dropped.shape[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We've lost 7 columns of data, but at this point we have successfully removed all the NaN's from our data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Loan_ID</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Married</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>Education</th>\n",
       "      <th>Self_Employed</th>\n",
       "      <th>ApplicantIncome</th>\n",
       "      <th>CoapplicantIncome</th>\n",
       "      <th>LoanAmount</th>\n",
       "      <th>Loan_Amount_Term</th>\n",
       "      <th>Credit_History</th>\n",
       "      <th>Property_Area</th>\n",
       "      <th>Loan_Status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LP001003</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>4583</td>\n",
       "      <td>1508.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Rural</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LP001005</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>Yes</td>\n",
       "      <td>3000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LP001006</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0</td>\n",
       "      <td>Not Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>2583</td>\n",
       "      <td>2358.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LP001008</td>\n",
       "      <td>Male</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>6000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Loan_ID Gender Married Dependents     Education Self_Employed  \\\n",
       "1  LP001003   Male     Yes          1      Graduate            No   \n",
       "2  LP001005   Male     Yes          0      Graduate           Yes   \n",
       "3  LP001006   Male     Yes          0  Not Graduate            No   \n",
       "4  LP001008   Male      No          0      Graduate            No   \n",
       "\n",
       "   ApplicantIncome  CoapplicantIncome  LoanAmount  Loan_Amount_Term  \\\n",
       "1             4583             1508.0       128.0             360.0   \n",
       "2             3000                0.0        66.0             360.0   \n",
       "3             2583             2358.0       120.0             360.0   \n",
       "4             6000                0.0       141.0             360.0   \n",
       "\n",
       "   Credit_History Property_Area Loan_Status  \n",
       "1             1.0         Rural           N  \n",
       "2             1.0         Urban           Y  \n",
       "3             1.0         Urban           Y  \n",
       "4             1.0         Urban           Y  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.head(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will now fill the missing values automatically. For this bit, we will get a  small sub-section of the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Loan_ID</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Married</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>Education</th>\n",
       "      <th>Self_Employed</th>\n",
       "      <th>ApplicantIncome</th>\n",
       "      <th>CoapplicantIncome</th>\n",
       "      <th>LoanAmount</th>\n",
       "      <th>Loan_Amount_Term</th>\n",
       "      <th>Credit_History</th>\n",
       "      <th>Property_Area</th>\n",
       "      <th>Loan_Status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LP001002</td>\n",
       "      <td>Male</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>5849</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LP001003</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>4583</td>\n",
       "      <td>1508.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Rural</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LP001005</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>Yes</td>\n",
       "      <td>3000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LP001006</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0</td>\n",
       "      <td>Not Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>2583</td>\n",
       "      <td>2358.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LP001008</td>\n",
       "      <td>Male</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>6000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Loan_ID Gender Married Dependents     Education Self_Employed  \\\n",
       "0  LP001002   Male      No          0      Graduate            No   \n",
       "1  LP001003   Male     Yes          1      Graduate            No   \n",
       "2  LP001005   Male     Yes          0      Graduate           Yes   \n",
       "3  LP001006   Male     Yes          0  Not Graduate            No   \n",
       "4  LP001008   Male      No          0      Graduate            No   \n",
       "\n",
       "   ApplicantIncome  CoapplicantIncome  LoanAmount  Loan_Amount_Term  \\\n",
       "0             5849                0.0         NaN             360.0   \n",
       "1             4583             1508.0       128.0             360.0   \n",
       "2             3000                0.0        66.0             360.0   \n",
       "3             2583             2358.0       120.0             360.0   \n",
       "4             6000                0.0       141.0             360.0   \n",
       "\n",
       "   Credit_History Property_Area Loan_Status  \n",
       "0             1.0         Urban           Y  \n",
       "1             1.0         Rural           N  \n",
       "2             1.0         Urban           Y  \n",
       "3             1.0         Urban           Y  \n",
       "4             1.0         Urban           Y  "
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get a small subset of the Dataset\n",
    "\n",
    "subset_df = df.loc[:20].head()\n",
    "subset_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Loan_ID</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Married</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>Education</th>\n",
       "      <th>Self_Employed</th>\n",
       "      <th>ApplicantIncome</th>\n",
       "      <th>CoapplicantIncome</th>\n",
       "      <th>LoanAmount</th>\n",
       "      <th>Loan_Amount_Term</th>\n",
       "      <th>Credit_History</th>\n",
       "      <th>Property_Area</th>\n",
       "      <th>Loan_Status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LP001002</td>\n",
       "      <td>Male</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>5849</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LP001003</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>4583</td>\n",
       "      <td>1508.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Rural</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LP001005</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>Yes</td>\n",
       "      <td>3000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LP001006</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0</td>\n",
       "      <td>Not Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>2583</td>\n",
       "      <td>2358.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LP001008</td>\n",
       "      <td>Male</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>6000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Loan_ID Gender Married Dependents     Education Self_Employed  \\\n",
       "0  LP001002   Male      No          0      Graduate            No   \n",
       "1  LP001003   Male     Yes          1      Graduate            No   \n",
       "2  LP001005   Male     Yes          0      Graduate           Yes   \n",
       "3  LP001006   Male     Yes          0  Not Graduate            No   \n",
       "4  LP001008   Male      No          0      Graduate            No   \n",
       "\n",
       "   ApplicantIncome  CoapplicantIncome  LoanAmount  Loan_Amount_Term  \\\n",
       "0             5849                0.0         0.0             360.0   \n",
       "1             4583             1508.0       128.0             360.0   \n",
       "2             3000                0.0        66.0             360.0   \n",
       "3             2583             2358.0       120.0             360.0   \n",
       "4             6000                0.0       141.0             360.0   \n",
       "\n",
       "   Credit_History Property_Area Loan_Status  \n",
       "0             1.0         Urban           Y  \n",
       "1             1.0         Rural           N  \n",
       "2             1.0         Urban           Y  \n",
       "3             1.0         Urban           Y  \n",
       "4             1.0         Urban           Y  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Replace all NA's with 0\n",
    "# Replacing and filling the Missing Values \n",
    "\n",
    "subset_df.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Loan_ID</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Married</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>Education</th>\n",
       "      <th>Self_Employed</th>\n",
       "      <th>ApplicantIncome</th>\n",
       "      <th>CoapplicantIncome</th>\n",
       "      <th>LoanAmount</th>\n",
       "      <th>Loan_Amount_Term</th>\n",
       "      <th>Credit_History</th>\n",
       "      <th>Property_Area</th>\n",
       "      <th>Loan_Status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LP001002</td>\n",
       "      <td>Male</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>5849</td>\n",
       "      <td>0.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>LP001003</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>4583</td>\n",
       "      <td>1508.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Rural</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LP001005</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>Yes</td>\n",
       "      <td>3000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LP001006</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0</td>\n",
       "      <td>Not Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>2583</td>\n",
       "      <td>2358.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LP001008</td>\n",
       "      <td>Male</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>6000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Urban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Loan_ID Gender Married Dependents     Education Self_Employed  \\\n",
       "0  LP001002   Male      No          0      Graduate            No   \n",
       "1  LP001003   Male     Yes          1      Graduate            No   \n",
       "2  LP001005   Male     Yes          0      Graduate           Yes   \n",
       "3  LP001006   Male     Yes          0  Not Graduate            No   \n",
       "4  LP001008   Male      No          0      Graduate            No   \n",
       "\n",
       "   ApplicantIncome  CoapplicantIncome  LoanAmount  Loan_Amount_Term  \\\n",
       "0             5849                0.0       128.0             360.0   \n",
       "1             4583             1508.0       128.0             360.0   \n",
       "2             3000                0.0        66.0             360.0   \n",
       "3             2583             2358.0       120.0             360.0   \n",
       "4             6000                0.0       141.0             360.0   \n",
       "\n",
       "   Credit_History Property_Area Loan_Status  \n",
       "0             1.0         Urban           Y  \n",
       "1             1.0         Rural           N  \n",
       "2             1.0         Urban           Y  \n",
       "3             1.0         Urban           Y  \n",
       "4             1.0         Urban           Y  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Replace all NA's the value that comes directly after it in the same column, \n",
    "# Then replace all the remaining na's with 0\n",
    "\n",
    "subset_df.fillna(method = 'bfill', axis=0).fillna(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Loan_ID</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Married</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>Education</th>\n",
       "      <th>Self_Employed</th>\n",
       "      <th>ApplicantIncome</th>\n",
       "      <th>CoapplicantIncome</th>\n",
       "      <th>LoanAmount</th>\n",
       "      <th>Loan_Amount_Term</th>\n",
       "      <th>Credit_History</th>\n",
       "      <th>Property_Area</th>\n",
       "      <th>Loan_Status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>210</th>\n",
       "      <td>LP001708</td>\n",
       "      <td>Female</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>10000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>214.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Semiurban</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>LP001047</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0</td>\n",
       "      <td>Not Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>2600</td>\n",
       "      <td>1911.0</td>\n",
       "      <td>116.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Semiurban</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>318</th>\n",
       "      <td>LP002043</td>\n",
       "      <td>Female</td>\n",
       "      <td>No</td>\n",
       "      <td>1</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>No</td>\n",
       "      <td>3541</td>\n",
       "      <td>0.0</td>\n",
       "      <td>112.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Semiurban</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Loan_ID  Gender Married Dependents     Education Self_Employed  \\\n",
       "210  LP001708  Female      No          0      Graduate            No   \n",
       "22   LP001047    Male     Yes          0  Not Graduate            No   \n",
       "318  LP002043  Female      No          1      Graduate            No   \n",
       "\n",
       "     ApplicantIncome  CoapplicantIncome  LoanAmount  Loan_Amount_Term  \\\n",
       "210            10000                0.0       214.0             360.0   \n",
       "22              2600             1911.0       116.0             360.0   \n",
       "318             3541                0.0       112.0             360.0   \n",
       "\n",
       "     Credit_History Property_Area Loan_Status  \n",
       "210             1.0     Semiurban           N  \n",
       "22              0.0     Semiurban           N  \n",
       "318             0.0     Semiurban           Y  "
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fill=df.fillna(0)\n",
    "fill.sample(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Loan_ID              0\n",
       "Gender               0\n",
       "Married              0\n",
       "Dependents           0\n",
       "Education            0\n",
       "Self_Employed        0\n",
       "ApplicantIncome      0\n",
       "CoapplicantIncome    0\n",
       "LoanAmount           0\n",
       "Loan_Amount_Term     0\n",
       "Credit_History       0\n",
       "Property_Area        0\n",
       "Loan_Status          0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f=fill.isnull().sum()\n",
    "f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  Dropping Columns in a DataFrame\n",
    "# Categories of data in a dataset not useful to you.\n",
    "\n",
    "to_drop = [\"Property_Area\", \"Self_Employed\"]\n",
    "df.drop(to_drop, inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Loan_ID</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Married</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>Education</th>\n",
       "      <th>ApplicantIncome</th>\n",
       "      <th>LoanAmount</th>\n",
       "      <th>Credit_History</th>\n",
       "      <th>Loan_Status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>325</th>\n",
       "      <td>LP002067</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>8666</td>\n",
       "      <td>376.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212</th>\n",
       "      <td>LP001713</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>7787</td>\n",
       "      <td>240.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>LP001531</td>\n",
       "      <td>Male</td>\n",
       "      <td>No</td>\n",
       "      <td>0</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>9166</td>\n",
       "      <td>244.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>585</th>\n",
       "      <td>LP002912</td>\n",
       "      <td>Male</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1</td>\n",
       "      <td>Graduate</td>\n",
       "      <td>4283</td>\n",
       "      <td>172.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Loan_ID Gender Married Dependents Education  ApplicantIncome  \\\n",
       "325  LP002067   Male     Yes          1  Graduate             8666   \n",
       "212  LP001713   Male     Yes          1  Graduate             7787   \n",
       "152  LP001531   Male      No          0  Graduate             9166   \n",
       "585  LP002912   Male     Yes          1  Graduate             4283   \n",
       "\n",
       "     LoanAmount  Credit_History Loan_Status  \n",
       "325       376.0             0.0           N  \n",
       "212       240.0             1.0           Y  \n",
       "152       244.0             1.0           N  \n",
       "585       172.0             1.0           N  "
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>ApplicantIncome</th>\n",
       "      <th>LoanAmount</th>\n",
       "      <th>Credit_History</th>\n",
       "      <th>Loan_Status</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Education</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Graduate</th>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>5849</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Graduate</th>\n",
       "      <td>Male</td>\n",
       "      <td>1</td>\n",
       "      <td>4583</td>\n",
       "      <td>128.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Graduate</th>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>3000</td>\n",
       "      <td>66.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Not Graduate</th>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>2583</td>\n",
       "      <td>120.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Graduate</th>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>6000</td>\n",
       "      <td>141.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Gender Dependents  ApplicantIncome  LoanAmount  Credit_History  \\\n",
       "Education                                                                     \n",
       "Graduate       Male          0             5849         NaN             1.0   \n",
       "Graduate       Male          1             4583       128.0             1.0   \n",
       "Graduate       Male          0             3000        66.0             1.0   \n",
       "Not Graduate   Male          0             2583       120.0             1.0   \n",
       "Graduate       Male          0             6000       141.0             1.0   \n",
       "\n",
       "             Loan_Status  \n",
       "Education                 \n",
       "Graduate               Y  \n",
       "Graduate               N  \n",
       "Graduate               Y  \n",
       "Not Graduate           Y  \n",
       "Graduate               Y  "
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " # Changing the Index of a DataFrame\n",
    "    \n",
    "df['Education'].is_unique\n",
    " df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dependents</th>\n",
       "      <th>ApplicantIncome</th>\n",
       "      <th>LoanAmount</th>\n",
       "      <th>Credit_History</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Loan_Status</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Y</th>\n",
       "      <td>0</td>\n",
       "      <td>5849</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>N</th>\n",
       "      <td>1</td>\n",
       "      <td>4583</td>\n",
       "      <td>128.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y</th>\n",
       "      <td>0</td>\n",
       "      <td>3000</td>\n",
       "      <td>66.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y</th>\n",
       "      <td>0</td>\n",
       "      <td>2583</td>\n",
       "      <td>120.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y</th>\n",
       "      <td>0</td>\n",
       "      <td>6000</td>\n",
       "      <td>141.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Dependents  ApplicantIncome  LoanAmount  Credit_History\n",
       "Loan_Status                                                        \n",
       "Y                    0             5849         NaN             1.0\n",
       "N                    1             4583       128.0             1.0\n",
       "Y                    0             3000        66.0             1.0\n",
       "Y                    0             2583       120.0             1.0\n",
       "Y                    0             6000       141.0             1.0"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Replacing the existing Index Loan ID  with Gender Column as the set index\n",
    "\n",
    "df = df.set_index('Loan_Status')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dependents</th>\n",
       "      <th>ApplicantIncome</th>\n",
       "      <th>LoanAmount</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Credit_History</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0</td>\n",
       "      <td>3993</td>\n",
       "      <td>207.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Dependents  ApplicantIncome  LoanAmount\n",
       "Credit_History                                        \n",
       "1.0                     0             3993       207.0"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# restoring the index values to normal\n",
    "\n",
    "df=df.set_index(\"Credit_History\")\n",
    "df.sample()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 614 entries, 0 to 613\n",
      "Data columns (total 13 columns):\n",
      "Loan_ID              614 non-null object\n",
      "Gender               601 non-null object\n",
      "Married              611 non-null object\n",
      "Dependents           599 non-null object\n",
      "Education            614 non-null object\n",
      "Self_Employed        582 non-null object\n",
      "ApplicantIncome      614 non-null int32\n",
      "CoapplicantIncome    614 non-null float64\n",
      "LoanAmount           592 non-null float64\n",
      "Loan_Amount_Term     600 non-null float64\n",
      "Credit_History       564 non-null float64\n",
      "Property_Area        614 non-null object\n",
      "Loan_Status          614 non-null object\n",
      "dtypes: float64(4), int32(1), object(8)\n",
      "memory usage: 60.0+ KB\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 614 entries, 0 to 613\n",
      "Data columns (total 13 columns):\n",
      "Loan_ID              614 non-null object\n",
      "Gender               601 non-null object\n",
      "Married              611 non-null object\n",
      "Dependents           599 non-null object\n",
      "Education            614 non-null object\n",
      "Self_Employed        582 non-null object\n",
      "ApplicantIncome      614 non-null int64\n",
      "CoapplicantIncome    614 non-null float64\n",
      "LoanAmount           592 non-null float64\n",
      "Loan_Amount_Term     600 non-null float64\n",
      "Credit_History       564 non-null float64\n",
      "Property_Area        614 non-null object\n",
      "Loan_Status          614 non-null object\n",
      "dtypes: float64(4), int64(1), object(8)\n",
      "memory usage: 62.4+ KB\n"
     ]
    }
   ],
   "source": [
    " # Normalizing Data Types in a data set\n",
    "    \n",
    "# When reading data in csv with a bunch of numbers, some of the numbers will read in as strings (str) instead of numeric values\n",
    "# Fix and normalize the data types\n",
    "\n",
    "z = pd.read_csv(\"train.csv\", dtype={\"ApplicantIncome\": int})\n",
    "z.info()\n",
    "\n",
    "# This tells Pandas that Column Applicant Income, needs to be an integer value\n",
    "\n",
    "y = pd.read_csv(\"train.csv\", dtype={\"Married\": str})\n",
    "y.info()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 614 entries, 0 to 613\n",
      "Data columns (total 13 columns):\n",
      "Loan_ID              614 non-null object\n",
      "Gender               601 non-null object\n",
      "Married              611 non-null object\n",
      "Dependents           599 non-null object\n",
      "Education            614 non-null object\n",
      "Self_Employed        582 non-null object\n",
      "ApplicantIncome      614 non-null int64\n",
      "CoapplicantIncome    614 non-null float64\n",
      "LoanAmount           592 non-null float64\n",
      "Loan_Amount_Term     600 non-null float64\n",
      "Credit_History       564 non-null float64\n",
      "Property_Area        614 non-null object\n",
      "Loan_Status          614 non-null object\n",
      "dtypes: float64(4), int64(1), object(8)\n",
      "memory usage: 62.4+ KB\n"
     ]
    }
   ],
   "source": [
    "# Similarly, if we want the Gender column to be a string and not a number, we can do the same kind of thing:\n",
    "\n",
    "df = pd.read_csv(\"train.csv\", dtype={\"Gender\": str})\n",
    "df.info()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    MALE\n",
       "1    MALE\n",
       "2    MALE\n",
       "3    MALE\n",
       "4    MALE\n",
       "5    MALE\n",
       "6    MALE\n",
       "7    MALE\n",
       "8    MALE\n",
       "9    MALE\n",
       "Name: Gender, dtype: object"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Change Casing in  Columns\n",
    "# Columns with user-provided data are ripe for corruption. People make typos, leave their caps lock on (or off), and add extra spaces where they shouldn’t.\n",
    "# To change all our Gender to uppercase:\n",
    " \n",
    "a=df[\"Gender\"].str.upper()\n",
    "a.head(10)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  Similarly, to get rid of trailing whitespace:\n",
    "\n",
    "\n",
    "c=df[\"Gender\"].str.strip()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 614 entries, 0 to 613\n",
      "Data columns (total 13 columns):\n",
      "Loan_ID               614 non-null object\n",
      "Sex                   601 non-null object\n",
      "Married               611 non-null object\n",
      "Dependents            599 non-null object\n",
      "Education             614 non-null object\n",
      "Self_Employed         582 non-null object\n",
      "Applicant's_Amount    614 non-null int64\n",
      "CoapplicantIncome     614 non-null float64\n",
      "LoanAmount            592 non-null float64\n",
      "Loan_Amount_Term      600 non-null float64\n",
      "Credit_History        564 non-null float64\n",
      "Property_Area         614 non-null object\n",
      "Loan_Status           614 non-null object\n",
      "dtypes: float64(4), int64(1), object(8)\n",
      "memory usage: 62.4+ KB\n"
     ]
    }
   ],
   "source": [
    "# Renaming Columns in a data set\n",
    "\n",
    "f=df.rename(columns = {\"Gender\":\"Sex\", \"ApplicantIncome\":\"Applicant's_Amount\"})\n",
    "f.info()\n",
    "\n",
    "# Here we’ve renamed ‘Gender’ to ‘Sex’ and ‘ApplicantIncome’ to simply ‘Applicant's_Amount’.\n",
    "\n",
    "# Since this is not an in-place operation, you’ll need to save the DataFrame by assigning it to a variable.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Loan_ID', 'Sex', 'Married', 'Dependents', 'Education', 'Self_Employed',\n",
       "       'Applicant's_Amount', 'CoapplicantIncome', 'LoanAmount',\n",
       "       'Loan_Amount_Term', 'Credit_History', 'Property_Area', 'Loan_Status'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save your results after you are done cleaning your data.\n",
    "# Export it back into CSV Format for further processing in another program\n",
    "\n",
    "f.to_csv(\"cleanfile.csv\", encoding = \"ISO-8859-1\")\n",
    "f.columns\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dealing with inconsistent data "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we use dataset from Kaggle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get all the unique values in the 'City' column\n",
    "\n",
    "City= df[\"City\"].unique()\n",
    "City\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sort the City column rows alphabetically and then take a closer look\n",
    "\n",
    "City.sort()\n",
    "City"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert the rows in the City Column to lower case from the dataset\n",
    "# Set the values in the dataset df and column city to lower case\n",
    "\n",
    "\n",
    "df[\"City\"]= df['City'].str.lower()\n",
    "df[\"City\"].sample(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove the trailing white spaces in the City Column\n",
    "\n",
    "df[\"City\"] =df['City'].str.strip()\n",
    "df[\"City\"].sample(4)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get all the unique values in the 'City' column\n",
    "\n",
    "City = df['City'].unique()\n",
    "City"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sort the City 2 Alphabetically and then take a closer look\n",
    "\n",
    "City.sort()\n",
    "City"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the top 10 closest matches to \"d.i khan\"\n",
    "\n",
    "Matches = fuzzywuzzy.process.extract(\"d.i khan\", Cities, limit=10, scorer=fuzzywuzzy.fuzz.token_sort_ratio)\n",
    "Matches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to replace rows in the provided column of the provided dataframe\n",
    "# that match the provided string above the provided ratio with the provided string\n",
    "def replace_matches_in_column(df, column, string_to_match, min_ratio = 90):\n",
    "    # get a list of unique strings\n",
    "    strings = df[column].unique()\n",
    "    \n",
    "    # get the top 10 closest matches to our input string\n",
    "    matches = fuzzywuzzy.process.extract(string_to_match, strings, \n",
    "                                         limit=10, scorer=fuzzywuzzy.fuzz.token_sort_ratio)\n",
    "\n",
    "    # only get matches with a ratio > 90\n",
    "    close_matches = [match[0] for match in matches if match[1] >= min_ratio]\n",
    "\n",
    "    # get the rows of all the close matches in our dataframe\n",
    "    rows_with_matches = df[column].isin(close_matches)\n",
    "\n",
    "    # replace all rows with close matches with the input matches \n",
    "    df.loc[rows_with_matches, column] = string_to_match\n",
    "    \n",
    "    # let us know the function's done\n",
    "    print(\"All done!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use the function we just wrote to replace close matches to \"d.i khan\" with \"d.i khan\"\n",
    "replace_matches_in_column(df=df, column='City', string_to_match=\"d.i khan\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And now let's can check the unique values in our City column again and make sure we've tidied up d.i khan correctly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get all the unique values in the 'City' column\n",
    "\n",
    "City = df['City'].unique()\n",
    "\n",
    "# sort them alphabetically and then take a closer look\n",
    "\n",
    "City.sort()\n",
    "City"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Excellent! Now we only have \"d.i khan\" in our dataframe and we didn't have to change anything by hand"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Important links:\n",
    "\n",
    "https://medium.com/@rrfd/cleaning-and-prepping-data-with-python-for-data-science-best-practices-and-helpful-packages-af1edfbe2a3\n",
    "    \n",
    "https://www.kaggle.com/pmarcelino/comprehensive-data-exploration-with-python\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}