RengaMain.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import math\n",
    "import numpy as np\n",
    "import datetime as dt\n",
    "import time\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Renganathan\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:2785: DtypeWarning: Columns (19) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "data = pd.read_csv('Data/TrainingData.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>application_key</th>\n",
       "      <th>mvar1</th>\n",
       "      <th>mvar2</th>\n",
       "      <th>mvar3</th>\n",
       "      <th>mvar4</th>\n",
       "      <th>mvar5</th>\n",
       "      <th>mvar6</th>\n",
       "      <th>mvar7</th>\n",
       "      <th>mvar8</th>\n",
       "      <th>mvar9</th>\n",
       "      <th>...</th>\n",
       "      <th>mvar39</th>\n",
       "      <th>mvar40</th>\n",
       "      <th>mvar41</th>\n",
       "      <th>mvar42</th>\n",
       "      <th>mvar43</th>\n",
       "      <th>mvar44</th>\n",
       "      <th>mvar45</th>\n",
       "      <th>mvar46</th>\n",
       "      <th>mvar48</th>\n",
       "      <th>default_ind</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>83000.000000</td>\n",
       "      <td>79267.000000</td>\n",
       "      <td>77114.000000</td>\n",
       "      <td>82465.000000</td>\n",
       "      <td>82465.000000</td>\n",
       "      <td>82465.000000</td>\n",
       "      <td>63299.000000</td>\n",
       "      <td>7.532600e+04</td>\n",
       "      <td>63291.000000</td>\n",
       "      <td>71318.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>76671.000000</td>\n",
       "      <td>17930.000000</td>\n",
       "      <td>25736.000000</td>\n",
       "      <td>80977.000000</td>\n",
       "      <td>82111.000000</td>\n",
       "      <td>74851.000000</td>\n",
       "      <td>37080.000000</td>\n",
       "      <td>59397.000000</td>\n",
       "      <td>83000.000000</td>\n",
       "      <td>83000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>318146.931060</td>\n",
       "      <td>1747.511865</td>\n",
       "      <td>1.054816</td>\n",
       "      <td>5.401784</td>\n",
       "      <td>0.461151</td>\n",
       "      <td>1.084012</td>\n",
       "      <td>1633.429280</td>\n",
       "      <td>1.750797e+04</td>\n",
       "      <td>6822.332227</td>\n",
       "      <td>34030.596329</td>\n",
       "      <td>...</td>\n",
       "      <td>0.087673</td>\n",
       "      <td>97.851158</td>\n",
       "      <td>72.016570</td>\n",
       "      <td>0.331550</td>\n",
       "      <td>6.553604</td>\n",
       "      <td>0.706595</td>\n",
       "      <td>0.070523</td>\n",
       "      <td>0.093187</td>\n",
       "      <td>556.021277</td>\n",
       "      <td>0.287410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>97179.489955</td>\n",
       "      <td>94.830127</td>\n",
       "      <td>1.556682</td>\n",
       "      <td>11.091569</td>\n",
       "      <td>1.704292</td>\n",
       "      <td>5.743899</td>\n",
       "      <td>3667.183981</td>\n",
       "      <td>4.677222e+04</td>\n",
       "      <td>10060.346814</td>\n",
       "      <td>50673.150005</td>\n",
       "      <td>...</td>\n",
       "      <td>0.423403</td>\n",
       "      <td>29.530657</td>\n",
       "      <td>24.508469</td>\n",
       "      <td>0.373973</td>\n",
       "      <td>6.273107</td>\n",
       "      <td>0.245915</td>\n",
       "      <td>0.290383</td>\n",
       "      <td>0.329054</td>\n",
       "      <td>1106.953295</td>\n",
       "      <td>0.452557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>230032.000000</td>\n",
       "      <td>1477.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.014000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.060660</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>250781.750000</td>\n",
       "      <td>1680.000000</td>\n",
       "      <td>0.131800</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>41.000000</td>\n",
       "      <td>1.750250e+03</td>\n",
       "      <td>496.000000</td>\n",
       "      <td>3542.250000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>90.149500</td>\n",
       "      <td>58.174750</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.515280</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>271531.500000</td>\n",
       "      <td>1743.000000</td>\n",
       "      <td>0.513000</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>297.000000</td>\n",
       "      <td>7.020500e+03</td>\n",
       "      <td>2507.000000</td>\n",
       "      <td>14389.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>78.839500</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>0.739500</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>362296.250000</td>\n",
       "      <td>1813.000000</td>\n",
       "      <td>1.386200</td>\n",
       "      <td>6.595000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1381.000000</td>\n",
       "      <td>1.811300e+04</td>\n",
       "      <td>9869.500000</td>\n",
       "      <td>44413.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>111.327000</td>\n",
       "      <td>91.596750</td>\n",
       "      <td>0.571430</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>0.945120</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>111.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>578068.000000</td>\n",
       "      <td>1950.000000</td>\n",
       "      <td>31.018100</td>\n",
       "      <td>399.334000</td>\n",
       "      <td>25.754000</td>\n",
       "      <td>165.492000</td>\n",
       "      <td>94302.000000</td>\n",
       "      <td>5.637108e+06</td>\n",
       "      <td>291810.000000</td>\n",
       "      <td>840658.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>631.360000</td>\n",
       "      <td>182.111000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>91.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>3247.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 49 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       application_key         mvar1         mvar2         mvar3  \\\n",
       "count     83000.000000  79267.000000  77114.000000  82465.000000   \n",
       "mean     318146.931060   1747.511865      1.054816      5.401784   \n",
       "std       97179.489955     94.830127      1.556682     11.091569   \n",
       "min      230032.000000   1477.000000      0.000000      0.000000   \n",
       "25%      250781.750000   1680.000000      0.131800      0.000000   \n",
       "50%      271531.500000   1743.000000      0.513000      0.300000   \n",
       "75%      362296.250000   1813.000000      1.386200      6.595000   \n",
       "max      578068.000000   1950.000000     31.018100    399.334000   \n",
       "\n",
       "              mvar4         mvar5         mvar6         mvar7          mvar8  \\\n",
       "count  82465.000000  82465.000000  63299.000000  7.532600e+04   63291.000000   \n",
       "mean       0.461151      1.084012   1633.429280  1.750797e+04    6822.332227   \n",
       "std        1.704292      5.743899   3667.183981  4.677222e+04   10060.346814   \n",
       "min        0.000000      0.000000      0.000000  0.000000e+00       0.000000   \n",
       "25%        0.000000      0.000000     41.000000  1.750250e+03     496.000000   \n",
       "50%        0.000000      0.000000    297.000000  7.020500e+03    2507.000000   \n",
       "75%        0.000000      0.000000   1381.000000  1.811300e+04    9869.500000   \n",
       "max       25.754000    165.492000  94302.000000  5.637108e+06  291810.000000   \n",
       "\n",
       "               mvar9      ...             mvar39        mvar40        mvar41  \\\n",
       "count   71318.000000      ...       76671.000000  17930.000000  25736.000000   \n",
       "mean    34030.596329      ...           0.087673     97.851158     72.016570   \n",
       "std     50673.150005      ...           0.423403     29.530657     24.508469   \n",
       "min         0.000000      ...           0.000000      0.014000      0.000000   \n",
       "25%      3542.250000      ...           0.000000     90.149500     58.174750   \n",
       "50%     14389.000000      ...           0.000000    100.000000     78.839500   \n",
       "75%     44413.000000      ...           0.000000    111.327000     91.596750   \n",
       "max    840658.000000      ...          21.000000    631.360000    182.111000   \n",
       "\n",
       "             mvar42        mvar43        mvar44        mvar45        mvar46  \\\n",
       "count  80977.000000  82111.000000  74851.000000  37080.000000  59397.000000   \n",
       "mean       0.331550      6.553604      0.706595      0.070523      0.093187   \n",
       "std        0.373973      6.273107      0.245915      0.290383      0.329054   \n",
       "min        0.000000      0.000000      0.060660      0.000000      0.000000   \n",
       "25%        0.000000      2.000000      0.515280      0.000000      0.000000   \n",
       "50%        0.200000      5.000000      0.739500      0.000000      0.000000   \n",
       "75%        0.571430     10.000000      0.945120      0.000000      0.000000   \n",
       "max        2.000000     91.000000      1.000000      7.000000      5.000000   \n",
       "\n",
       "             mvar48   default_ind  \n",
       "count  83000.000000  83000.000000  \n",
       "mean     556.021277      0.287410  \n",
       "std     1106.953295      0.452557  \n",
       "min        2.000000      0.000000  \n",
       "25%        8.000000      0.000000  \n",
       "50%       17.000000      0.000000  \n",
       "75%      111.000000      1.000000  \n",
       "max     3247.000000      1.000000  \n",
       "\n",
       "[8 rows x 49 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.replace('missing', np.nan, inplace=True)\n",
    "data.replace('na', np.nan, inplace=True)\n",
    "int_cols = list(set(data.columns) - {'mvar47'})\n",
    "data[int_cols] = data[int_cols].astype(float)\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = data['default_ind']\n",
    "X_cols = list(set(data.columns) - {'default_ind'})\n",
    "X = data[X_cols]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}