Init.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "###### import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data fields\n",
    "\n",
    "Most of the fields are self-explanatory. The following are descriptions for those that aren't.\n",
    "\n",
    "- Id - an Id that represents a (Store, Date) duple within the test set\n",
    "- Store - a unique Id for each store\n",
    "- Sales - the turnover for any given day (this is what you are predicting)\n",
    "- Customers - the number of customers on a given day\n",
    "- Open - an indicator for whether the store was open: 0 = closed, 1 = open\n",
    "- StateHoliday - indicates a state holiday. Normally all stores, with few exceptions, are closed on state holidays. Note that all schools are closed on public holidays and weekends. a = public holiday, b = Easter holiday, c = Christmas, 0 = None\n",
    "- SchoolHoliday - indicates if the (Store, Date) was affected by the closure of public schools\n",
    "- StoreType - differentiates between 4 different store models: a, b, c, d\n",
    "- Assortment - describes an assortment level: a = basic, b = extra, c = extended\n",
    "- CompetitionDistance - distance in meters to the nearest competitor store\n",
    "- CompetitionOpenSince[Month/Year] - gives the approximate year and month of the time the nearest competitor was opened\n",
    "- Promo - indicates whether a store is running a promo on that day\n",
    "- Promo2 - Promo2 is a continuing and consecutive promotion for some stores: 0 = store is not participating, 1 = store is participating\n",
    "- Promo2Since[Year/Week] - describes the year and calendar week when the store started participating in Promo2\n",
    "- PromoInterval - describes the consecutive intervals Promo2 is started, naming the months the promotion is started anew. E.g. \"Feb,May,Aug,Nov\" means each round starts in February, May, August, November of any given year for that store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv', low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Store</th>\n",
       "      <th>DayOfWeek</th>\n",
       "      <th>Date</th>\n",
       "      <th>Sales</th>\n",
       "      <th>Customers</th>\n",
       "      <th>Open</th>\n",
       "      <th>Promo</th>\n",
       "      <th>StateHoliday</th>\n",
       "      <th>SchoolHoliday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>5263</td>\n",
       "      <td>555</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>6064</td>\n",
       "      <td>625</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>8314</td>\n",
       "      <td>821</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>13995</td>\n",
       "      <td>1498</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>4822</td>\n",
       "      <td>559</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Store  DayOfWeek        Date  Sales  Customers  Open  Promo StateHoliday  \\\n",
       "0      1          5  2015-07-31   5263        555     1      1            0   \n",
       "1      2          5  2015-07-31   6064        625     1      1            0   \n",
       "2      3          5  2015-07-31   8314        821     1      1            0   \n",
       "3      4          5  2015-07-31  13995       1498     1      1            0   \n",
       "4      5          5  2015-07-31   4822        559     1      1            0   \n",
       "\n",
       "   SchoolHoliday  \n",
       "0              1  \n",
       "1              1  \n",
       "2              1  \n",
       "3              1  \n",
       "4              1  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test = pd.read_csv('test.csv', low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Store</th>\n",
       "      <th>DayOfWeek</th>\n",
       "      <th>Date</th>\n",
       "      <th>Open</th>\n",
       "      <th>Promo</th>\n",
       "      <th>StateHoliday</th>\n",
       "      <th>SchoolHoliday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2015-09-17</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2015-09-17</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>2015-09-17</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>2015-09-17</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>2015-09-17</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  Store  DayOfWeek        Date  Open  Promo StateHoliday  SchoolHoliday\n",
       "0   1      1          4  2015-09-17     1      1            0              0\n",
       "1   2      3          4  2015-09-17     1      1            0              0\n",
       "2   3      7          4  2015-09-17     1      1            0              0\n",
       "3   4      8          4  2015-09-17     1      1            0              0\n",
       "4   5      9          4  2015-09-17     1      1            0              0"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "store = pd.read_csv('store.csv', low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Store</th>\n",
       "      <th>StoreType</th>\n",
       "      <th>Assortment</th>\n",
       "      <th>CompetitionDistance</th>\n",
       "      <th>CompetitionOpenSinceMonth</th>\n",
       "      <th>CompetitionOpenSinceYear</th>\n",
       "      <th>Promo2</th>\n",
       "      <th>Promo2SinceWeek</th>\n",
       "      <th>Promo2SinceYear</th>\n",
       "      <th>PromoInterval</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>c</td>\n",
       "      <td>a</td>\n",
       "      <td>1270</td>\n",
       "      <td>9</td>\n",
       "      <td>2008</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>570</td>\n",
       "      <td>11</td>\n",
       "      <td>2007</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>2010</td>\n",
       "      <td>Jan,Apr,Jul,Oct</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>14130</td>\n",
       "      <td>12</td>\n",
       "      <td>2006</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>2011</td>\n",
       "      <td>Jan,Apr,Jul,Oct</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>c</td>\n",
       "      <td>c</td>\n",
       "      <td>620</td>\n",
       "      <td>9</td>\n",
       "      <td>2009</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>29910</td>\n",
       "      <td>4</td>\n",
       "      <td>2015</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Store StoreType Assortment  CompetitionDistance  CompetitionOpenSinceMonth  \\\n",
       "0      1         c          a                 1270                          9   \n",
       "1      2         a          a                  570                         11   \n",
       "2      3         a          a                14130                         12   \n",
       "3      4         c          c                  620                          9   \n",
       "4      5         a          a                29910                          4   \n",
       "\n",
       "   CompetitionOpenSinceYear  Promo2  Promo2SinceWeek  Promo2SinceYear  \\\n",
       "0                      2008       0              NaN              NaN   \n",
       "1                      2007       1               13             2010   \n",
       "2                      2006       1               14             2011   \n",
       "3                      2009       0              NaN              NaN   \n",
       "4                      2015       0              NaN              NaN   \n",
       "\n",
       "     PromoInterval  \n",
       "0              NaN  \n",
       "1  Jan,Apr,Jul,Oct  \n",
       "2  Jan,Apr,Jul,Oct  \n",
       "3              NaN  \n",
       "4              NaN  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "store.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.7976931348623157e+308"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def competitionIsOpen(val, store):\n",
    "    s = val.Store\n",
    "    x = store[store.Store == s]\n",
    "    mnt = x.CompetitionOpenSinceMonth\n",
    "    if mnt in range(10):\n",
    "        mnt = '0'+str(mnt)\n",
    "    else:\n",
    "        mnt = str(mnt)\n",
    "    dte = str(x.CompetitionOpenSinceYear)+'-'+mnt+'-01'\n",
    "    if np.datetime64(test.loc[0].Date) > dte:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2015-08-01 00:00:00\n",
      "2015-09-17 00:00:00\n",
      "2013-01-01 00:00:00\n",
      "2015-07-31 00:00:00\n"
     ]
    }
   ],
   "source": [
    "print np.min(test.Date.astype('datetime64'))\n",
    "print np.max(test.Date.astype('datetime64'))\n",
    "print np.min(train.Date.astype('datetime64'))\n",
    "print np.max(train.Date.astype('datetime64'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "75860.0"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(store.CompetitionDistance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Store</th>\n",
       "      <th>StoreType</th>\n",
       "      <th>Assortment</th>\n",
       "      <th>CompetitionDistance</th>\n",
       "      <th>CompetitionOpenSinceMonth</th>\n",
       "      <th>CompetitionOpenSinceYear</th>\n",
       "      <th>Promo2</th>\n",
       "      <th>Promo2SinceWeek</th>\n",
       "      <th>Promo2SinceYear</th>\n",
       "      <th>PromoInterval</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>290</th>\n",
       "      <td>291</td>\n",
       "      <td>d</td>\n",
       "      <td>a</td>\n",
       "      <td>80000</td>\n",
       "      <td>10</td>\n",
       "      <td>2015</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>621</th>\n",
       "      <td>622</td>\n",
       "      <td>a</td>\n",
       "      <td>c</td>\n",
       "      <td>80000</td>\n",
       "      <td>10</td>\n",
       "      <td>2015</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>878</th>\n",
       "      <td>879</td>\n",
       "      <td>d</td>\n",
       "      <td>a</td>\n",
       "      <td>80000</td>\n",
       "      <td>10</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2013</td>\n",
       "      <td>Feb,May,Aug,Nov</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Store StoreType Assortment  CompetitionDistance  \\\n",
       "290    291         d          a                80000   \n",
       "621    622         a          c                80000   \n",
       "878    879         d          a                80000   \n",
       "\n",
       "     CompetitionOpenSinceMonth  CompetitionOpenSinceYear  Promo2  \\\n",
       "290                         10                      2015       0   \n",
       "621                         10                      2015       0   \n",
       "878                         10                      2015       1   \n",
       "\n",
       "     Promo2SinceWeek  Promo2SinceYear    PromoInterval  \n",
       "290              NaN              NaN              NaN  \n",
       "621              NaN              NaN              NaN  \n",
       "878                5             2013  Feb,May,Aug,Nov  "
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "store.loc[store.CompetitionDistance == 80000.0, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "store.loc[np.isnan(store.CompetitionDistance), 'CompetitionDistance'] = [80000.0,80000.0,80000.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "store.loc[store.CompetitionDistance == 80000.0, 'CompetitionOpenSinceMonth'] = [10,10,10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "store.loc[store.CompetitionDistance == 80000.0, 'CompetitionOpenSinceYear'] = [2015,2015,2015]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   12.,  2012.],\n",
       "       [   12.,  2012.],\n",
       "       [   12.,  2012.],\n",
       "       [   12.,  2012.],\n",
       "       [   12.,  2012.],\n",
       "       [   12.,  2012.],\n",
       "       [   12.,  2012.],\n",
       "       [   12.,  2012.],\n",
       "       [   12.,  2012.],\n",
       "       [   12.,  2012.]])"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.ones((10,2))*(np.array([12, 2012]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "store.loc[np.isnan(store.CompetitionOpenSinceMonth), ['CompetitionOpenSinceMonth', 'CompetitionOpenSinceYear']] = np.ones((351,2))*(np.array([12, 2012]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def setdate(val):\n",
    "    mnt = int(val.CompetitionOpenSinceMonth)\n",
    "    if mnt < 10:\n",
    "        mnt = '0'+str(mnt)\n",
    "    else:\n",
    "        mnt = str(mnt)\n",
    "    yr = str(int(val.CompetitionOpenSinceYear))\n",
    "    return yr+'-'+mnt+'-'+'01'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dates = [setdate(i[1]) for i in store.loc[:,['CompetitionOpenSinceMonth', 'CompetitionOpenSinceYear']].iterrows()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "store['CompetitionOpenSince'] = np.array(dates).astype('datetime64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Store</th>\n",
       "      <th>StoreType</th>\n",
       "      <th>Assortment</th>\n",
       "      <th>CompetitionDistance</th>\n",
       "      <th>CompetitionOpenSinceMonth</th>\n",
       "      <th>CompetitionOpenSinceYear</th>\n",
       "      <th>Promo2</th>\n",
       "      <th>Promo2SinceWeek</th>\n",
       "      <th>Promo2SinceYear</th>\n",
       "      <th>PromoInterval</th>\n",
       "      <th>CompetitionOpenSince</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>c</td>\n",
       "      <td>a</td>\n",
       "      <td>1270</td>\n",
       "      <td>9</td>\n",
       "      <td>2008</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2008-09-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>570</td>\n",
       "      <td>11</td>\n",
       "      <td>2007</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>2010</td>\n",
       "      <td>Jan,Apr,Jul,Oct</td>\n",
       "      <td>2007-11-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>14130</td>\n",
       "      <td>12</td>\n",
       "      <td>2006</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>2011</td>\n",
       "      <td>Jan,Apr,Jul,Oct</td>\n",
       "      <td>2006-12-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>c</td>\n",
       "      <td>c</td>\n",
       "      <td>620</td>\n",
       "      <td>9</td>\n",
       "      <td>2009</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009-09-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>29910</td>\n",
       "      <td>4</td>\n",
       "      <td>2015</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-04-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Store StoreType Assortment  CompetitionDistance  CompetitionOpenSinceMonth  \\\n",
       "0      1         c          a                 1270                          9   \n",
       "1      2         a          a                  570                         11   \n",
       "2      3         a          a                14130                         12   \n",
       "3      4         c          c                  620                          9   \n",
       "4      5         a          a                29910                          4   \n",
       "\n",
       "   CompetitionOpenSinceYear  Promo2  Promo2SinceWeek  Promo2SinceYear  \\\n",
       "0                      2008       0              NaN              NaN   \n",
       "1                      2007       1               13             2010   \n",
       "2                      2006       1               14             2011   \n",
       "3                      2009       0              NaN              NaN   \n",
       "4                      2015       0              NaN              NaN   \n",
       "\n",
       "     PromoInterval CompetitionOpenSince  \n",
       "0              NaN           2008-09-01  \n",
       "1  Jan,Apr,Jul,Oct           2007-11-01  \n",
       "2  Jan,Apr,Jul,Oct           2006-12-01  \n",
       "3              NaN           2009-09-01  \n",
       "4              NaN           2015-04-01  "
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "store.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "store['StoreType_d'] = (store.StoreType == 'd').astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "store['Assortment_c'] = (store.Assortment == 'c').astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Store</th>\n",
       "      <th>StoreType</th>\n",
       "      <th>Assortment</th>\n",
       "      <th>CompetitionDistance</th>\n",
       "      <th>CompetitionOpenSinceMonth</th>\n",
       "      <th>CompetitionOpenSinceYear</th>\n",
       "      <th>Promo2</th>\n",
       "      <th>Promo2SinceWeek</th>\n",
       "      <th>Promo2SinceYear</th>\n",
       "      <th>PromoInterval</th>\n",
       "      <th>CompetitionOpenSince</th>\n",
       "      <th>StoreType_a</th>\n",
       "      <th>StoreType_b</th>\n",
       "      <th>StoreType_c</th>\n",
       "      <th>StoreType_d</th>\n",
       "      <th>Assortment_a</th>\n",
       "      <th>Assortment_b</th>\n",
       "      <th>Assortment_c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>c</td>\n",
       "      <td>a</td>\n",
       "      <td>1270</td>\n",
       "      <td>9</td>\n",
       "      <td>2008</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2008-09-01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>570</td>\n",
       "      <td>11</td>\n",
       "      <td>2007</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>2010</td>\n",
       "      <td>Jan,Apr,Jul,Oct</td>\n",
       "      <td>2007-11-01</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>14130</td>\n",
       "      <td>12</td>\n",
       "      <td>2006</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>2011</td>\n",
       "      <td>Jan,Apr,Jul,Oct</td>\n",
       "      <td>2006-12-01</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>c</td>\n",
       "      <td>c</td>\n",
       "      <td>620</td>\n",
       "      <td>9</td>\n",
       "      <td>2009</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009-09-01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>29910</td>\n",
       "      <td>4</td>\n",
       "      <td>2015</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-04-01</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Store StoreType Assortment  CompetitionDistance  CompetitionOpenSinceMonth  \\\n",
       "0      1         c          a                 1270                          9   \n",
       "1      2         a          a                  570                         11   \n",
       "2      3         a          a                14130                         12   \n",
       "3      4         c          c                  620                          9   \n",
       "4      5         a          a                29910                          4   \n",
       "\n",
       "   CompetitionOpenSinceYear  Promo2  Promo2SinceWeek  Promo2SinceYear  \\\n",
       "0                      2008       0              NaN              NaN   \n",
       "1                      2007       1               13             2010   \n",
       "2                      2006       1               14             2011   \n",
       "3                      2009       0              NaN              NaN   \n",
       "4                      2015       0              NaN              NaN   \n",
       "\n",
       "     PromoInterval CompetitionOpenSince  StoreType_a  StoreType_b  \\\n",
       "0              NaN           2008-09-01            0            0   \n",
       "1  Jan,Apr,Jul,Oct           2007-11-01            1            0   \n",
       "2  Jan,Apr,Jul,Oct           2006-12-01            1            0   \n",
       "3              NaN           2009-09-01            0            0   \n",
       "4              NaN           2015-04-01            1            0   \n",
       "\n",
       "   StoreType_c  StoreType_d  Assortment_a  Assortment_b  Assortment_c  \n",
       "0            1            0             1             0             0  \n",
       "1            0            0             1             0             0  \n",
       "2            0            0             1             0             0  \n",
       "3            1            0             0             0             1  \n",
       "4            0            0             1             0             0  "
      ]
     },
     "execution_count": 230,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "store.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "store.to_csv('store_new.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Store</th>\n",
       "      <th>DayOfWeek</th>\n",
       "      <th>Date</th>\n",
       "      <th>Sales</th>\n",
       "      <th>Customers</th>\n",
       "      <th>Open</th>\n",
       "      <th>Promo</th>\n",
       "      <th>StateHoliday</th>\n",
       "      <th>SchoolHoliday</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>5263</td>\n",
       "      <td>555</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>6064</td>\n",
       "      <td>625</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>8314</td>\n",
       "      <td>821</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>13995</td>\n",
       "      <td>1498</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>4822</td>\n",
       "      <td>559</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Store  DayOfWeek        Date  Sales  Customers  Open  Promo StateHoliday  \\\n",
       "0      1          5  2015-07-31   5263        555     1      1            0   \n",
       "1      2          5  2015-07-31   6064        625     1      1            0   \n",
       "2      3          5  2015-07-31   8314        821     1      1            0   \n",
       "3      4          5  2015-07-31  13995       1498     1      1            0   \n",
       "4      5          5  2015-07-31   4822        559     1      1            0   \n",
       "\n",
       "   SchoolHoliday  \n",
       "0              1  \n",
       "1              1  \n",
       "2              1  \n",
       "3              1  \n",
       "4              1  "
      ]
     },
     "execution_count": 238,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['0', 'a', 'b', 'c'], dtype=object)"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(train.StateHoliday)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test['StateHoliday_0'] = (test.StateHoliday == '0').astype('int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Store</th>\n",
       "      <th>DayOfWeek</th>\n",
       "      <th>Date</th>\n",
       "      <th>Sales</th>\n",
       "      <th>Customers</th>\n",
       "      <th>Open</th>\n",
       "      <th>Promo</th>\n",
       "      <th>StateHoliday</th>\n",
       "      <th>SchoolHoliday</th>\n",
       "      <th>StateHoliday_0</th>\n",
       "      <th>StateHoliday_a</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>5263</td>\n",
       "      <td>555</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>6064</td>\n",
       "      <td>625</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>8314</td>\n",
       "      <td>821</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>13995</td>\n",
       "      <td>1498</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>4822</td>\n",
       "      <td>559</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Store  DayOfWeek        Date  Sales  Customers  Open  Promo StateHoliday  \\\n",
       "0      1          5  2015-07-31   5263        555     1      1            0   \n",
       "1      2          5  2015-07-31   6064        625     1      1            0   \n",
       "2      3          5  2015-07-31   8314        821     1      1            0   \n",
       "3      4          5  2015-07-31  13995       1498     1      1            0   \n",
       "4      5          5  2015-07-31   4822        559     1      1            0   \n",
       "\n",
       "   SchoolHoliday  StateHoliday_0  StateHoliday_a  \n",
       "0              1               1               0  \n",
       "1              1               1               0  \n",
       "2              1               1               0  \n",
       "3              1               1               0  \n",
       "4              1               1               0  "
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['DayOfWeek_1'] = (train.DayOfWeek == 1).astype('int')\n",
    "train['DayOfWeek_2'] = (train.DayOfWeek == 2).astype('int')\n",
    "train['DayOfWeek_3'] = (train.DayOfWeek == 3).astype('int')\n",
    "train['DayOfWeek_4'] = (train.DayOfWeek == 4).astype('int')\n",
    "train['DayOfWeek_5'] = (train.DayOfWeek == 5).astype('int')\n",
    "train['DayOfWeek_6'] = (train.DayOfWeek == 6).astype('int')\n",
    "train['DayOfWeek_7'] = (train.DayOfWeek == 7).astype('int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test['DayOfWeek_1'] = (test.DayOfWeek == 1).astype('int')\n",
    "test['DayOfWeek_2'] = (test.DayOfWeek == 2).astype('int')\n",
    "test['DayOfWeek_3'] = (test.DayOfWeek == 3).astype('int')\n",
    "test['DayOfWeek_4'] = (test.DayOfWeek == 4).astype('int')\n",
    "test['DayOfWeek_5'] = (test.DayOfWeek == 5).astype('int')\n",
    "test['DayOfWeek_6'] = (test.DayOfWeek == 6).astype('int')\n",
    "test['DayOfWeek_7'] = (test.DayOfWeek == 7).astype('int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Store</th>\n",
       "      <th>DayOfWeek</th>\n",
       "      <th>Date</th>\n",
       "      <th>Sales</th>\n",
       "      <th>Customers</th>\n",
       "      <th>Open</th>\n",
       "      <th>Promo</th>\n",
       "      <th>StateHoliday</th>\n",
       "      <th>SchoolHoliday</th>\n",
       "      <th>StateHoliday_0</th>\n",
       "      <th>StateHoliday_a</th>\n",
       "      <th>DayOfWeek_1</th>\n",
       "      <th>DayOfWeek_2</th>\n",
       "      <th>DayOfWeek_3</th>\n",
       "      <th>DayOfWeek_4</th>\n",
       "      <th>DayOfWeek_5</th>\n",
       "      <th>DayOfWeek_6</th>\n",
       "      <th>DayOfWeek_7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>5263</td>\n",
       "      <td>555</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>6064</td>\n",
       "      <td>625</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>8314</td>\n",
       "      <td>821</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>13995</td>\n",
       "      <td>1498</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>4822</td>\n",
       "      <td>559</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Store  DayOfWeek        Date  Sales  Customers  Open  Promo StateHoliday  \\\n",
       "0      1          5  2015-07-31   5263        555     1      1            0   \n",
       "1      2          5  2015-07-31   6064        625     1      1            0   \n",
       "2      3          5  2015-07-31   8314        821     1      1            0   \n",
       "3      4          5  2015-07-31  13995       1498     1      1            0   \n",
       "4      5          5  2015-07-31   4822        559     1      1            0   \n",
       "\n",
       "   SchoolHoliday  StateHoliday_0  StateHoliday_a  DayOfWeek_1  DayOfWeek_2  \\\n",
       "0              1               1               0            0            0   \n",
       "1              1               1               0            0            0   \n",
       "2              1               1               0            0            0   \n",
       "3              1               1               0            0            0   \n",
       "4              1               1               0            0            0   \n",
       "\n",
       "   DayOfWeek_3  DayOfWeek_4  DayOfWeek_5  DayOfWeek_6  DayOfWeek_7  \n",
       "0            0            0            1            0            0  \n",
       "1            0            0            1            0            0  \n",
       "2            0            0            1            0            0  \n",
       "3            0            0            1            0            0  \n",
       "4            0            0            1            0            0  "
      ]
     },
     "execution_count": 262,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Store</th>\n",
       "      <th>StoreType</th>\n",
       "      <th>Assortment</th>\n",
       "      <th>CompetitionDistance</th>\n",
       "      <th>CompetitionOpenSinceMonth</th>\n",
       "      <th>CompetitionOpenSinceYear</th>\n",
       "      <th>Promo2</th>\n",
       "      <th>Promo2SinceWeek</th>\n",
       "      <th>Promo2SinceYear</th>\n",
       "      <th>PromoInterval</th>\n",
       "      <th>CompetitionOpenSince</th>\n",
       "      <th>StoreType_a</th>\n",
       "      <th>StoreType_b</th>\n",
       "      <th>StoreType_c</th>\n",
       "      <th>StoreType_d</th>\n",
       "      <th>Assortment_a</th>\n",
       "      <th>Assortment_b</th>\n",
       "      <th>Assortment_c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>c</td>\n",
       "      <td>a</td>\n",
       "      <td>1270</td>\n",
       "      <td>9</td>\n",
       "      <td>2008</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2008-09-01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>570</td>\n",
       "      <td>11</td>\n",
       "      <td>2007</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>2010</td>\n",
       "      <td>Jan,Apr,Jul,Oct</td>\n",
       "      <td>2007-11-01</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>14130</td>\n",
       "      <td>12</td>\n",
       "      <td>2006</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>2011</td>\n",
       "      <td>Jan,Apr,Jul,Oct</td>\n",
       "      <td>2006-12-01</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>c</td>\n",
       "      <td>c</td>\n",
       "      <td>620</td>\n",
       "      <td>9</td>\n",
       "      <td>2009</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009-09-01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>a</td>\n",
       "      <td>a</td>\n",
       "      <td>29910</td>\n",
       "      <td>4</td>\n",
       "      <td>2015</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2015-04-01</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Store StoreType Assortment  CompetitionDistance  CompetitionOpenSinceMonth  \\\n",
       "0      1         c          a                 1270                          9   \n",
       "1      2         a          a                  570                         11   \n",
       "2      3         a          a                14130                         12   \n",
       "3      4         c          c                  620                          9   \n",
       "4      5         a          a                29910                          4   \n",
       "\n",
       "   CompetitionOpenSinceYear  Promo2  Promo2SinceWeek  Promo2SinceYear  \\\n",
       "0                      2008       0              NaN              NaN   \n",
       "1                      2007       1               13             2010   \n",
       "2                      2006       1               14             2011   \n",
       "3                      2009       0              NaN              NaN   \n",
       "4                      2015       0              NaN              NaN   \n",
       "\n",
       "     PromoInterval CompetitionOpenSince  StoreType_a  StoreType_b  \\\n",
       "0              NaN           2008-09-01            0            0   \n",
       "1  Jan,Apr,Jul,Oct           2007-11-01            1            0   \n",
       "2  Jan,Apr,Jul,Oct           2006-12-01            1            0   \n",
       "3              NaN           2009-09-01            0            0   \n",
       "4              NaN           2015-04-01            1            0   \n",
       "\n",
       "   StoreType_c  StoreType_d  Assortment_a  Assortment_b  Assortment_c  \n",
       "0            1            0             1             0             0  \n",
       "1            0            0             1             0             0  \n",
       "2            0            0             1             0             0  \n",
       "3            1            0             0             0             1  \n",
       "4            0            0             1             0             0  "
      ]
     },
     "execution_count": 268,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "store.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fts = ['Store', 'CompetitionDistance', 'Promo2', 'CompetitionOpenSince', 'StoreType_a', \n",
    "       'StoreType_b', 'StoreType_c', 'StoreType_d', 'Assortment_a', 'Assortment_b', 'Assortment_c']\n",
    "test_new = pd.merge(test, store[fts], how = 'left', on = 'Store')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_new['Date'] = test_new.Date.astype('datetime64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_new['CompetitionOpen'] = (test_new.Date > test_new.CompetitionOpenSince).astype('int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_new.loc[test_new.CompetitionOpen == 0, 'CompetitionDistance'] = 100000*np.ones(len(test_new.loc[test_new.CompetitionOpen == 0,:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fts = ['Store', 'CompetitionDistance', 'Promo2', 'CompetitionOpenSince', 'StoreType_a', \n",
    "       'StoreType_b', 'StoreType_c', 'StoreType_d', 'Assortment_a', 'Assortment_b', 'Assortment_c']\n",
    "train_new = pd.merge(train, store[fts], how = 'left', on = 'Store')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_new['Date'] = train_new.Date.astype('datetime64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_new['CompetitionOpen'] = (train_new.Date > train_new.CompetitionOpenSince).astype('int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_new.loc[train_new.CompetitionOpen == 0, 'CompetitionDistance'] = 100000*np.ones(len(train_new.loc[train_new.CompetitionOpen == 0, 'CompetitionDistance']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Store</th>\n",
       "      <th>DayOfWeek</th>\n",
       "      <th>Date</th>\n",
       "      <th>Sales</th>\n",
       "      <th>Customers</th>\n",
       "      <th>Open</th>\n",
       "      <th>Promo</th>\n",
       "      <th>StateHoliday</th>\n",
       "      <th>SchoolHoliday</th>\n",
       "      <th>StateHoliday_0</th>\n",
       "      <th>...</th>\n",
       "      <th>Promo2</th>\n",
       "      <th>CompetitionOpenSince</th>\n",
       "      <th>StoreType_a</th>\n",
       "      <th>StoreType_b</th>\n",
       "      <th>StoreType_c</th>\n",
       "      <th>StoreType_d</th>\n",
       "      <th>Assortment_a</th>\n",
       "      <th>Assortment_b</th>\n",
       "      <th>Assortment_c</th>\n",
       "      <th>CompetitionOpen</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>5263</td>\n",
       "      <td>555</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2008-09-01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>6064</td>\n",
       "      <td>625</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>2007-11-01</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>8314</td>\n",
       "      <td>821</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>2006-12-01</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>13995</td>\n",
       "      <td>1498</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2009-09-01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>2015-07-31</td>\n",
       "      <td>4822</td>\n",
       "      <td>559</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2015-04-01</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Store  DayOfWeek       Date  Sales  Customers  Open  Promo StateHoliday  \\\n",
       "0      1          5 2015-07-31   5263        555     1      1            0   \n",
       "1      2          5 2015-07-31   6064        625     1      1            0   \n",
       "2      3          5 2015-07-31   8314        821     1      1            0   \n",
       "3      4          5 2015-07-31  13995       1498     1      1            0   \n",
       "4      5          5 2015-07-31   4822        559     1      1            0   \n",
       "\n",
       "   SchoolHoliday  StateHoliday_0       ...         Promo2  \\\n",
       "0              1               1       ...              0   \n",
       "1              1               1       ...              1   \n",
       "2              1               1       ...              1   \n",
       "3              1               1       ...              0   \n",
       "4              1               1       ...              0   \n",
       "\n",
       "   CompetitionOpenSince  StoreType_a  StoreType_b  StoreType_c  StoreType_d  \\\n",
       "0            2008-09-01            0            0            1            0   \n",
       "1            2007-11-01            1            0            0            0   \n",
       "2            2006-12-01            1            0            0            0   \n",
       "3            2009-09-01            0            0            1            0   \n",
       "4            2015-04-01            1            0            0            0   \n",
       "\n",
       "   Assortment_a  Assortment_b  Assortment_c  CompetitionOpen  \n",
       "0             1             0             0                1  \n",
       "1             1             0             0                1  \n",
       "2             1             0             0                1  \n",
       "3             0             0             1                1  \n",
       "4             1             0             0                1  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 296,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_new.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Store</th>\n",
       "      <th>DayOfWeek</th>\n",
       "      <th>Date</th>\n",
       "      <th>Open</th>\n",
       "      <th>Promo</th>\n",
       "      <th>StateHoliday</th>\n",
       "      <th>SchoolHoliday</th>\n",
       "      <th>StateHoliday_a</th>\n",
       "      <th>StateHoliday_0</th>\n",
       "      <th>...</th>\n",
       "      <th>Promo2</th>\n",
       "      <th>CompetitionOpenSince</th>\n",
       "      <th>StoreType_a</th>\n",
       "      <th>StoreType_b</th>\n",
       "      <th>StoreType_c</th>\n",
       "      <th>StoreType_d</th>\n",
       "      <th>Assortment_a</th>\n",
       "      <th>Assortment_b</th>\n",
       "      <th>Assortment_c</th>\n",
       "      <th>CompetitionOpen</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2015-09-17</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2008-09-01</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2015-09-17</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>2006-12-01</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>2015-09-17</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-04-01</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>2015-09-17</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2014-10-01</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>2015-09-17</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2000-08-01</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  Store  DayOfWeek       Date  Open  Promo StateHoliday  SchoolHoliday  \\\n",
       "0   1      1          4 2015-09-17     1      1            0              0   \n",
       "1   2      3          4 2015-09-17     1      1            0              0   \n",
       "2   3      7          4 2015-09-17     1      1            0              0   \n",
       "3   4      8          4 2015-09-17     1      1            0              0   \n",
       "4   5      9          4 2015-09-17     1      1            0              0   \n",
       "\n",
       "   StateHoliday_a  StateHoliday_0       ...        Promo2  \\\n",
       "0               0               1       ...             0   \n",
       "1               0               1       ...             1   \n",
       "2               0               1       ...             0   \n",
       "3               0               1       ...             0   \n",
       "4               0               1       ...             0   \n",
       "\n",
       "   CompetitionOpenSince  StoreType_a  StoreType_b  StoreType_c  StoreType_d  \\\n",
       "0            2008-09-01            0            0            1            0   \n",
       "1            2006-12-01            1            0            0            0   \n",
       "2            2013-04-01            1            0            0            0   \n",
       "3            2014-10-01            1            0            0            0   \n",
       "4            2000-08-01            1            0            0            0   \n",
       "\n",
       "   Assortment_a  Assortment_b  Assortment_c CompetitionOpen  \n",
       "0             1             0             0               1  \n",
       "1             1             0             0               1  \n",
       "2             0             0             1               1  \n",
       "3             1             0             0               1  \n",
       "4             0             0             1               1  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 298,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_new.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_new.to_csv('train_new.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_new.to_csv('test_new.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([u'Id', u'Store', u'DayOfWeek', u'Date', u'Open', u'Promo',\n",
       "       u'StateHoliday', u'SchoolHoliday', u'StateHoliday_a', u'StateHoliday_0',\n",
       "       u'DayOfWeek_1', u'DayOfWeek_2', u'DayOfWeek_3', u'DayOfWeek_4',\n",
       "       u'DayOfWeek_5', u'DayOfWeek_6', u'DayOfWeek_7', u'CompetitionDistance',\n",
       "       u'Promo2', u'CompetitionOpenSince', u'StoreType_a', u'StoreType_b',\n",
       "       u'StoreType_c', u'StoreType_d', u'Assortment_a', u'Assortment_b',\n",
       "       u'Assortment_c', u'CompetitionOpen'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 301,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_new.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([u'Store', u'DayOfWeek', u'Date', u'Sales', u'Customers', u'Open',\n",
       "       u'Promo', u'StateHoliday', u'SchoolHoliday', u'StateHoliday_0',\n",
       "       u'StateHoliday_a', u'DayOfWeek_1', u'DayOfWeek_2', u'DayOfWeek_3',\n",
       "       u'DayOfWeek_4', u'DayOfWeek_5', u'DayOfWeek_6', u'DayOfWeek_7',\n",
       "       u'CompetitionDistance', u'Promo2', u'CompetitionOpenSince',\n",
       "       u'StoreType_a', u'StoreType_b', u'StoreType_c', u'StoreType_d',\n",
       "       u'Assortment_a', u'Assortment_b', u'Assortment_c', u'CompetitionOpen'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 302,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_new.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_features = [u'Open', u'Promo', u'SchoolHoliday', u'StateHoliday_0',\n",
    "       u'StateHoliday_a', u'DayOfWeek_1', u'DayOfWeek_2', u'DayOfWeek_3',\n",
    "       u'DayOfWeek_4', u'DayOfWeek_5', u'DayOfWeek_6', u'DayOfWeek_7',\n",
    "       u'CompetitionDistance', u'Promo2',\n",
    "       u'StoreType_a', u'StoreType_b', u'StoreType_c', u'StoreType_d',\n",
    "       u'Assortment_a', u'Assortment_b', u'Assortment_c', u'CompetitionOpen']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.473947500782\n",
      "0.473839387915\n",
      "0.473768682013\n",
      "0.473729101826\n",
      "0.473715455335\n",
      "0.473723432366\n",
      "0.473749440724\n",
      "0.473790475852\n",
      "0.473844016503\n",
      "0.473907940743\n"
     ]
    }
   ],
   "source": [
    "bstrap = 1\n",
    "\n",
    "vals = np.linspace(400000, 600000, num = 10)\n",
    "errs = []\n",
    "\n",
    "for i in vals:\n",
    "    train_n, test_n = train_test_split(train_new[train_features + ['Sales']], train_size = 0.7, random_state = 1)\n",
    "    #reg = XGBRegressor()\n",
    "    reg = Ridge(alpha=i)\n",
    "    reg.fit(train_n[train_features], train_n.Sales)\n",
    "    preds = reg.predict(test_n[train_features])\n",
    "    errs.append(RMSPE(test_n.Sales, preds))\n",
    "    print errs[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11b021c90>]"
      ]
     },
     "execution_count": 310,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAEGCAYAAACzYDhlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XecVOXZ//HPJUURy4oaimAHa1QsiEZklQddsGBXNIKA\nwdhLVLAkgo/G8uRnCBY0iooaRYIl2EF0Q2JDiogisYHCIouiiGBQyvX74z4Lw7K7M1tmzpTv+/Wa\n18ycuc851xxHrr3vcxdzd0RERNJpo7gDEBGR/KdkIyIiaadkIyIiaadkIyIiaadkIyIiaadkIyIi\naVdwycbMfmdma8ysRRWf7WZm0xMe35vZJdFn/2tmM8zsPTObaGbtou1nVdpntZntU88Ydzezt8xs\nhZn9rj7HEhHJBpaP42zMrBjo6+79Km1vB9wP7AYc4O7f1nCMjYAyoJO7zzOzzd39h+izi4F93f3c\nSvvsDTzj7u3rGf+2wA7ACcB37v7/6nM8EZG45WvNproMegdwdYrH+B/gM3efB1CRaCKbAd9Usc+Z\nwOiKN2Z2lJm9aWZTzWyMmTVP5cTu/rW7TwFWphiriEhWy9dkYxtsMOsFzHf391M8xhnA45WOcbOZ\nfQn0BW6tYp/TgCeistsA1wHd3P0AYCpwRcrfQEQkjzSOO4CGZGZvAxsTah4tzGx69NENwDXAUYnF\nazhOU+A4YFDidne/DrjOzAYDfwb6JexzMPCju8+KNnUG9gTeNDOApsCbUdlbgGOrOPUz7v6HlL6s\niEgOyatk4+6dAcysK3BOxT2b6F7KTsCM6B/+tsBUM+vk7ouqOFQPYKq7f13NqR4HXqy0bYOaEDDB\n3c+sIs5rCMlPRKQgJG1GM7MSM5ttZp+Y2aBqygyPPp9hZh2T7WtmLcxsgpl9bGbjzawo2t7dzKaY\n2fvR8xEJ+5RGx6ro9bVNTWEnvnH3D9y9pbvv5O47AfOB/atJNAC9iZrDEs6feNO/FzA94bONgFNJ\nuF8DvA38ysx2ico0r3SMVFRb+xIRySU1JhszawTcBZQQmoR6m9kelcr0BHaNemANBEaksO9gwl/9\nHYCJ0XuAr4Fj3X0fwn2RRxNO5cCZ7t4xelR1gz6xbE3d7NZ+ZmZtzOyFhPfNCZ0Dnq60zy1mNtPM\n3gOKgcQuyYcDX7r73LUnCPGdAzxhZjMITWi71RDTWmbWyszmAZcD15vZl2a2WSr7iohkoxq7PpvZ\nIcAN7l4SvR8M4O63JpS5F3jd3Z+M3s8m/GO8U3X7RmW6unu5mbUCSt1990rnNkKPr1buvtLMXgeu\ndPepDfPVRUQkU5I1o20HzEt4Pz/alkqZNjXs29Ldy6PX5UDLKs59MuG+SWL331FRE9r1SeIWEZEs\nkizZpDriM5V7C1bV8TxUrdbbbmZ7EboWn5ew+Sx33xvoAnQxs7NTjE1ERGKWrDdaGdAu4X07Qg2l\npjJtozJNqtheFr0uN7NW7r7QzFoDa2/Um1lbwv2Ss919TsV2d18QPS8zs8eBTqx/Twczy7/pEERE\nMsDd09ohKVnNZgrQ3sx2jMaenA6Mq1RmHNAHwMw6A0uiJrKa9h1H6ABA9PxstH8R8AIwyN3fqjiB\nmTWq6H1mZk0IY2BmVhWwu+vRQI8bbrgh9hjy6aHrqWuZrY9MqLFm4+6rzOwi4BWgETDS3T8ys/Oi\nz+9z9xfNrKeZfQosJxroWN2+0aFvBcaY2QBgLmHkPcBFwC7ADWZ2Q7StO/Bf4OUo0TQCJhDmONvA\nnDmw0061vQwiIpJOSQd1uvtLwEuVtt1X6f1Fqe4bbf+W0L248vabgJuqCeXAZLECPPwwDB2aSkkR\nEcmUvJsb7aGHYPXquKPID8XFxXGHkFd0PRuOrmXuyaslBszM99/f+eMf4eij445GRCQ3mBkecweB\nnDNgADz4YNxRiIhIoryr2Xz7rbPTTvDZZ7D11nFHJCKS/VSzqYOttoJjjoHHHos7EhERqZB3yQZC\nU9rIkZBHlTYRkZyWl8mmuBiWLYOpmrJTRCQr5GWy2Wgj6Ncv1G5ERCR+eddBoOL7zJsH++4L8+fD\nppvGHJiISBZTB4F6aNcOOnWCpysvgSYiIhmXt8kG1nUUEBGReOVtMxrATz9B27bw9tuwyy4xBiYi\nksXUjFZPG28MZ50V5ksTEZH45HXNBmDmTOjRA774Aho1iikwEZEspppNA/jlL6FNGxg/Pu5IREQK\nV94nG4D+/dVRQEQkTnnfjAbw/fewww7wySew7bYxBCYiksXUjNZAttwSjj9ek3OKiMSlIJINrGtK\ny6OKnIhIziiYZNO1K6xYAe++G3ckIiKFp2CSjZk6CoiIxKUgOghUKCsLXaHnzYPmzTMYmIhIFlMH\ngQa23XZwyCHw1FNxRyIiUlgKKtmAJucUEYlDQTWjAfz8c1h+4N//hvbtMxSYiEgWUzNaGjRtCr/+\ntSbnFBHJpIKr2QB8+CEcdVSYnLNx4wwEJiKSxVSzSZO99gpNaS+/HHckIiKFoSCTDYSOAg8+GHcU\nIiKFoSCb0QCWLoXtt4f//AdatkxzYCIiWUzNaGm0xRZwwgnw6KNxRyIikv8KNtnAuqa0PKrciYhk\npYJONocdBqtWwdtvxx2JiEg85s/PzHkKOtlock4RKWTu8NvfZuZcBZ1sAPr2DXOlLVsWdyQiIpn1\n+OPw5ZeZOVfBJ5vWraFLF/j73+OOREQkcxYtgiuuyFzLTtJkY2YlZjbbzD4xs0HVlBkefT7DzDom\n29fMWpjZBDP72MzGm1lRtL27mU0xs/ej5yMS9jnAzGZGx/pL/b72+tSUJiKF5uKLQ8vOQQdl5nw1\nJhszawTcBZQAewK9zWyPSmV6Aru6e3tgIDAihX0HAxPcvQMwMXoP8DVwrLvvA/QFEjsmjwAGROdp\nb2YldfvKGzrmGPj00zDmRkQk3z37LEyfDkOHZu6cyWo2nYBP3X2uu68ERgO9KpU5HhgF4O7vAEVm\n1irJvmv3iZ5PiPZ/z90XRttnAc3MrImZtQY2d/fJ0WePVOzTEJo0gT59NKOAiOS/776DCy+EBx6A\nZs0yd95kyWY7YF7C+/nRtlTKtKlh35buXh69LgeqGsN/MjA1SlTbRftXKKsijnrp3x8eeQRWrmzI\no4qIZJcrrwwD2g8/PLPnTTbncarDHVOZ5sCqOp67u5mtt93M9gJuBbqneP61hgwZsvZ1cXExxcXF\nKe23++6w887w0ktw/PG1PauISPabMAEmToR77illyJDSjJ47WbIpA9olvG/H+jWMqsq0jco0qWJ7\nWfS63MxaufvCqIlsUUUhM2sLPA2c7e5zEs7RtppjrScx2dRWxSqeSjYikm+WLYOBA+Hee6GkpJie\nPYvXfjY0AzdvkjWjTSHcjN/RzJoCpwPjKpUZB/QBMLPOwJKoiaymfccROgAQPT8b7V8EvAAMcve3\nKk7g7l8BS83sYDMz4OyKfRrSqafCpEnw1VcNfWQRkXhde21oOitpsK5VtZN01mcz6wEMAxoBI939\nFjM7D8Dd74vKVPQ6Ww70c/dp1e0bbW8BjAG2B+YCp7n7EjO7ntAz7ZOEELq7+zdmdgDwMNAMeNHd\nL6ki1pRnfa7OgAGw225w9dX1OoyISNZ4443wx/QHH0CLFht+nolZnwt2iYHqvPkm9OsHs2eH6WxE\nRHLZihWw335w881w8slVl9ESAzE45JCQZN54I+5IRETqb+hQ2Hvv6hNNpiTrIFBwzNYtPXDYYXFH\nIyJSd9OmhX/LZsyIOxI1o1WpvDzct5k3DzbfvAECExHJsJUroVMnuPzyMGi9JmpGi0nLllBcDGPG\nxB2JiEjd3H47tGoFZ58ddySBajbVeO45uOWW0GFARCSXzJoFXbvC1Kmw/fbJy6tmE6MePWDuXPjo\no7gjERFJ3erV4b7z0KGpJZpMUbKpRuPGmpxTRHLPnXdC06aZW4EzVWpGq8HHH4cRt/PmhZmhRUSy\n2eefh04Bb70F7dunvp+a0WLWoUN4PP983JGIiNTMHX7zGxg0qHaJJlOUbJLo319NaSKS/UaOhKVL\nQ1fnbKRmtCSWL4d27cKcQm3aNOihRUQaRFlZmJLmtdfgl7+s/f5qRssCzZvDKafAqFHJy4qIZJo7\nnH9+WH2zLokmU5RsUlDRlJZHlUARyROjR4eOAddeG3ckNVOyScHBB4euhP/6V9yRiIis8/XX4R7N\ngw+Gf6OymZJNCiom5xw5Mu5IRETWueQS+PWvQ3fnbKcOAilatCh0g/7yS9hii7ScQkQkZePGwe9+\nF2Z03nTT+h1LHQSyyC9+Ad26hfZREZE4LVkCF1wADzxQ/0STKUo2taCmNBHJBlddBccdFybbzBVq\nRquFVatghx3glVfCynciIpk2cWLoITtzZsM16asZLcs0bgznnKMZBUQkHsuXhylpRozIvXvHqtnU\n0qefwqGHwvz52d/VUETyy2WXweLF8OijDXtc1Wyy0K67wp57hsXVREQy5c034cknYdiwuCOpGyWb\nOlBHARHJpBUrwr87w4fD1lvHHU3dqBmtDn78Edq2hfffD88iIul03XVh1eCnngqDzBuamtGy1Kab\nwmmnaXJOEUm/6dPh/vvh7rvTk2gyRcmmjgYMCL3S1qyJOxIRyVcrV4ZuzrffDq1bxx1N/SjZ1NGB\nB4Yazj//GXckIpKv/vSnMHtJ375xR1J/umdTD8OGwdSpDd8NUURk9mw47DCYMgV23DG958rEPRsl\nm3r45pvQFXruXCgqythpRSTPrV4Nhx8OvXvDRRel/3zqIJDlttkGuneHJ56IOxIRySd33w0bbRQm\n28wXqtnU08svw+9/D+++m9HTikiemjMHDjoI3ngDdtstM+dUzSYHdO8OCxeGMTciIvXhDgMHhlmd\nM5VoMkXJpp4aNQqTc2pGARGprwcfhO++C4ui5Rs1ozWAzz+Hgw8Ok3NuvHHGTy8ieWDBAth3X3j1\n1fCcSWpGyxE77wz77AP/+EfckYhILnKH888Pj0wnmkxJmmzMrMTMZpvZJ2Y2qJoyw6PPZ5hZx2T7\nmlkLM5tgZh+b2XgzK0rY/rqZ/WBmd1Y6R2l0rOnRY5u6f+2G17+/1rkRkboZMyYsX3LddXFHkj41\nJhszawTcBZQAewK9zWyPSmV6Aru6e3tgIDAihX0HAxPcvQMwMXoPsAK4HriyinAcONPdO0aPb2r7\nZdPppJNCj7Qvv4w7EhHJJd98A5deGu775nMzfLKaTSfgU3ef6+4rgdFAr0pljgdGAbj7O0CRmbVK\nsu/afaLnE6L9f3T3N4Cfqokna6eha9YMzjgDHn447khEJJdceimceSZ07hx3JOmVLNlsB8xLeD8/\n2pZKmTY17NvS3cuj1+VAy0rHrO4u/6ioCe36JHHHon9/eOghTc4pIql5/nl4+2246aa4I0m/ZMkm\n1a5dqdQ4rKrjRd3HUjnPWe6+N9AF6GJmZ6cYW8bsvz9suSW8/nrckYhItvv++9Ah4IEHwqS++a5x\nks/LgHYJ79sRaig1lWkblWlSxfay6HW5mbVy94Vm1hpYlCxQd18QPS8zs8cJzXQbTIE5ZMiQta+L\ni4spLi5OdugGY7ZuFc9u3TJ2WhHJQVddBT17whFHZP7cpaWllJaWZvScNY6zMbPGwH+AbsACYDLQ\n290/SijTE7jI3XuaWWdgmLt3rmlfM7sdWOzut5nZYKDI3QcnHPMc4AB3vzh63wjYyt2/MbMmwBPA\neHf/a6V4Yxlnk2jxYthllzDlxFZbxRqKiGSp114LywZ88EFoDYlbVsz6bGY9gGFAI2Cku99iZucB\nuPt9UZmKXmfLgX7uPq26faPtLYAxwPbAXOA0d18SfTYX2BxoCiwBugNfApMItaVGwATgisqZJRuS\nDYSOAl26wIUXxh2JiGSb5cvDuLzhw+GYY+KOJsiKZJNLsiXZTJgAgwbBtGlxRyIi2eaKK6C8HP72\nt7gjWScTySbZPRupg27dQnPa9OnQsWPy8iJSGN5+OyxJMnNm3JFknqarSYONNoJ+/TSjgIis8913\nYTzNnXeGtbAKjZrR0uSLL+CAA8LknJtsEnc0IhKnNWvguOOgffuwnHy20UScOWyHHUIT2rPPxh2J\niMTt5pvDuJr/+7+4I4mPkk0aVYy5EZHC9corMGJEmGyzSZO4o4mPmtHSaMUK2GkneO45OPDAuKMR\nkUz74ouw1tWTT0LXrnFHUz01o+W4TTYJcx5dfLHmSxMpNCtWwCmnhJkCsjnRZIqSTZr16werV8Nj\nj8UdiYhk0iWXwI47hnE1oma0jHjnHTjxRJg9G7bYIu5oRCTdHnoIbrstrHG1+eZxR5OcZhCopWxN\nNhCWH2jRAv70p7gjEZF0mj4djjoK/vlP2HPPuKNJjZJNLWVzsikvh733hkmTYI89kpcXkdzz3Xdh\nfN0tt8Dpp8cdTeqUbGopm5MNhMFcL74YukJa1q45KiJ1UTFws0MH+POf446mdtQbLc9ceCGUlWmg\np0g+uukmWLoUbr897kiyk2o2GTZxIpx7LsyaBc2axR2NiDSEl18Og7inTIHWreOOpvZUs8lD3bqF\nNt1CnrZCJJ/MnRsWQnviidxMNJmimk0MKibpnDo1zKEmIrlpxQo47LAwm3Muj6dRB4FaypVkA3Dj\njfD++zB2bNyRiEhdDRwYeqCNGZPbnX7UjJbHrroqrOQ5cWLckYhIXTz4IPzrX+E5lxNNpqhmE6Nn\nn4Vrr4UZMwp7NliRXDNtGhx9dP6Mm1PNJs/16gXt2sFdd8UdiYik6ttvwwSbd9+dH4kmU1Szidns\n2eEG44cfQsuWcUcjIjWpGLi5225wxx1xR9Nw1EGglnIx2QBceSUsXhwm7xOR7HXjjTBhArz2Wn41\nfSvZ1FKuJpulS2H33eGZZ8JCSyKSfXJ94GZNdM+mQGyxBdx6qxZZE8lWFQM3R4/Ov0STKUo2WeLX\nv4bGjdWUJpJtKlbcHDQIunSJO5rcpWa0LDJtGvTsGToNFBXFHY2IAPzmN/D99/Dkk/k7nkb3bGop\n15MNwHnnwSabwF/+EnckIjJyZFjwcPLk3Fhxs66UbGopH5LN11+H1f1efz0stiYi8Zg6FUpK8mfg\nZk3UQaAAbbst3HBD6CyQ43lTJGdVDNy85578TzSZomSThX772zDuRpN0imTemjWhw86JJ8Kpp8Yd\nTf5QM1qW+uc/oU+fsMha8+ZxRyNSOIYODRPkTpyYXwM3a6JmtALWtSscemgYfyMimfHSS/DXv4ae\nZ4WSaDJFNZssNn8+7Ldf6Amz885xRyOS3+bMgc6dQ/N1oY2nUc2mwLVtC7/7HVx+edyRiOS3ioGb\ngwcXXqLJFNVsstxPP8Fee4VlCEpK4o5GJD+dey788EOYjiZfB27WRDUbYeONYdgwuPRS+PnnuKMR\nyT8jR8Kbb4bnQkw0mZI02ZhZiZnNNrNPzGxQNWWGR5/PMLOOyfY1sxZmNsHMPjaz8WZWlLD9dTP7\nwczurHSOA8xsZnSsghpff+yxsOuumlVApKFNnRqazp5+GjbbLO5o8luNycbMGgF3ASXAnkBvM9uj\nUpmewK7u3h4YCIxIYd/BwAR37wBMjN4DrACuB66sIpwRwIDoPO3NrKAalYYNg9tugwUL4o5EJD8s\nXhzu04wYEZb4kPRKVrPpBHzq7nPdfSUwGuhVqczxwCgAd38HKDKzVkn2XbtP9HxCtP+P7v4G8FPi\nCcysNbC5u0+ONj1SsU+haN8+TAg4qMq6pYjUxurVYeDmSSeFhCPplyzZbAfMS3g/P9qWSpk2Nezb\n0t3Lo9flQOUFkSvf5d8u2r9CWRVx5L3rroPSUnjjjbgjEclt//u/8OOPGseWScmSTapdu1K5rWZV\nHS/qPpZfXcjSZLPN4Pbb4aKLwl9mIlJ7L70EDzyggZuZ1jjJ52VAu4T37Vi/hlFVmbZRmSZVbC+L\nXpebWSt3Xxg1kS1KIY621RxrPUOGDFn7uri4mOLi4iSHzi1nnBHamO+/P8yhJiKpmzMHzjkHnnoK\nWrWKO5r4lJaWUlpamtFz1jjOxswaA/8BugELgMlAb3f/KKFMT+Aid+9pZp2BYe7euaZ9zex2YLG7\n32Zmg4Eidx+ccMxzgAPc/eKEbe8Al0THeQEY7u4vV4o378bZVGXGDOjeHT76CLbeOu5oRHLDf/8L\nv/pVmHPwssvijia7ZMV6NmbWAxgGNAJGuvstZnYegLvfF5Wp6HW2HOjn7tOq2zfa3gIYA2wPzAVO\nc/cl0Wdzgc2BpsASoLu7zzazA4CHgWbAi+5+SRWxFkSyAbjwwvB8993xxiGSKwYMgOXL4YknNJ6m\nsqxINrmkkJLNt9+GdTbGj4d99407GpHs9sADcMcdYZ5BjafZkJJNLRVSsgG47z7429/CcgT6S02k\nalOnQo8eYcVNjaepmqarkRqde+66ZgER2VDFwM177lGiiZtqNjnuzTfhtNNg9mw1D4gkWr0ajjkG\n9t4b/vSnuKPJbqrZSFKHHgpHHgk33RR3JCLZ5YYbQg80DdzMDqrZ5IGvvoJf/jLUcjp0iDsakfj9\n8Y8walS4n1nI42lSpZqNpKR16zBn2mWXQQHmWpH13HgjPPpomNpJiSZ7KNnkiUsvhc8/hxdeiDsS\nkXi4wx/+EKahKS0Nf4RJ9lCyyRNNm4b1bi67LCxxK1JI3OH66+HZZ+H116Fl5al9JXZKNnnk6KND\nz5s77og7EpHMcYdrrgm1+tdeg1/8Iu6IpCrqIJBnPv8cOnWC6dOhXbvk5UVymTtcdVWozUyYAC1a\nxB1RblIHAam1nXeGCy4I/wOK5DN3uPzy0OPs1VeVaLKdajZ56Mcfw7xpo0ZBnq2wIAKERHPxxTBl\nCrz8MhQVxR1RblPNRupk003DiOlLLoFVq+KORqRhrVkTau/TpsErryjR5Aolmzx1yimwzTZw771x\nRyLScNasgfPOgw8+CIlmyy3jjkhSpWa0PPbhh3DEEeF5223jjkakflavDpPPVown01yADUdLDNSS\nks2GLrss3MP561/jjkSk7lavhn79YP58eO45aN487ojyi5JNLSnZbGjJktBZ4Lnn4MAD445GpPZW\nrYK+fWHRIvjHP8I9SWlY6iAg9VZUBDffHHrurFkTdzQitbNyJZx1VliXZtw4JZpcpmRTAM45JzRD\nPPpo3JGIpG7lSujdG5YtC9PQNGsWd0RSH2pGKxDvvAMnnggffaQePJL9fv4ZTj89/JH097/DxhvH\nHVF+0z2bWlKyqVn//mGUtVYtlGz2009w6qnQqFGYwblp07gjyn9KNrWkZFOz8vIwUeekSaHTgEi2\nWbECTj45NJk98QQ0aRJ3RIVBHQSkQbVsCdddF9a+UU6WbPPf/8IJJ8DmmyvR5CMlmwJz4YWwcGFY\nNlckW/z4Ixx/PGy9NTz2mBJNPmocdwCSWU2ahIkLu3eH5ctDt2hLa+VZpGbLl4dEs9128NBD4V6N\n5B/VbApQmzZhWvZXXglNahp/I3FZtgyOOQa2316JJt8p2RSobbYJqxpOnQoDBoQupiKZ9MMP0KMH\ntG8PI0cq0eQ7JZsCtuWWMH48zJsXBs/9/HPcEUmhWLo0LGO+115w332wkf4lynv6T1zgmjeH558P\nYxtOOin0CBJJpyVL4KijYP/9YcQIJZpCof/MwiabwNixocvpMceEdnSRdPjuu9A5pXNnuPNOdU4p\nJEo2AoReao89BrvsEv4x+O67uCOSfLN4MXTrBocfDn/+sxJNoVGykbUaNQrr3nTuDEceGaZ0F2kI\n33wTEk337mG6JCWawqNkI+sxgzvugOOOg65doaws7ogk1y1aFP54OeYYuPVWJZpCpUGdsgEzuPHG\nsOxuly7w6quw885xRyW5qLw8JJpTToEhQ5RoCpmSjVTr6qtDwunaNXSR1uSdUhtffRUSTe/e8Ic/\nxB2NxE3JRmp0wQUh4Rx5JLz4InTsGHdEkgvKysJvpm9fuPbauKORbJD0no2ZlZjZbDP7xMwGVVNm\nePT5DDPrmGxfM2thZhPM7GMzG29mRQmfXROVn21mRyVsL422TY8e29T9a0tt9OkDd90FJSXw1ltx\nRyPZbt48KC4OM1Mo0UiFGpONmTUC7gJKgD2B3ma2R6UyPYFd3b09MBAYkcK+g4EJ7t4BmBi9x8z2\nBE6PypcA95itbeV14Ex37xg9vqnXN5daOflkePhh6NUrTHMjUpUvvgiJ5vzzQzOsSIVkNZtOwKfu\nPtfdVwKjgV6VyhwPjAJw93eAIjNrlWTftftEzydEr3sBT7j7SnefC3wKHJxwLt1ejFGPHmGJ3jPO\nCLMOiCSaMyckmksvhSuuiDsayTbJks12wLyE9/OjbamUaVPDvi3dvTx6XQ60jF63icol7tMm4f2o\nqAnt+iRxS5p07RoSzYABMGZM3NFItvj8czjiCLjySrjkkrijkWyUrINAqus5plLjsKqO5+5uZqmc\n5yx3X2BmmwFPmdnZ7v5o5UJDhgxZ+7q4uJji4uIUDi210akTTJgQ7uEsXw79+sUdkcTphRfgvPPg\n978Pz5L9SktLKS0tzeg5kyWbMqBdwvt2rF/zqKpM26hMkyq2VwwRLDezVu6+0MxaAxVj1as6VhmA\nuy+InpeZ2eOEZroak42kzz77QGkp/M//hLnULr447ogk0xYsCE1m770Ho0aFGQIkN1T+Q3zo0KFp\nP2eyZrQpQHsz29HMmhJu3o+rVGYc0AfAzDoDS6Imspr2HQf0jV73BZ5N2H6GmTU1s52A9sBkM2tU\n0fvMzJoAxwEz6/SNpcF06ACTJsFf/qJlpgvJ6tVw992w775h7NXMmUo0klyNNRt3X2VmFwGvAI2A\nke7+kZmdF31+n7u/aGY9zexTYDnQr6Z9o0PfCowxswHAXOC0aJ9ZZjYGmAWsAi6Imtk2AV6OEk0j\nYAJwf8NdBqmrHXeEf/0r1HB++CEkHY0Sz1/vvQcDB4aZwidN0kBfSZ25p3pbJvuZmefT98kl33wT\nFsM69NBQ09EaJfll2bIw3cyjj8Itt8A55+i/cT4xM9w9rX8m6uciDaJimenp07XMdL55/nnYe2/4\n+mv44AMa4sewAAAJnUlEQVTo31+JRmpPNRtpUMuXwwknwFZbhfVxmjaNOyKpq7Ky0I155ky4994w\n/YzkJ9VsJOc0bw7PPadlpnPZ6tVhFc399gs1mvffV6KR+lPNRtJi5cowCePChTBuXJjMU7Lf9Omh\nA8Cmm8J998Huu8cdkWSCajaSs5o0CTeTtcx0bli2LEwxU1ICF14YxlAp0UhDUrKRtKlYZvqQQ7TM\ndDYbNw722iv8QfDhh6GnmbqvS0NTM5qknXvoNjtmTFj1c7vKs+tJLObPDx0APvwwdAA44oi4I5K4\nqBlN8oIZDB0ausx26RImbZT4rF4Nw4eHhfD23Td0AFCikXTTSp2SMVddpWWm4zZ1apgsc/PN4d//\nht12izsiKRRKNpJR55+vZabj8MMPYVbm0aPh9tvh7LN1X0YyS81oknFnnx0mctQy05nx7LOhA8DS\npWEGgD59lGgk81SzkVicdFIYy9GrV/hrW4MGG968eWHph9mz4ZFHwiqaInFRzUZiU1ICY8dqmemG\ntmoVDBsWmij33x9mzFCikfip67PE7t134bjjwviO008P06SomadupkwJHQCKimDEiLDmkEgy6vos\nBeGgg0LPKIBTTw2zDlx9NUyeHMboSHJLl4ZVM489Fi67LIxnUqKRbKJkI1lh113h1lvhk0/g6afD\nbNF9+oTF2a64At54A9asiTvK7OMertdee4UpZz78UD3NJDupGU2yljvMmhXu64wdC99+GzoWnHIK\nHHZYmA6nkH35JVx0UUjQ994bxi+J1EUmmtGUbCRnzJ4NTz0VHgsWwIknhsTTtSs0LqB+latWhRkA\n/vjH0GR21VWw8cZxRyW5TMmmlpRsCsdnn4WkM3YszJkTulCfckroQp2PC7YtXBim/58+Pcwxt802\noQNA+/ZxRyb5QMmmlpRsCtMXX6xLPP/5T+jZdvLJYWmDTTaJO7racQ9zx1UklorHTz+FrswdO8Lh\nh4fvqPsy0lCUbGpJyUbmz4dnngmJ5/33oWfPUOMpKYFmzeKObn0rV8JHH62fVGbMCPOWVSSWisf2\n2yu5SPoo2dSSko0kWrgwJJ6nngrjT44+OiSeHj0yv3Lo8uUh+SUmllmzQhKpnFi22SazsYko2dSS\nko1U5+uv4R//CDWet96Cbt1C4jn2WNhii4Y91+LFGzaDffFFmOU6Manss4+Wy5bsoGRTS0o2kopv\nvw2rU44dC5MmhalcTj4Zjj8ettoq9eO4h/nHKieWJUvCLAiJiWWPPfKz44LkByWbWlKykdr6/vsw\nL9vYsTBxIvzqV6HG06vX+s1Zq1fDxx+vn1Teey90ua7cDLbzzrCRhktLDlGyqSUlG6mPH34Ia+yM\nHRsWdzvooNC1+L33YOZMaNlyw8TSunXcUYvUn5JNLSnZSEP58Ud46aXQu22//cJjyy3jjkokPZRs\naknJRkSk9jTrs4iI5AUlGxERSTslGxERSTslGxERSTslGxERSTslGxERSTslGxERSbukycbMSsxs\ntpl9YmaDqikzPPp8hpl1TLavmbUwswlm9rGZjTezooTPronKzzazoxK2H2BmM6PP/lL3rywiIplW\nY7Ixs0bAXUAJsCfQ28z2qFSmJ7Cru7cHBgIjUth3MDDB3TsAE6P3mNmewOlR+RLgHrO1q3iMAAZE\n52lvZiX1+eKSXGlpadwh5BVdz4aja5l7ktVsOgGfuvtcd18JjAZ6VSpzPDAKwN3fAYrMrFWSfdfu\nEz2fEL3uBTzh7ivdfS7wKXCwmbUGNnf3yVG5RxL2kTTR/9ANS9ez4eha5p5kyWY7YF7C+/nRtlTK\ntKlh35buXh69LgdaRq/bROWqOlbi9rIq4hARkSyVLNmkOtFYKnPqWFXHiyYz04RmIiJ5rHGSz8uA\ndgnv27F+DaOqMm2jMk2q2F4WvS43s1buvjBqIluU5Fhl0euqjrUe00LtDWro0KFxh5BXdD0bjq5l\nbkmWbKYQbsbvCCwg3LzvXanMOOAiYLSZdQaWuHu5mS2uYd9xQF/gtuj52YTtj5vZHYRmsvbAZHd3\nM1tqZgcDk4GzgeGVg033rKUiIlI3NSYbd19lZhcBrwCNgJHu/pGZnRd9fp+7v2hmPc3sU2A50K+m\nfaND3wqMMbMBwFzgtGifWWY2BpgFrAIuSFgz4ALgYaAZ8KK7v9wgV0BERNIur9azERGR7JRVMwiY\nWSMzm25mz0XvG2zwp5ltbGZPRtvfNrMdEj7rG53jYzPrk6nvm05VXMshZjY/2jbdzHoklNW1rIGZ\nzTWz96PrNjnapt9mHVVzPfX7rCMzKzKzsWb2kZnNMrODs/L36e5Z8wCuAP4GjIve3w5cHb0eBNwa\nvd4TeI/QCWFHwnicilraZKBT9PpFoCR6fQFwT/T6dGB09LoF8BlQFD0+A4rivhZpuJY3AFdUUU7X\nMvm1nAO0qLRNv82GvZ76fdb9eo4C+kevGwNbZuPvM2tqNmbWFugJPMC6rtQNOfgz8VhPAd2i10cD\n4919ibsvASYQZi/IWdVcS0t4nUjXMjWVr51+m/VT1W9Rv89aMrMtgS7u/iCEe+Xu/j1Z+PvMmmQD\n/Bm4CliTsK0hB3+uHXzq7quA781s6xqOlcuqupYOXGxh/rqRCdVqXcvkHHjVzKaY2W+ibfpt1l1V\n1xP0+6yLnYCvzewhM5tmZvebWXOy8PeZFcnGzI4FFrn7dKoZIOqh3qbeDEnUcC1HEH6Y+wFfAf8v\nhvBy1a/cvSPQA7jQzLokfqjfZq1VdT31+6ybxsD+hGau/Qk9ggcnFsiW32dWJBvgUOB4M5sDPAEc\naWaPEg3+BLC6D/6cn7DP9tGxGgNbuvviKo5V1cDVXFLVtXzE3Rd5hNC81ikqr2uZhLt/FT1/DTxD\nuHb6bdZRVddTv886mw/Md/d3o/djCclnYdb9PuO+uVXFza6uwHPR69uBQdHrwWx4k6sp4a+hz1h3\nk+sd4GDCX/WVb3KNiF6fwfo3uT4n3ODaquJ13NchDdeydcL2y4HHdS1TuoabEtqyAZoDbwBH6bfZ\n4NezlX6fdb6mk4AO0esh0W8z636fsV+oKi5cV9b1oGoBvAp8DIxP/CLAtYSbW7OBoxO2HwDMjD4b\nnrB9Y2AM8AnwNrBjwmf9ou2fAH3jvgYNeC2LE67lo8D7wAzCjA0tdS1TuoY7Rf9zvgd8AFyj32Za\nrucj+n3W+ZruC7wbXbunCb3Rsu73qUGdIiKSdtlyz0ZERPKYko2IiKSdko2IiKSdko2IiKSdko2I\niKSdko2IiKSdko2IiKSdko2IiKTd/wetct3ymbdKuwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10de4f2d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(vals, errs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.299887752778\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "train_n, test_n = train_test_split(train_new[train_features + ['Sales']], train_size = 0.7, random_state = 1)\n",
    "#reg = XGBRegressor()\n",
    "#reg = Ridge(alpha=i)\n",
    "reg = RandomForestRegressor(n_estimators = 100, max_depth = 30, max_features = 7, n_jobs = -1)\n",
    "reg.fit(train_n[train_features], train_n.Sales)\n",
    "preds = reg.predict(test_n[train_features])\n",
    "print RMSPE(test_n.Sales, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 20 0.344537986449\n",
      "3 30 0.300597202677\n",
      "3 50 0.299039105814\n",
      "3 None 0.299039105814\n",
      "6 20 0.340778064187\n",
      "6 30 0.300166290559\n",
      "6 50 0.299065039438\n",
      "6 None 0.299065039438\n",
      "9 20 0.336810136252\n",
      "9 30 0.30003422471\n",
      "9 50 0.299035644148\n",
      "9 None 0.299035644148\n",
      "12 20 0.333540877894\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-317-1dc8efda7f40>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0;31m#reg = Ridge(alpha=i)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0mreg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRandomForestRegressor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_estimators\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_depth\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_features\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m         \u001b[0mreg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_n\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtrain_features\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_n\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSales\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m         \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_n\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtrain_features\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0;32mprint\u001b[0m \u001b[0mmf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mRMSPE\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_n\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSales\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpreds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sklearn/ensemble/forest.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    271\u001b[0m                     \u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrees\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    272\u001b[0m                     verbose=self.verbose, class_weight=self.class_weight)\n\u001b[0;32m--> 273\u001b[0;31m                 for i, t in enumerate(trees))\n\u001b[0m\u001b[1;32m    274\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    275\u001b[0m             \u001b[0;31m# Collect newly grown trees\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m    664\u001b[0m                 \u001b[0;31m# consumption.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    665\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 666\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    667\u001b[0m             \u001b[0;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    668\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc\u001b[0m in \u001b[0;36mretrieve\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    516\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    517\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 518\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    519\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexceptions\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    520\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/multiprocessing/pool.pyc\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    559\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    560\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 561\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    562\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ready\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    563\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/multiprocessing/pool.pyc\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    554\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    555\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ready\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 556\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    557\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    558\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.pyc\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    338\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m    \u001b[0;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    339\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 340\u001b[0;31m                 \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    341\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0m__debug__\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    342\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_note\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"%s.wait(): got it\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "maxf = [15, 18]\n",
    "maxd = [30, None]\n",
    "for mf in maxf:\n",
    "    for md in maxd:\n",
    "        train_n, test_n = train_test_split(train_new[train_features + ['Sales']], train_size = 0.7, random_state = 1)\n",
    "        #reg = XGBRegressor()\n",
    "        #reg = Ridge(alpha=i)\n",
    "        reg = RandomForestRegressor(n_estimators = 100, max_depth = md, max_features = mf, n_jobs = -1, random_state=1)\n",
    "        reg.fit(train_n[train_features], train_n.Sales)\n",
    "        preds = reg.predict(test_n[train_features])\n",
    "        print mf, md, RMSPE(test_n.Sales, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15 30 0.299623998979\n",
      "15 None 0.298999015671\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-322-4edc64affff4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0;31m#reg = Ridge(alpha=i)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0mreg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRandomForestRegressor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_estimators\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'friedman_mse'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_depth\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_features\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m         \u001b[0mreg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_n\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtrain_features\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_n\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSales\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m         \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_n\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtrain_features\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0;32mprint\u001b[0m \u001b[0mmf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mRMSPE\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_n\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSales\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpreds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sklearn/ensemble/forest.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    271\u001b[0m                     \u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrees\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    272\u001b[0m                     verbose=self.verbose, class_weight=self.class_weight)\n\u001b[0;32m--> 273\u001b[0;31m                 for i, t in enumerate(trees))\n\u001b[0m\u001b[1;32m    274\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    275\u001b[0m             \u001b[0;31m# Collect newly grown trees\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m    664\u001b[0m                 \u001b[0;31m# consumption.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    665\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 666\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    667\u001b[0m             \u001b[0;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    668\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sklearn/externals/joblib/parallel.pyc\u001b[0m in \u001b[0;36mretrieve\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    516\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    517\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 518\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    519\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexceptions\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    520\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/multiprocessing/pool.pyc\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    559\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    560\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 561\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    562\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ready\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    563\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/multiprocessing/pool.pyc\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    554\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    555\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ready\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 556\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    557\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    558\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/threading.pyc\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    338\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m    \u001b[0;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    339\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 340\u001b[0;31m                 \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    341\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0m__debug__\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    342\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_note\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"%s.wait(): got it\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "maxf = [15, 18]\n",
    "maxd = [30, None]\n",
    "for mf in maxf:\n",
    "    for md in maxd:\n",
    "        train_n, test_n = train_test_split(train_new[train_features + ['Sales']], train_size = 0.7, random_state = 1)\n",
    "        #reg = XGBRegressor()\n",
    "        #reg = Ridge(alpha=i)\n",
    "        reg = RandomForestRegressor(n_estimators = 100, criterion='friedman_mse', max_depth = md, max_features = mf, n_jobs = -1, random_state=1)\n",
    "        reg.fit(train_n[train_features], train_n.Sales)\n",
    "        preds = reg.predict(test_n[train_features])\n",
    "        print mf, md, RMSPE(test_n.Sales, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Store  DayOfWeek  Promo\n",
      "1      1          0        3893.306452\n",
      "                  1        5852.708333\n",
      "       2          0        3835.285714\n",
      "                  1        5364.597222\n",
      "       3          0        3746.746032\n",
      "                  1        5073.736111\n",
      "Name: Sales, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "means = train_new.groupby([ 'Store', 'DayOfWeek', 'Promo' ])['Sales'].mean()\n",
    "print(means.head(6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "means = means.reset_index()\n",
    "print(means.head(6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Store</th>\n",
       "      <th>DayOfWeek</th>\n",
       "      <th>Promo</th>\n",
       "      <th>Mean_Sales</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3893.306452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5852.708333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3835.285714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>5364.597222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3746.746032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>5073.736111</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Store  DayOfWeek  Promo   Mean_Sales\n",
       "0      1          1      0  3893.306452\n",
       "1      1          1      1  5852.708333\n",
       "2      1          2      0  3835.285714\n",
       "3      1          2      1  5364.597222\n",
       "4      1          3      0  3746.746032\n",
       "5      1          3      1  5073.736111"
      ]
     },
     "execution_count": 334,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "means.head(6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "means.columns = [u'Store', u'DayOfWeek', u'Promo', u'Mean_Sales']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_nnew = pd.merge(test_new, means, on = ['Store','DayOfWeek','Promo'], how='left')\n",
    "test_nnew.fillna(train.Sales.mean(), inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_nnew = pd.merge(train_new, means, on = ['Store','DayOfWeek','Promo'], how='left')\n",
    "train_nnew.fillna(train.Sales.mean(), inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_features = [u'Open', u'Promo', u'SchoolHoliday', u'StateHoliday_0',\n",
    "       u'StateHoliday_a', u'DayOfWeek_1', u'DayOfWeek_2', u'DayOfWeek_3',\n",
    "       u'DayOfWeek_4', u'DayOfWeek_5', u'DayOfWeek_6', u'DayOfWeek_7',\n",
    "       u'CompetitionDistance', u'Promo2', u'Mean_Sales',\n",
    "       u'StoreType_a', u'StoreType_b', u'StoreType_c', u'StoreType_d',\n",
    "       u'Assortment_a', u'Assortment_b', u'Assortment_c', u'CompetitionOpen']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_nnew.to_csv('test_nnew.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_nnew.to_csv('train_nnew.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "reg = RandomForestRegressor(n_estimators = 500, max_depth = None, max_features = 'auto', n_jobs = -1, random_state=1)\n",
    "reg.fit(train_nnew[train_features], train_nnew.Sales)\n",
    "preds = reg.predict(test_nnew[train_features])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "res = pd.DataFrame(columns=['Id', 'Sales'])\n",
    "\n",
    "res.Id = test_new.Id\n",
    "\n",
    "res.Sales = preds\n",
    "\n",
    "res.to_csv('rf-2.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "features = [u'Store', u'DayOfWeek', u'Date', u'Open', u'Promo', u'StateHoliday', u'SchoolHoliday']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_n, test_n = train_test_split(train[features + ['Sales']], train_size = 0.7, random_state = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def RMSPE(true, pred):\n",
    "    df = pd.DataFrame()\n",
    "    df['true'] = true\n",
    "    df['pred'] = pred\n",
    "    df_n = df[df['true'] != 0]\n",
    "    return np.sqrt((1/float(len(df_n.true)))*np.sum(((df_n.true - df_n.pred)/df_n.true)**2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from xgboost import XGBRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.614032200069\n",
      "0.614117136846\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-126-227e011cc24a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     36\u001b[0m     \u001b[0;31m#reg = XGBRegressor()\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m     \u001b[0mreg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRidge\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0malpha\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m     \u001b[0mreg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_l\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfeats\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_l\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSales\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m     \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_l\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfeats\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m     \u001b[0merrs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mRMSPE\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_l\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSales\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpreds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sklearn/linear_model/ridge.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    508\u001b[0m         \u001b[0mself\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0mreturns\u001b[0m \u001b[0man\u001b[0m \u001b[0minstance\u001b[0m \u001b[0mof\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    509\u001b[0m         \"\"\"\n\u001b[0;32m--> 510\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mRidge\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    511\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    512\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sklearn/linear_model/ridge.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    388\u001b[0m         X, y, X_mean, y_mean, X_std = self._center_data(\n\u001b[1;32m    389\u001b[0m             \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_intercept\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnormalize\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy_X\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 390\u001b[0;31m             sample_weight=sample_weight)\n\u001b[0m\u001b[1;32m    391\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    392\u001b[0m         \u001b[0msolver\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_deprecate_dense_cholesky\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msolver\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sklearn/linear_model/base.pyc\u001b[0m in \u001b[0;36mcenter_data\u001b[0;34m(X, y, fit_intercept, normalize, copy, sample_weight)\u001b[0m\n\u001b[1;32m     88\u001b[0m     \u001b[0;32mis\u001b[0m \u001b[0mzero\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mmean\u001b[0m \u001b[0mitself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m     \"\"\"\n\u001b[0;32m---> 90\u001b[0;31m     \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mas_float_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     91\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfit_intercept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnumbers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNumber\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sklearn/utils/validation.pyc\u001b[0m in \u001b[0;36mas_float_array\u001b[0;34m(X, copy, force_all_finite)\u001b[0m\n\u001b[1;32m     91\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcopy\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat64\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# is numpy array\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'F'\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflags\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'F_CONTIGUOUS'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m'C'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcopy\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint32\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat64\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "bstrap = 1\n",
    "\n",
    "\n",
    "vals = np.linspace(5000000, 6000000, num = 10)\n",
    "errs = []\n",
    "\n",
    "for i in vals:\n",
    "    features = [u'Store', u'DayOfWeek', u'Open', u'Promo', u'StateHoliday', u'SchoolHoliday']\n",
    "    feats = [u'Store', u'DayOfWeek', u'Promo', u'StateHoliday', u'SchoolHoliday']\n",
    "    catfeats = [u'DayOfWeek', u'StateHoliday']\n",
    "    train_nn, test_n = train_test_split(train[features + ['Sales']], train_size = 0.7, random_state = 1)\n",
    "    train_n = train_nn[(train_nn.Open != 0) * (train_nn.Sales != 0)]\n",
    "    train_l = train_n.copy()\n",
    "    test_l = test_n.copy()\n",
    "    encoders = {}\n",
    "    ohe = {}\n",
    "    for c in catfeats:\n",
    "        encoders[c] = LabelEncoder()\n",
    "        ohe[c] = OneHotEncoder(sparse=False)\n",
    "        encoders[c].fit(train_n[c])\n",
    "        x = encoders[c].transform(train_n[c])\n",
    "        xx = encoders[c].transform(test_n[c])\n",
    "        ohe[c].fit(x.reshape((len(x), 1)))\n",
    "        y = ohe[c].transform(x.reshape((len(x), 1)))\n",
    "        yy = ohe[c].transform(xx.reshape((len(xx), 1)))\n",
    "        features.remove(c)\n",
    "        feats.remove(c)\n",
    "        for z in xrange(y.shape[1]):\n",
    "            train_l[c+'_'+str(z)] = y[:,z]\n",
    "            test_l[c+'_'+str(z)] = yy[:,z]\n",
    "            features.append(c+'_'+str(z))\n",
    "            feats.append(c+'_'+str(z))\n",
    "        #train_l[c] = \n",
    "        #test_l[c] = encoders[c].transform(test_n[c])\n",
    "    \n",
    "    #reg = XGBRegressor()\n",
    "    reg = Ridge(alpha=i)\n",
    "    reg.fit(train_l[feats], train_l.Sales)\n",
    "    preds = reg.predict(test_l[feats])\n",
    "    errs.append(RMSPE(test_l.Sales, preds))\n",
    "    print errs[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.plot(vals, errs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/ipykernel/__main__.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  if __name__ == '__main__':\n"
     ]
    }
   ],
   "source": [
    "test_l.Open[np.isnan(test_l.Open)] = np.ones(len(test_l.Open[np.isnan(test_l.Open)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/ipykernel/__main__.py:14: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "features = [u'Store', u'DayOfWeek', u'Open', u'Promo', u'StateHoliday', u'SchoolHoliday']\n",
    "catfeats = [u'DayOfWeek', u'StateHoliday']\n",
    "train_n = train[features + ['Sales']]\n",
    "test_n = test[features]\n",
    "train_l = train_n.copy()\n",
    "test_l = test_n.copy()\n",
    "encoders = {}\n",
    "for c in catfeats:\n",
    "    encoders[c] = LabelEncoder()\n",
    "    encoders[c].fit(train_n[c])\n",
    "    train_l[c] = encoders[c].transform(train_n[c])\n",
    "    test_l[c] = encoders[c].transform(test_n[c])\n",
    "\n",
    "test_l.Open[np.isnan(test_l.Open)] = np.ones(len(test_l.Open[np.isnan(test_l.Open)]))\n",
    "reg = Ridge(alpha = 5380500)\n",
    "reg.fit(train_l[features], train_l.Sales)\n",
    "preds = reg.predict(test_l[features])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "res = pd.DataFrame(columns=['Id', 'Sales'])\n",
    "\n",
    "res.Id = test.Id\n",
    "\n",
    "res.Sales = preds\n",
    "\n",
    "res.to_csv('dummy.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4473684.2105263155, 5263157.8947368423)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vals[5], vals[7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "reg = Ridge()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "encl = LabelEncoder()\n",
    "enc = OneHotEncoder(sparse = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LabelEncoder()"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encl.fit(train_n.DayOfWeek)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "n = encl.transform(train_n.DayOfWeek)\n",
    "nn = encl.transform(test_n.DayOfWeek)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0],\n",
       "       [3],\n",
       "       [0],\n",
       "       ..., \n",
       "       [0],\n",
       "       [3],\n",
       "       [2]])"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n.reshape((712046, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OneHotEncoder(categorical_features='all', dtype=<type 'float'>,\n",
       "       handle_unknown='error', n_values='auto', sparse=False)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "enc.fit(n.reshape((712046, 1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "y = enc.transform(n.reshape((712046, 1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  0.,  0., ...,  0.,  0.,  0.])"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "z = enc.transform(nn.reshape((len(nn), 1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(305163, 7)"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sklearn/preprocessing/data.py:1150: VisibleDeprecationWarning: boolean index did not match indexed array along dimension 0; dimension is 712046 but corresponding boolean dimension is 5696368\n",
      "  \"during transform.\" % X[~mask])\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "index 712050 is out of bounds for axis 0 with size 712046",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-47-9d470bd91b1b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0menc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_n\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sklearn/preprocessing/data.pyc\u001b[0m in \u001b[0;36mtransform\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m   1176\u001b[0m         \"\"\"\n\u001b[1;32m   1177\u001b[0m         return _transform_selected(X, self._transform,\n\u001b[0;32m-> 1178\u001b[0;31m                                    self.categorical_features, copy=True)\n\u001b[0m",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sklearn/preprocessing/data.pyc\u001b[0m in \u001b[0;36m_transform_selected\u001b[0;34m(X, transform, selected, copy)\u001b[0m\n\u001b[1;32m    934\u001b[0m     \"\"\"\n\u001b[1;32m    935\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mselected\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"all\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 936\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    937\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    938\u001b[0m     \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'csc'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sklearn/preprocessing/data.pyc\u001b[0m in \u001b[0;36m_transform\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m   1148\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandle_unknown\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'error'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1149\u001b[0m                 raise ValueError(\"unknown categorical feature present %s \"\n\u001b[0;32m-> 1150\u001b[0;31m                                  \"during transform.\" % X[~mask])\n\u001b[0m\u001b[1;32m   1151\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1152\u001b[0m         \u001b[0mcolumn_indices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mindices\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mravel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: index 712050 is out of bounds for axis 0 with size 712046"
     ]
    }
   ],
   "source": [
    "enc.transform(train_n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "could not convert string to float: a",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-39-454f45c94857>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mreg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_n\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_n\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Sales'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sklearn/linear_model/ridge.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    508\u001b[0m         \u001b[0mself\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0mreturns\u001b[0m \u001b[0man\u001b[0m \u001b[0minstance\u001b[0m \u001b[0mof\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    509\u001b[0m         \"\"\"\n\u001b[0;32m--> 510\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mRidge\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    511\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    512\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sklearn/linear_model/ridge.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    380\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    381\u001b[0m         X, y = check_X_y(X, y, ['csr', 'csc', 'coo'], dtype=np.float,\n\u001b[0;32m--> 382\u001b[0;31m                          multi_output=True, y_numeric=True)\n\u001b[0m\u001b[1;32m    383\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    384\u001b[0m         if ((sample_weight is not None) and\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sklearn/utils/validation.pyc\u001b[0m in \u001b[0;36mcheck_X_y\u001b[0;34m(X, y, accept_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, multi_output, ensure_min_samples, ensure_min_features, y_numeric)\u001b[0m\n\u001b[1;32m    442\u001b[0m     X = check_array(X, accept_sparse, dtype, order, copy, force_all_finite,\n\u001b[1;32m    443\u001b[0m                     \u001b[0mensure_2d\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mallow_nd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mensure_min_samples\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 444\u001b[0;31m                     ensure_min_features)\n\u001b[0m\u001b[1;32m    445\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mmulti_output\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    446\u001b[0m         y = check_array(y, 'csr', force_all_finite=True, ensure_2d=False,\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sklearn/utils/validation.pyc\u001b[0m in \u001b[0;36mcheck_array\u001b[0;34m(array, accept_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features)\u001b[0m\n\u001b[1;32m    342\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    343\u001b[0m                 \u001b[0mdtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 344\u001b[0;31m         \u001b[0marray\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    345\u001b[0m         \u001b[0;31m# make sure we actually converted to numeric:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    346\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdtype_numeric\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"O\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: could not convert string to float: a"
     ]
    }
   ],
   "source": [
    "reg.fit(train_n[features], train_n['Sales'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}