TalkingBattle Models-Keras.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "from scipy.sparse import csr_matrix, hstack\n",
    "from datetime import date\n",
    "mingw_path = 'C:\\\\mingw64\\\\bin'\n",
    "os.environ['PATH'] = mingw_path + ';' + os.environ['PATH']\n",
    "\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "import xgboost as xgb\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def rstr(df): return df.dtypes, df.head(3) ,df.apply(lambda x: [x.unique()]), df.apply(lambda x: [len(x.unique())]),df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def batch_generator(X, y, batch_size, shuffle):\n",
    "    #chenglong code for fiting from generator (https://www.kaggle.com/c/talkingdata-mobile-user-demographics/forums/t/22567/neural-network-for-sparse-matrices)\n",
    "    number_of_batches = np.ceil(X.shape[0]/batch_size)\n",
    "    counter = 0\n",
    "    sample_index = np.arange(X.shape[0])\n",
    "    if shuffle:\n",
    "        np.random.shuffle(sample_index)\n",
    "    while True:\n",
    "        batch_index = sample_index[batch_size*counter:batch_size*(counter+1)]\n",
    "        X_batch = X[batch_index,:].toarray()\n",
    "        y_batch = y[batch_index]\n",
    "        counter += 1\n",
    "        yield X_batch, y_batch\n",
    "        if (counter == number_of_batches):\n",
    "            if shuffle:\n",
    "                np.random.shuffle(sample_index)\n",
    "            counter = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def batch_generatorp(X, batch_size, shuffle):\n",
    "    number_of_batches = X.shape[0] / np.ceil(X.shape[0]/batch_size)\n",
    "    counter = 0\n",
    "    sample_index = np.arange(X.shape[0])\n",
    "    while True:\n",
    "        batch_index = sample_index[batch_size * counter:batch_size * (counter + 1)]\n",
    "        X_batch = X[batch_index, :].toarray()\n",
    "        counter += 1\n",
    "        yield X_batch\n",
    "        if (counter == number_of_batches):\n",
    "            counter = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seed = 1851\n",
    "np.random.seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "datadir = ''\n",
    "gatrain = pd.read_csv(os.path.join(datadir,'gender_age_train.csv'),\n",
    "                      index_col='device_id')\n",
    "gatest = pd.read_csv(os.path.join(datadir,'gender_age_test.csv'),\n",
    "                     index_col = 'device_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "gatrain['trainrow'] = np.arange(gatrain.shape[0])\n",
    "gatest['testrow'] = np.arange(gatest.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "gatrain.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "phone = pd.read_csv(os.path.join(datadir,'phone_brand_device_model.csv'))\n",
    "phone.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "phone.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "phone.duplicated().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "phone = phone.drop_duplicates('device_id',keep='first').set_index('device_id')\n",
    "phone.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "phone.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "events = pd.read_csv(os.path.join(datadir,'events.csv'),\n",
    "                     parse_dates=['timestamp'])\n",
    "events.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "appevents = pd.read_csv(os.path.join(datadir,'app_events.csv'),\n",
    "                        usecols=['event_id','app_id','is_active'],\n",
    "                        dtype={'is_active':bool})\n",
    "appevents.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "applabels = pd.read_csv(os.path.join(datadir,'app_labels.csv'))\n",
    "applabels.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "labelcategories = pd.read_csv(os.path.join(datadir,'label_categories.csv'))\n",
    "labelcategories.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print('Number of unique categores: {}, Number of unique categores in apps {}, Number of categories that match {}'.format(labelcategories.shape[0],len(applabels.label_id.unique()),labelcategories.label_id.isin(applabels.label_id).sum()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Device Brand"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "brandencoder = LabelEncoder().fit(phone.phone_brand)\n",
    "phone['brand'] = brandencoder.transform(phone['phone_brand'])\n",
    "gatrain['brand'] = phone['brand']\n",
    "gatest['brand'] = phone['brand']\n",
    "Xtr_brand = csr_matrix((np.ones(gatrain.shape[0]), \n",
    "                       (gatrain.trainrow, gatrain.brand)))\n",
    "Xte_brand = csr_matrix((np.ones(gatest.shape[0]), \n",
    "                       (gatest.testrow, gatest.brand)))\n",
    "print('Brand features: train shape {}, test shape {}'.format(Xtr_brand.shape, Xte_brand.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Device Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "m = phone.phone_brand.str.cat(phone.device_model)\n",
    "m.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "modelencoder = LabelEncoder().fit(m)\n",
    "phone['model'] = modelencoder.transform(m)\n",
    "gatrain['model'] = phone['model']\n",
    "gatest['model'] = phone['model']\n",
    "Xtr_model = csr_matrix((np.ones(gatrain.shape[0]), \n",
    "                       (gatrain.trainrow, gatrain.model)))\n",
    "Xte_model = csr_matrix((np.ones(gatest.shape[0]), \n",
    "                       (gatest.testrow, gatest.model)))\n",
    "print('Model features: train shape {}, test shape {}'.format(Xtr_model.shape, Xte_model.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "appencoder = LabelEncoder().fit(appevents.app_id)\n",
    "appevents['app'] = appencoder.transform(appevents.app_id)\n",
    "napps = len(appencoder.classes_)\n",
    "appevents.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "events.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "latLong = pd.read_csv(\"latLong.csv\")\n",
    "latLong.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "events = events.merge(latLong[['longitude','latitude','regionLabel']],how='left',left_on=['longitude','latitude'],right_on=['longitude','latitude'])\n",
    "events.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "events = events.drop(['longitude','latitude'],axis=1)\n",
    "events.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "events.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "events['day_of_week']=events.timestamp.dt.dayofweek\n",
    "events['time_of_day']=events.timestamp.dt.hour\n",
    "events['period']=pd.cut(events.time_of_day,bins=[0,5,12,17,21,24],right=False,labels=[0,1,2,3,4])\n",
    "events['period'].replace(4,0,inplace=True)\n",
    "# 0:Night,1:Morning,2:Afternoon,3:Evening\n",
    "events =events.drop(['time_of_day','timestamp'],axis=1)\n",
    "events.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "appevents.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "appencoder = LabelEncoder().fit(appevents.app_id)\n",
    "appevents['app'] = appencoder.transform(appevents.app_id)\n",
    "napps = len(appencoder.classes_)\n",
    "appevents.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "deviceapps=appevents.merge(events,how='left',left_on=['event_id'],right_on=['event_id'])\n",
    "deviceapps.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Apps and Activity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "appactivity = deviceapps.groupby(['device_id','app'])['is_active'].agg(['size',np.sum,np.mean]).reset_index()\n",
    "appactivity=(appactivity.merge(gatrain[['trainrow']],how='left',left_on='device_id',right_index=True)\n",
    ".merge(gatest[['testrow']],how='left',left_on='device_id',right_index=True))\n",
    "appactivity.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = appactivity.dropna(subset=['trainrow'])\n",
    "Xtr_app = csr_matrix((np.ones(d.shape[0]), (d.trainrow, d.app)), \n",
    "                      shape=(gatrain.shape[0],napps))\n",
    "d = appactivity.dropna(subset=['testrow'])\n",
    "Xte_app = csr_matrix((np.ones(d.shape[0]), (d.testrow, d.app)), \n",
    "                      shape=(gatest.shape[0],napps))\n",
    "print('Apps data: train shape {}, test shape {}'.format(Xtr_app.shape, Xte_app.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = appactivity.dropna(subset=['trainrow'])\n",
    "Xtr_app_size = csr_matrix((d['size'], (d.trainrow, d.app)), \n",
    "                      shape=(gatrain.shape[0],napps))\n",
    "d = appactivity.dropna(subset=['testrow'])\n",
    "Xte_app_size = csr_matrix((d['size'], (d.testrow, d.app)), \n",
    "                      shape=(gatest.shape[0],napps))\n",
    "print('Apps data: train shape {}, test shape {}'.format(Xtr_app_size.shape, Xte_app_size.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = appactivity.dropna(subset=['trainrow'])\n",
    "Xtr_app_act = csr_matrix((np.log1p(d['sum']), (d.trainrow, d.app)), \n",
    "                      shape=(gatrain.shape[0],napps))\n",
    "d = appactivity.dropna(subset=['testrow'])\n",
    "Xte_app_act = csr_matrix((np.log1p(d['sum']), (d.testrow, d.app)), \n",
    "                      shape=(gatest.shape[0],napps))\n",
    "print('Apps data: train shape {}, test shape {}'.format(Xtr_app_act.shape, Xte_app_act.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = appactivity.dropna(subset=['trainrow'])\n",
    "Xtr_app_act_m = csr_matrix((d['mean'], (d.trainrow, d.app)), \n",
    "                      shape=(gatrain.shape[0],napps))\n",
    "d = appactivity.dropna(subset=['testrow'])\n",
    "Xte_app_act_m = csr_matrix((d['mean'], (d.testrow, d.app)), \n",
    "                      shape=(gatest.shape[0],napps))\n",
    "print('Apps data: train shape {}, test shape {}'.format(Xtr_app_act_m.shape, Xte_app_act_m.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Location and Activity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "locactivity = deviceapps.groupby(['device_id','regionLabel'])['is_active'].agg(['size',np.sum,np.mean]).reset_index()\n",
    "locactivity=(locactivity.merge(gatrain[['trainrow']],how='left',left_on='device_id',right_index=True)\n",
    ".merge(gatest[['testrow']],how='left',left_on='device_id',right_index=True))\n",
    "locactivity.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "locEncoder = LabelEncoder().fit(locactivity['regionLabel'])\n",
    "locactivity['loc'] = locEncoder.transform(locactivity['regionLabel'])\n",
    "nlocs = len(locEncoder.classes_)\n",
    "locactivity.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = locactivity.dropna(subset=['trainrow'])\n",
    "Xtr_loc = csr_matrix((np.ones(d.shape[0]), (d.trainrow, d['loc'])), \n",
    "                      shape=(gatrain.shape[0],nlocs))\n",
    "d = locactivity.dropna(subset=['testrow'])\n",
    "Xte_loc = csr_matrix((np.ones(d.shape[0]), (d.testrow, d['loc'])), \n",
    "                      shape=(gatest.shape[0],nlocs))\n",
    "print('Apps data: train shape {}, test shape {}'.format(Xtr_loc.shape, Xte_loc.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = locactivity.dropna(subset=['trainrow'])\n",
    "Xtr_loc_size = csr_matrix((d['size'], (d.trainrow, d['loc'])), \n",
    "                      shape=(gatrain.shape[0],nlocs))\n",
    "d = locactivity.dropna(subset=['testrow'])\n",
    "Xte_loc_size = csr_matrix((d['size'], (d.testrow, d['loc'])), \n",
    "                      shape=(gatest.shape[0],nlocs))\n",
    "print('Apps data: train shape {}, test shape {}'.format(Xtr_loc_size.shape, Xte_loc_size.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = locactivity.dropna(subset=['trainrow'])\n",
    "Xtr_loc_act = csr_matrix((np.log1p(d['sum']), (d.trainrow, d['loc'])), \n",
    "                      shape=(gatrain.shape[0],nlocs))\n",
    "d = locactivity.dropna(subset=['testrow'])\n",
    "Xte_loc_act = csr_matrix((np.log1p(d['sum']), (d.testrow, d['loc'])), \n",
    "                      shape=(gatest.shape[0],nlocs))\n",
    "print('Apps data: train shape {}, test shape {}'.format(Xtr_loc_act.shape, Xte_loc_act.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = locactivity.dropna(subset=['trainrow'])\n",
    "Xtr_loc_act_m = csr_matrix((d['mean'], (d.trainrow, d['loc'])), \n",
    "                      shape=(gatrain.shape[0],nlocs))\n",
    "d = locactivity.dropna(subset=['testrow'])\n",
    "Xte_loc_act_m = csr_matrix((d['mean'], (d.testrow, d['loc'])), \n",
    "                      shape=(gatest.shape[0],nlocs))\n",
    "print('Apps data: train shape {}, test shape {}'.format(Xtr_loc_act_m.shape, Xte_loc_act_m.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Week day and Activity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "weekactivity = deviceapps.groupby(['device_id','day_of_week'])['is_active'].agg(['size',np.sum,np.mean]).reset_index()\n",
    "weekactivity=(weekactivity.merge(gatrain[['trainrow']],how='left',left_on='device_id',right_index=True)\n",
    ".merge(gatest[['testrow']],how='left',left_on='device_id',right_index=True))\n",
    "weekactivity.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = weekactivity.dropna(subset=['trainrow'])\n",
    "Xtr_week = csr_matrix((np.ones(d.shape[0]), (d.trainrow, d['day_of_week'])), \n",
    "                      shape=(gatrain.shape[0],7))\n",
    "d = weekactivity.dropna(subset=['testrow'])\n",
    "Xte_week = csr_matrix((np.ones(d.shape[0]), (d.testrow, d['day_of_week'])), \n",
    "                      shape=(gatest.shape[0],7))\n",
    "print('Apps data: train shape {}, test shape {}'.format(Xtr_week.shape, Xte_week.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = weekactivity.dropna(subset=['trainrow'])\n",
    "Xtr_week_size = csr_matrix((d['size'], (d.trainrow, d['day_of_week'])), \n",
    "                      shape=(gatrain.shape[0],7))\n",
    "d = weekactivity.dropna(subset=['testrow'])\n",
    "Xte_week_size = csr_matrix((d['size'], (d.testrow, d['day_of_week'])), \n",
    "                      shape=(gatest.shape[0],7))\n",
    "print('Apps data: train shape {}, test shape {}'.format(Xtr_week_size.shape, Xte_week_size.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = weekactivity.dropna(subset=['trainrow'])\n",
    "Xtr_week_act = csr_matrix((np.log1p(d['sum']), (d.trainrow, d['day_of_week'])), \n",
    "                      shape=(gatrain.shape[0],7))\n",
    "d = weekactivity.dropna(subset=['testrow'])\n",
    "Xte_week_act = csr_matrix((np.log1p(d['sum']), (d.testrow, d['day_of_week'])), \n",
    "                      shape=(gatest.shape[0],7))\n",
    "print('Apps data: train shape {}, test shape {}'.format(Xtr_week_act.shape, Xte_week_act.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = weekactivity.dropna(subset=['trainrow'])\n",
    "Xtr_week_act_m = csr_matrix((d['mean'], (d.trainrow, d['day_of_week'])), \n",
    "                      shape=(gatrain.shape[0],7))\n",
    "d = weekactivity.dropna(subset=['testrow'])\n",
    "Xte_week_act_m = csr_matrix((d['mean'], (d.testrow, d['day_of_week'])), \n",
    "                      shape=(gatest.shape[0],7))\n",
    "print('Apps data: train shape {}, test shape {}'.format(Xtr_week_act_m.shape, Xte_week_act_m.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Day time & Activity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "periodactivity = deviceapps.groupby(['device_id','period'])['is_active'].agg(['size',np.sum,np.mean]).reset_index()\n",
    "periodactivity=(periodactivity.merge(gatrain[['trainrow']],how='left',left_on='device_id',right_index=True)\n",
    ".merge(gatest[['testrow']],how='left',left_on='device_id',right_index=True))\n",
    "periodactivity.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = periodactivity.dropna(subset=['trainrow'])\n",
    "Xtr_period = csr_matrix((np.ones(d.shape[0]), (d.trainrow, d['period'])), \n",
    "                      shape=(gatrain.shape[0],4))\n",
    "d = periodactivity.dropna(subset=['testrow'])\n",
    "Xte_period = csr_matrix((np.ones(d.shape[0]), (d.testrow, d['period'])), \n",
    "                      shape=(gatest.shape[0],4))\n",
    "print('Apps data: train shape {}, test shape {}'.format(Xtr_period.shape, Xte_period.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = periodactivity.dropna(subset=['trainrow'])\n",
    "Xtr_period_size = csr_matrix((d['size'], (d.trainrow, d['period'])), \n",
    "                      shape=(gatrain.shape[0],4))\n",
    "d = periodactivity.dropna(subset=['testrow'])\n",
    "Xte_period_size = csr_matrix((d['size'], (d.testrow, d['period'])), \n",
    "                      shape=(gatest.shape[0],4))\n",
    "print('Apps data: train shape {}, test shape {}'.format(Xtr_period_size.shape, Xte_period_size.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = periodactivity.dropna(subset=['trainrow'])\n",
    "Xtr_period_act = csr_matrix((np.log1p(d['sum']), (d.trainrow, d['period'])), \n",
    "                      shape=(gatrain.shape[0],4))\n",
    "d = periodactivity.dropna(subset=['testrow'])\n",
    "Xte_period_act = csr_matrix((np.log1p(d['sum']), (d.testrow, d['period'])), \n",
    "                      shape=(gatest.shape[0],4))\n",
    "print('Apps data: train shape {}, test shape {}'.format(Xtr_period_act.shape, Xte_period_act.shape))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = periodactivity.dropna(subset=['trainrow'])\n",
    "Xtr_period_act_m = csr_matrix((d['mean'], (d.trainrow, d['period'])), \n",
    "                      shape=(gatrain.shape[0],4))\n",
    "d = periodactivity.dropna(subset=['testrow'])\n",
    "Xte_period_act_m = csr_matrix((d['mean'], (d.testrow, d['period'])), \n",
    "                      shape=(gatest.shape[0],4))\n",
    "print('Apps data: train shape {}, test shape {}'.format(Xtr_period_act_m.shape, Xte_period_act_m.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del appactivity\n",
    "del locactivity\n",
    "del weekactivity\n",
    "del periodactivity\n",
    "del events"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# App labels features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "applabels = applabels.loc[applabels.app_id.isin(appevents.app_id.unique())]\n",
    "applabels['app'] = appencoder.transform(applabels.app_id)\n",
    "labelencoder = LabelEncoder().fit(applabels.label_id)\n",
    "applabels['label'] = labelencoder.transform(applabels.label_id)\n",
    "nlabels = len(labelencoder.classes_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# del appevents\n",
    "gatrain.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#devicelabels = (deviceapps[['device_id','app']]\n",
    "#                .merge(applabels[['app','label']])\n",
    "#                .groupby(['device_id','label'])['app'].agg(['size'])\n",
    "#                .merge(gatrain[['trainrow']], how='left',  left_index=True, right_index=True)\n",
    "#                .merge(gatest[['testrow']], how='left', left_index=True, right_index=True)\n",
    "#               .reset_index())\n",
    "devicelabels = pd.read_csv(os.path.join(datadir,'devicelabels.csv'))\n",
    "devicelabels.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = devicelabels.dropna(subset=['trainrow'])\n",
    "Xtr_label = csr_matrix((np.ones(d.shape[0]), (d.trainrow, d.label)), \n",
    "                      shape=(gatrain.shape[0],nlabels))\n",
    "d = devicelabels.dropna(subset=['testrow'])\n",
    "Xte_label = csr_matrix((np.ones(d.shape[0]), (d.testrow, d.label)), \n",
    "                      shape=(gatest.shape[0],nlabels))\n",
    "print('Labels data: train shape {}, test shape {}'.format(Xtr_label.shape, Xte_label.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Join all features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Xtrain = hstack((Xtr_brand, Xtr_model, Xtr_app, Xtr_label, Xtr_loc, Xtr_period, Xtr_loc_act_m, Xtr_period_act_m, Xtr_loc_act, Xtr_period_act), format='csr')\n",
    "Xtest =  hstack((Xte_brand, Xte_model, Xte_app, Xte_label, Xte_loc, Xte_period, Xte_loc_act_m, Xte_period_act_m, Xte_loc_act, Xte_period_act), format='csr')\n",
    "print('All features: train shape {}, test shape {}'.format(Xtrain.shape, Xtest.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Xtrain_log = hstack((Xtr_brand, Xtr_model, Xtr_app, Xtr_label, Xtr_loc, Xtr_period), format='csr')\n",
    "Xtest_log =  hstack((Xte_brand, Xte_model, Xte_app, Xte_label, Xte_loc, Xte_period), format='csr')\n",
    "print('All features: train shape {}, test shape {}'.format(Xtrain.shape, Xtest.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def baseline_model():\n",
    "    # create model\n",
    "    model = Sequential()\n",
    "    #model.add(Dense(10, input_dim=Xtrain.shape[1], init='normal', activation='relu'))\n",
    "    #model.add(Dropout(0.2))\n",
    "    model.add(Dense(50, input_dim=Xtrain.shape[1], init='normal', activation='tanh'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(12, init='normal', activation='sigmoid'))\n",
    "    # Compile model\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy'])  #logloss\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "targetencoder = LabelEncoder().fit(gatrain.group)\n",
    "y = targetencoder.transform(gatrain.group)\n",
    "nclasses = len(targetencoder.classes_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "params = {}\n",
    "params['booster'] = 'gblinear'\n",
    "params['objective'] = \"multi:softprob\"\n",
    "params['eval_metric'] = 'mlogloss'\n",
    "params['eta'] = 0.01\n",
    "params['num_class'] = 12\n",
    "params['lambda'] = 5\n",
    "params['alpha'] = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "parameters = {'C':0.02,'l1_ratio':1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model=baseline_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def score(parameters, random_state = 0,mix={'logit':0.475,'xgb':0.475,'sgd':0.05}):\n",
    "    kf = StratifiedKFold(y, n_folds=5, shuffle=True, random_state=random_state)\n",
    "    pred = np.zeros((y.shape[0],nclasses))\n",
    "    pred_logistic = np.zeros((y.shape[0],nclasses))\n",
    "    pred_xgb = np.zeros((y.shape[0],nclasses))\n",
    "    pred_sgd = np.zeros((y.shape[0],nclasses))\n",
    "    logit_val = False\n",
    "    xgb_val = False\n",
    "    sgd_val = False\n",
    "    if mix['logit']>0:\n",
    "        logit_val = True\n",
    "    if mix['xgb']>0:\n",
    "        xgb_val = True\n",
    "    if mix['sgd']>0:\n",
    "        sgd_val = True\n",
    "    for itrain, itest in kf:\n",
    "        Xtr, Xte = Xtrain[itrain, :], Xtrain[itest, :]\n",
    "        #Xtr_log, Xte_log = Xtrain_log[itrain, :], Xtrain_log[itest, :]\n",
    "        ytr, yte = y[itrain], y[itest]\n",
    "        if logit_val:\n",
    "            print(\"Starting Logistic Regression\")\n",
    "            # Logistic Regression\n",
    "            clf1 = LogisticRegression(C=0.02, multi_class='multinomial',solver='lbfgs')\n",
    "            clf1.fit(Xtr, ytr)\n",
    "            pred_logistic[itest,:] = clf1.predict_proba(Xte)\n",
    "        if xgb_val:\n",
    "            print(\"Starting XGBoost\")\n",
    "            # XGBoost\n",
    "            d_train = xgb.DMatrix(Xtr, label=ytr)\n",
    "            d_valid = xgb.DMatrix(Xte, label=yte)\n",
    "            watchlist = [(d_train, 'train'), (d_valid, 'eval')]\n",
    "            clf2 = xgb.train(params, d_train, 1000, watchlist, early_stopping_rounds=10,verbose_eval=False)\n",
    "            pred_xgb[itest,:] = clf2.predict(d_valid)\n",
    "        if sgd_val:\n",
    "            # SGD Classifier\n",
    "            print(\"SGD Classifier\")\n",
    "            clf3 = SGDClassifier(loss='log',penalty='elasticnet',l1_ratio=parameters['l1_ratio'])\n",
    "            clf3.fit(Xtr, ytr)\n",
    "            pred_sgd[itest,:] = clf3.predict_proba(Xte)\n",
    "        # Downsize to one fold only for kernels\n",
    "        # Combine predictions\n",
    "        pred[itest,:] = (mix['logit']*pred_logistic[itest,:])+(mix['xgb']*pred_xgb[itest,:])+(mix['sgd']*pred_sgd[itest,:])\n",
    "        #return log_loss(yte, pred[itest, :])\n",
    "        print(\"Logistic {:.5f}, XGB {:.5f}, SGD {:.5f}, Average {:.5f}\".format(log_loss(yte, pred_logistic[itest,:]),log_loss(yte, pred_xgb[itest,:]),log_loss(yte, pred_sgd[itest,:]),log_loss(yte, pred[itest,:])), end=' ')\n",
    "        print('')\n",
    "    print('')\n",
    "    return log_loss(y, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Cs = np.logspace(-3,0,4)\n",
    "res = []\n",
    "for C in Cs:\n",
    "    res.append(score(LogisticRegression(C = C)))\n",
    "plt.semilogx(Cs, res,'-o');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "score(LogisticRegression(C=0.02))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "score(parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "score(parameters,mix=0.75)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "score(parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "score(parameters={'C':0.02,'l1_ratio':0.5},mix={'logit':0.475,'xgb':0.475,'sgd':0.05})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "regularization = np.arange(0.5,0.7,0.05)\n",
    "res = []\n",
    "for l1_ratio in regularization:\n",
    "    parameters['l1_ratio'] = l1_ratio\n",
    "    res.append(score(parameters,mix={'logit':0,'xgb':0,'sgd':1}))\n",
    "for l1_ratio,scores in zip(regularization,res):\n",
    "    print(\"Regularization {:.2f}, Score {:.5f}\".format(l1_ratio,scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "score(parameters,mix={'logit':0.5,'xgb':0.5,'sgd':0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "clf_logit = LogisticRegression(C=0.02, multi_class='multinomial',solver='lbfgs')\n",
    "clf_logit.fit(Xtrain,y)\n",
    "pred1 = clf_logit.predict_proba(Xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d_train = xgb.DMatrix(Xtrain, label=y)\n",
    "d_valid = xgb.DMatrix(Xtest)\n",
    "watchlist = [(d_train, 'train')]\n",
    "clf_xgb = xgb.train(params, d_train, 300, watchlist, early_stopping_rounds=10,verbose_eval=True)\n",
    "pred2 = clf_xgb.predict(d_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pred = pred1*0.5 + pred2*0.5\n",
    "pred = pd.DataFrame(pred, index = gatest.index, columns=targetencoder.classes_)\n",
    "pred.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred.to_csv('logit_xgb_subm.csv',index=True)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [py27]",
   "language": "python",
   "name": "Python [py27]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}