demo-terminal5-fc-attention.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import keras\n",
    "import tensorflow as tf\n",
    "from datetime import datetime\n",
    "from sklearn import datasets, linear_model\n",
    "from matplotlib import pyplot as plt\n",
    "from matplotlib import cm\n",
    "from keras.layers.core import Dense, Activation, Dropout\n",
    "from keras.layers.recurrent import LSTM\n",
    "from keras.layers import BatchNormalization, Input, Embedding, Concatenate, Conv1D, MaxPooling1D, Flatten, merge\n",
    "from keras.layers import merge, Concatenate, Permute, RepeatVector, Reshape\n",
    "from keras.models import Sequential, Model\n",
    "import keras.backend as K\n",
    "import statsmodels.formula.api as smf\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "# prevent tensorflow from allocating the entire GPU memory at once\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth=True\n",
    "sess = tf.Session(config=config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GLOBAL PARAMETERS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_LAGS = 10\n",
    "sel = [5,7] # weather features to use\n",
    "sel2 = [0,1,2,7] # eventlags featurs to use\n",
    "\n",
    "# word embeddings parameters\n",
    "#GLOVE_DIR = \"/home/fmpr/datasets/glove.6B/\"\n",
    "GLOVE_DIR = \"/mnt/sdb1/datasets/glove.6B/\"\n",
    "MAX_SEQUENCE_LENGTH = 350 #600\n",
    "MAX_NB_WORDS = 600 #5000\n",
    "EMBEDDING_DIM = 300 #300"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load weather data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading weather data...\n"
     ]
    }
   ],
   "source": [
    "print \"loading weather data...\"\n",
    "\n",
    "# load data\n",
    "df = pd.read_csv(\"central_park_weather.csv\")\n",
    "df = df.set_index(\"date\")\n",
    "\n",
    "# replace predefined values with NaN\n",
    "df = df.replace(99.99, np.nan)\n",
    "df = df.replace(999.9, np.nan)\n",
    "df = df.replace(9999.9, np.nan)\n",
    "\n",
    "# replace NaN with 0 for snow depth\n",
    "df[\"snow_depth\"] = df[\"snow_depth\"].fillna(0)\n",
    "\n",
    "# do interpolation for the remaining NaNs\n",
    "df = df.interpolate()\n",
    "\n",
    "# standardize data\n",
    "removed_mean = df.mean()\n",
    "removed_std = df.std()\n",
    "weather = (df - removed_mean) / removed_std"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load events data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading events data...\n"
     ]
    }
   ],
   "source": [
    "print \"loading events data...\"\n",
    "\n",
    "events = pd.read_csv(\"terminal5_events_preprocessed.tsv\", sep=\"\\t\")\n",
    "events.head()\n",
    "\n",
    "events['start_time'] = pd.to_datetime(events['start_time'], format='%Y-%m-%d %H:%M')\n",
    "events['date'] = events['start_time'].dt.strftime(\"%Y-%m-%d\")\n",
    "events = events[[\"date\",\"start_time\",\"title\",\"url\",\"description\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load taxi data (and merge with others and detrend)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading taxi data (and merging and detrending)...\n"
     ]
    }
   ],
   "source": [
    "print \"loading taxi data (and merging and detrending)...\"\n",
    "\n",
    "df = pd.read_csv(\"pickups_terminal_5_0.003.csv\")\n",
    "\n",
    "df_sum = pd.DataFrame(df.groupby(\"date\")[\"pickups\"].sum())\n",
    "df_sum[\"date\"] = df_sum.index\n",
    "df_sum.index = pd.to_datetime(df_sum.index, format='%Y-%m-%d %H:%M')\n",
    "df_sum[\"dow\"] = df_sum.index.weekday\n",
    "\n",
    "# add events information\n",
    "event_col = np.zeros((len(df_sum)))\n",
    "late_event = np.zeros((len(df_sum)))\n",
    "really_late_event = np.zeros((len(df_sum)))\n",
    "event_desc_col = []\n",
    "for i in xrange(len(df_sum)):\n",
    "    if df_sum.iloc[i].date in events[\"date\"].values:\n",
    "        event_col[i] = 1\n",
    "        event_descr = \"\"\n",
    "        for e in events[events.date == df_sum.iloc[i].date][\"description\"]:\n",
    "            event_descr += str(e) + \" \"\n",
    "        event_desc_col.append(event_descr)\n",
    "        for e in events[events.date == df_sum.iloc[i].date][\"start_time\"]:\n",
    "            if e.hour >= 20:\n",
    "                late_event[i] = 1\n",
    "            if e.hour >= 21:\n",
    "                really_late_event[i] = 1\n",
    "    else:\n",
    "        event_desc_col.append(\"None\")\n",
    "\n",
    "df_sum[\"event\"] = event_col\n",
    "df_sum[\"late_event\"] = late_event\n",
    "df_sum[\"really_late_event\"] = really_late_event\n",
    "df_sum[\"event_desc\"] = event_desc_col\n",
    "df_sum[\"event_next_day\"] = pd.Series(df_sum[\"event\"]).shift(-1)\n",
    "df_sum[\"late_event_next_day\"] = pd.Series(df_sum[\"late_event\"]).shift(-1)\n",
    "df_sum[\"really_late_event_next_day\"] = pd.Series(df_sum[\"really_late_event\"]).shift(-1)\n",
    "df_sum[\"event_next_day_desc\"] = pd.Series(df_sum[\"event_desc\"]).shift(-1)\n",
    "\n",
    "# merge with weather data\n",
    "df_sum = df_sum.join(weather, how='inner')\n",
    "df_sum.head()\n",
    "\n",
    "# keep only data after 2013\n",
    "START_YEAR = 2013\n",
    "df_sum = df_sum.loc[df_sum.index.year >= START_YEAR]\n",
    "df_sum.head()\n",
    "\n",
    "df_sum[\"year\"] = df_sum.index.year\n",
    "\n",
    "trend_mean = df_sum[df_sum.index.year < 2015].groupby([\"dow\"]).mean()[\"pickups\"]\n",
    "trend_std = df_sum[\"pickups\"].std()\n",
    "\n",
    "# build vectors with trend to remove and std\n",
    "trend = []\n",
    "std = []\n",
    "for ix, row in df_sum.iterrows():\n",
    "    trend.append(trend_mean[row.dow])\n",
    "    std.append(trend_std)\n",
    "\n",
    "df_sum[\"trend\"] = trend\n",
    "df_sum[\"std\"] = std\n",
    "\n",
    "# detrend data\n",
    "df_sum[\"detrended\"] = (df_sum[\"pickups\"] - df_sum[\"trend\"]) / df_sum[\"std\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build lags and features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "building lags...\n"
     ]
    }
   ],
   "source": [
    "print \"building lags...\"\n",
    "\n",
    "lags = pd.concat([pd.Series(df_sum[\"detrended\"]).shift(x) for x in range(0,NUM_LAGS)],axis=1).as_matrix()\n",
    "event_feats = np.concatenate([df_sum[\"event_next_day\"].as_matrix()[:,np.newaxis],\n",
    "                             df_sum[\"late_event\"].as_matrix()[:,np.newaxis],\n",
    "                             df_sum[\"really_late_event\"].as_matrix()[:,np.newaxis],\n",
    "                             df_sum[\"really_late_event_next_day\"].as_matrix()[:,np.newaxis]], axis=1)\n",
    "lags_event_feats = pd.concat([pd.Series(df_sum[\"event_next_day\"]).shift(x) for x in range(0,NUM_LAGS)],axis=1).as_matrix()\n",
    "event_texts = df_sum[\"event_next_day_desc\"].as_matrix()\n",
    "weather_feats = df_sum[['min_temp', u'max_temp', u'wind_speed',\n",
    "       u'wind_gust', u'visibility', u'pressure', u'precipitation',\n",
    "       u'snow_depth', u'fog', u'rain_drizzle', u'snow_ice', u'thunder']].as_matrix()\n",
    "preds = pd.Series(df_sum[\"detrended\"]).shift(-1).as_matrix()\n",
    "trends = df_sum[\"trend\"].as_matrix()\n",
    "stds = df_sum[\"std\"].as_matrix()\n",
    "\n",
    "lags = lags[NUM_LAGS:-1,:]\n",
    "event_feats = event_feats[NUM_LAGS:-1,:]\n",
    "lags_event_feats = lags_event_feats[NUM_LAGS:-1,:]\n",
    "event_texts = event_texts[NUM_LAGS:-1]\n",
    "weather_feats = weather_feats[NUM_LAGS:-1,:]\n",
    "preds = preds[NUM_LAGS:-1]\n",
    "trends = trends[NUM_LAGS:-1]\n",
    "stds = stds[NUM_LAGS:-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train/test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading train/val/test split...\n"
     ]
    }
   ],
   "source": [
    "print \"loading train/val/test split...\"\n",
    "\n",
    "i_train = 365*2 # 2013 and 2014\n",
    "i_val = 365*3\n",
    "i_test = -1 # 2015 and 2016 (everything else)\n",
    "\n",
    "lags_train = lags[:i_train,:] # time series lags\n",
    "event_feats_train = event_feats[:i_train,:] # event/no_event\n",
    "lags_event_feats_train = lags_event_feats[:i_train,:] # lags for event/no_event\n",
    "event_texts_train = event_texts[:i_train] # event text descriptions\n",
    "weather_feats_train = weather_feats[:i_train,:] # weather data\n",
    "y_train = preds[:i_train] # target values\n",
    "\n",
    "lags_val = lags[i_train:i_val,:] # time series lags\n",
    "event_feats_val = event_feats[i_train:i_val,:] # event/no_event\n",
    "lags_event_feats_val = lags_event_feats[i_train:i_val,:] # lags for event/no_event\n",
    "event_texts_val = event_texts[i_train:i_val] # event text descriptions\n",
    "weather_feats_val = weather_feats[i_train:i_val,:] # weather data\n",
    "y_val = preds[i_train:i_val] # target values\n",
    "\n",
    "lags_test = lags[i_val:i_test,:]\n",
    "event_feats_test = event_feats[i_val:i_test,:]\n",
    "lags_event_feats_test = lags_event_feats[i_val:i_test,:]\n",
    "event_texts_test = event_texts[i_val:i_test]\n",
    "weather_feats_test = weather_feats[i_val:i_test,:]\n",
    "y_test = preds[i_val:i_test]\n",
    "trend_test = trends[i_val:i_test]\n",
    "std_test = stds[i_val:i_test]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_error(trues, predicted):\n",
    "    corr = np.corrcoef(predicted, trues)[0,1]\n",
    "    mae = np.mean(np.abs(predicted - trues))\n",
    "    rae = np.sum(np.abs(predicted - trues)) / np.sum(np.abs(trues - np.mean(trues)))\n",
    "    rmse = np.sqrt(np.mean((predicted - trues)**2))\n",
    "    rrse = np.sqrt(np.sum((predicted - trues)**2) / np.sum((trues - np.mean(trues))**2))\n",
    "    mape = np.mean(np.abs((predicted - trues) / trues)) * 100\n",
    "    r2 = max(0, 1 - np.sum((predicted - trues)**2) / np.sum((trues - np.mean(trues))**2))\n",
    "    return corr, mae, rae, rmse, rrse, mape, r2\n",
    "\n",
    "\n",
    "def compute_error_filtered(trues, predicted, filt):\n",
    "    trues = trues[filt]\n",
    "    predicted = predicted[filt]\n",
    "    corr = np.corrcoef(predicted, trues)[0,1]\n",
    "    mae = np.mean(np.abs(predicted - trues))\n",
    "    mse = np.mean((predicted - trues)**2)\n",
    "    rae = np.sum(np.abs(predicted - trues)) / np.sum(np.abs(trues - np.mean(trues)))\n",
    "    rmse = np.sqrt(np.mean((predicted - trues)**2))\n",
    "    r2 = max(0, 1 - np.sum((trues-predicted)**2) / np.sum((trues - np.mean(trues))**2))\n",
    "    return corr, mae, rae, rmse, rrse, mape, r2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MLP (just lags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "running MLP with just lags...\n",
      "Total number of iterations:   500\n",
      "Best loss at iteratation:     129    Best: 0.4800713890219388\n",
      "Best val_loss at iteratation: 354    Best: 0.39486040791420085\n",
      "MAE:  183.819\tRMSE: 251.618\tR2:   0.439\n"
     ]
    }
   ],
   "source": [
    "def build_model(num_inputs, num_lags, num_preds):\n",
    "    input_lags = Input(shape=(num_lags,))\n",
    "    \n",
    "    x = input_lags\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dense(units=100, activation=\"tanh\", kernel_regularizer=keras.regularizers.l2(0.05))(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    preds = Dense(units=num_preds)(x)\n",
    "    \n",
    "    model = Model(input_lags, preds)\n",
    "    model.compile(loss=\"mse\", optimizer=\"adam\")\n",
    "    \n",
    "    return model, input_lags, preds\n",
    "\n",
    "\n",
    "print \"\\nrunning MLP with just lags...\"\n",
    "\n",
    "# checkpoint best model\n",
    "checkpoint = ModelCheckpoint(\"weights.best.hdf5\", monitor='val_loss', verbose=0, save_best_only=True, mode='min')\n",
    "\n",
    "model, input_lags, preds = build_model(1, NUM_LAGS, 1)\n",
    "model.fit(\n",
    "    np.concatenate([lags_train], axis=1),\n",
    "    y_train,\n",
    "    batch_size=64,\n",
    "    epochs=500,\n",
    "    validation_data=(np.concatenate([lags_val], axis=1), y_val),\n",
    "    callbacks=[checkpoint],\n",
    "    verbose=0)   \n",
    "\n",
    "print \"Total number of iterations:  \", len(model.history.history[\"loss\"])\n",
    "print \"Best loss at iteratation:    \", np.argmin(model.history.history[\"loss\"]), \"   Best:\", np.min(model.history.history[\"loss\"])\n",
    "print \"Best val_loss at iteratation:\", np.argmin(model.history.history[\"val_loss\"]), \"   Best:\", np.min(model.history.history[\"val_loss\"])\n",
    "\n",
    "# load weights\n",
    "model.load_weights(\"weights.best.hdf5\")\n",
    "\n",
    "# make predictions\n",
    "preds_lstm = model.predict(np.concatenate([lags_test[:,:]], axis=1))\n",
    "preds_lstm = preds_lstm[:,0] * std_test + trend_test\n",
    "y_true = y_test * std_test + trend_test\n",
    "corr, mae, rae, rmse, rrse, mape, r2 = compute_error(y_true, preds_lstm)\n",
    "print \"MAE:  %.3f\\tRMSE: %.3f\\tR2:   %.3f\" % (mae, rmse, r2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MLP lags + weather"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "running MLP with lags + weather...\n",
      "Total number of iterations:   500\n",
      "Best loss at iteratation:     433    Best: 0.47747223148607226\n",
      "Best val_loss at iteratation: 290    Best: 0.39909215014274807\n",
      "MAE:  183.542\tRMSE: 252.244\tR2:   0.437\n"
     ]
    }
   ],
   "source": [
    "print \"\\nrunning MLP with lags + weather...\"\n",
    "\n",
    "# checkpoint best model\n",
    "checkpoint = ModelCheckpoint(\"weights.best.hdf5\", monitor='val_loss', verbose=0, save_best_only=True, mode='min')\n",
    "\n",
    "model, input_lags, preds = build_model(1, NUM_LAGS+len(sel), 1)\n",
    "model.fit(\n",
    "    #lags_train,\n",
    "    np.concatenate([lags_train, weather_feats_train[:,sel]], axis=1),\n",
    "    y_train,\n",
    "    batch_size=64,\n",
    "    epochs=500,\n",
    "    validation_data=(np.concatenate([lags_val, weather_feats_val[:,sel]], axis=1), y_val),\n",
    "    callbacks=[checkpoint],\n",
    "    verbose=0)   \n",
    "\n",
    "print \"Total number of iterations:  \", len(model.history.history[\"loss\"])\n",
    "print \"Best loss at iteratation:    \", np.argmin(model.history.history[\"loss\"]), \"   Best:\", np.min(model.history.history[\"loss\"])\n",
    "print \"Best val_loss at iteratation:\", np.argmin(model.history.history[\"val_loss\"]), \"   Best:\", np.min(model.history.history[\"val_loss\"])\n",
    "\n",
    "# load weights\n",
    "model.load_weights(\"weights.best.hdf5\")\n",
    "\n",
    "# make predictions\n",
    "preds_lstm = model.predict(np.concatenate([lags_test[:,:], weather_feats_test[:,sel]], axis=1))\n",
    "preds_lstm = preds_lstm[:,0] * std_test + trend_test\n",
    "corr, mae, rae, rmse, rrse, mape, r2 = compute_error(y_true, preds_lstm)\n",
    "print \"MAE:  %.3f\\tRMSE: %.3f\\tR2:   %.3f\" % (mae, rmse, r2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MLP with weather + events information (no text) + late + event_lags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "running MLP with lags + weather + event + late + event lags...\n",
      "Total number of iterations:   500\n",
      "Best loss at iteratation:     309    Best: 0.4418567157771489\n",
      "Best val_loss at iteratation: 352    Best: 0.35749858013571123\n",
      "0.39762449772918923\n",
      "MAE:  169.099\tRMSE: 242.947\tR2:   0.477\n"
     ]
    }
   ],
   "source": [
    "print \"\\nrunning MLP with lags + weather + event + late + event lags...\"\n",
    "\n",
    "def build_model_events(num_inputs, num_lags, num_feat, num_preds):\n",
    "    input_lags = Input(shape=(num_lags,))\n",
    "    input_events = Input(shape=(num_feat,))\n",
    "    \n",
    "    feat = Concatenate(axis=1)([input_lags, input_events])\n",
    "    \n",
    "    x = feat\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dense(units=100, activation=\"tanh\", kernel_regularizer=keras.regularizers.l2(0.05))(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    \n",
    "    preds = Dense(units=num_preds)(x)\n",
    "    preds = Activation(\"linear\")(preds)\n",
    "    \n",
    "    model = Model([input_lags, input_events], preds)\n",
    "    model.compile(loss=\"mse\", optimizer=\"adam\")\n",
    "    \n",
    "    return model, input_lags, preds\n",
    "\n",
    "# checkpoint best model\n",
    "checkpoint = ModelCheckpoint(\"weights.best.hdf5\", monitor='val_loss', verbose=0, save_best_only=True, mode='min')\n",
    "\n",
    "# fit model to the mean\n",
    "model, input_lags, preds = build_model_events(1, NUM_LAGS+len(sel), 4+len(sel2), 1)\n",
    "model.fit(\n",
    "    [np.concatenate([lags_train, weather_feats_train[:,sel]], axis=1), \n",
    "     np.concatenate([event_feats_train[:,:], lags_event_feats_train[:,sel2]], axis=1)],\n",
    "    y_train,\n",
    "    batch_size=64,\n",
    "    epochs=500,\n",
    "    validation_data=([np.concatenate([lags_val, weather_feats_val[:,sel]], axis=1), \n",
    "                      np.concatenate([event_feats_val[:,:], lags_event_feats_val[:,sel2]], axis=1)], y_val),\n",
    "    callbacks=[checkpoint],\n",
    "    verbose=0)   \n",
    "\n",
    "print \"Total number of iterations:  \", len(model.history.history[\"loss\"])\n",
    "print \"Best loss at iteratation:    \", np.argmin(model.history.history[\"loss\"]), \"   Best:\", np.min(model.history.history[\"loss\"])\n",
    "print \"Best val_loss at iteratation:\", np.argmin(model.history.history[\"val_loss\"]), \"   Best:\", np.min(model.history.history[\"val_loss\"])\n",
    "\n",
    "# load weights\n",
    "model.load_weights(\"weights.best.hdf5\")\n",
    "\n",
    "print model.evaluate([np.concatenate([lags_test[:,:], weather_feats_test[:,sel]], axis=1), \n",
    "                      np.concatenate([event_feats_test[:,:], lags_event_feats_test[:,sel2]], axis=1)], \n",
    "                      y_test, verbose=2)\n",
    "\n",
    "# make predictions\n",
    "preds_lstm = model.predict([np.concatenate([lags_test[:,:], weather_feats_test[:,sel]], axis=1), \n",
    "                            np.concatenate([event_feats_test[:,:], lags_event_feats_test[:,sel2]], axis=1)])\n",
    "preds_lstm = preds_lstm[:,0] * std_test + trend_test\n",
    "corr, mae, rae, rmse, rrse, mape, r2 = compute_error(y_true, preds_lstm)\n",
    "print \"MAE:  %.3f\\tRMSE: %.3f\\tR2:   %.3f\" % (mae, rmse, r2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MLP with weather + events information (no text) + event_lags + TEXT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "preparing word embeddings for NNs with text...\n",
      "Found 965 unique tokens.\n",
      "Shape of train tensor: (730, 350)\n",
      "Shape of val tensor: (365, 350)\n",
      "Shape of test tensor: (170, 350)\n",
      "Preparing embedding matrix.\n",
      "\n",
      "running MLP with lags + weather + events + late + text...\n",
      "text_embedding: Tensor(\"flatten_1/Reshape:0\", shape=(?, ?), dtype=float32)\n",
      "temp1: Tensor(\"permute_1/transpose:0\", shape=(?, 180, 1), dtype=float32)\n",
      "x_lags: Tensor(\"batch_normalization_2/cond/Merge:0\", shape=(?, 100), dtype=float32)\n",
      "temp2: Tensor(\"dropout_5/cond/Merge:0\", shape=(?, 180, 100), dtype=float32)\n",
      "concatenated: Tensor(\"concatenate_2/concat:0\", shape=(?, 180, 101), dtype=float32)\n",
      "after tanh: Tensor(\"dense_2/Tanh:0\", shape=(?, 180, 1), dtype=float32)\n",
      "Tensor(\"batch_normalization_4/cond/Merge:0\", shape=(?, 180), dtype=float32)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:110: UserWarning: The `merge` function is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n",
      "/usr/local/lib/python2.7/dist-packages/keras/legacy/layers.py:465: UserWarning: The `Merge` layer is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n",
      "  name=name)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of iterations:   700\n",
      "Best loss at iteratation:     522    Best: 0.44004683772178543\n",
      "Best val_loss at iteratation: 228    Best: 0.3504277820456518\n",
      "0.3916780401678646\n",
      "MAE:  167.574\tRMSE: 241.148\tR2:   0.485\n",
      "\n",
      "running MLP with lags + weather + events + late + event_lags + text...\n",
      "text_embedding: Tensor(\"flatten_2/Reshape:0\", shape=(?, ?), dtype=float32)\n",
      "temp1: Tensor(\"permute_3/transpose:0\", shape=(?, 180, 1), dtype=float32)\n",
      "x_lags: Tensor(\"batch_normalization_8/cond/Merge:0\", shape=(?, 100), dtype=float32)\n",
      "temp2: Tensor(\"dropout_10/cond/Merge:0\", shape=(?, 180, 100), dtype=float32)\n",
      "concatenated: Tensor(\"concatenate_5/concat:0\", shape=(?, 180, 101), dtype=float32)\n",
      "after tanh: Tensor(\"dense_5/Tanh:0\", shape=(?, 180, 1), dtype=float32)\n",
      "Tensor(\"batch_normalization_10/cond/Merge:0\", shape=(?, 180), dtype=float32)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:244: UserWarning: The `merge` function is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of iterations:   700\n",
      "Best loss at iteratation:     670    Best: 0.4278505651101674\n",
      "Best val_loss at iteratation: 214    Best: 0.340420556313371\n",
      "0.37101990738335777\n",
      "MAE:  152.953\tRMSE: 233.719\tR2:   0.516\n"
     ]
    }
   ],
   "source": [
    "print \"\\npreparing word embeddings for NNs with text...\"\n",
    "\n",
    "# Build index mapping words in the embeddings set to their embedding vector\n",
    "embeddings_index = {}\n",
    "f = open(GLOVE_DIR + 'glove.6B.%dd.txt' % (EMBEDDING_DIM,))\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()\n",
    "\n",
    "# Vectorize the text samples into a 2D integer tensor and pad sequences\n",
    "tokenizer = Tokenizer(num_words=MAX_NB_WORDS)\n",
    "tokenizer.fit_on_texts(event_texts)\n",
    "sequences_train = tokenizer.texts_to_sequences(event_texts_train)\n",
    "sequences_val = tokenizer.texts_to_sequences(event_texts_val)\n",
    "sequences_test = tokenizer.texts_to_sequences(event_texts_test)\n",
    "\n",
    "word_index = tokenizer.word_index\n",
    "print 'Found %s unique tokens.' % len(word_index)\n",
    "\n",
    "data_train = pad_sequences(sequences_train, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "data_val = pad_sequences(sequences_val, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "data_test = pad_sequences(sequences_test, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "print 'Shape of train tensor:', data_train.shape\n",
    "print 'Shape of val tensor:', data_val.shape\n",
    "print 'Shape of test tensor:', data_test.shape\n",
    "\n",
    "# Prepare embedding matrix\n",
    "print('Preparing embedding matrix.')\n",
    "num_words = min(MAX_NB_WORDS, len(word_index)+1)\n",
    "embedding_matrix = np.zeros((num_words, EMBEDDING_DIM))\n",
    "for word, i in word_index.items():\n",
    "    if i >= MAX_NB_WORDS:\n",
    "        continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    #print i\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector\n",
    "\n",
    "\n",
    "def build_model_text(num_inputs, num_lags, num_feat, num_preds):\n",
    "    input_lags = Input(shape=(num_lags,))\n",
    "    input_events = Input(shape=(num_feat,))\n",
    "    \n",
    "    x_lags = Concatenate(axis=1)([input_lags, input_events])\n",
    "    #x_lags = BatchNormalization()(x_lags)\n",
    "    \n",
    "    x = x_lags\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dense(units=100, activation=\"tanh\", kernel_regularizer=keras.regularizers.l2(0.05))(x)\n",
    "    #x = Dense(units=50, activation=\"tanh\", kernel_regularizer=keras.regularizers.l2(0.1))(x)\n",
    "    #x = BatchNormalization()(x)\n",
    "    #x = Dropout(0.5)(x)\n",
    "    #x = BatchNormalization()(x)\n",
    "    #x = Dense(units=50, activation=\"tanh\", kernel_regularizer=keras.regularizers.l2(0.1))(x)\n",
    "    #x = BatchNormalization()(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    x_lags = BatchNormalization()(x)\n",
    "    #x_lags = x\n",
    "    \n",
    "    sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "    embedding_layer = Embedding(num_words,\n",
    "                                EMBEDDING_DIM,\n",
    "                                weights=[embedding_matrix],\n",
    "                                input_length=MAX_SEQUENCE_LENGTH,\n",
    "                                trainable=True)\n",
    "    embedded_sequences = embedding_layer(sequence_input)\n",
    "    x = Conv1D(50, 3, activation='relu')(embedded_sequences)\n",
    "    x = MaxPooling1D(3)(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    x = Conv1D(30, 3, activation='relu')(x)\n",
    "    x = MaxPooling1D(3)(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    x = Conv1D(30, 5, activation='relu')(x)\n",
    "    x = MaxPooling1D(5)(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    #x = Conv1D(50, 5, activation='relu')(x)\n",
    "    #x = MaxPooling1D(5)(x)\n",
    "    text_embedding = Flatten()(x)\n",
    "    #text_embedding = Dropout(0.5)(text_embedding)\n",
    "    #text_embedding = Dense(units=100, activation='relu')(text_embedding)\n",
    "    #text_embedding = Dropout(0.5)(text_embedding)\n",
    "    \n",
    "    print \"text_embedding:\", text_embedding\n",
    "    #temp1 = Permute([1,2])(RepeatVector(180)(BatchNormalization()(text_embedding)))\n",
    "    temp1 = Reshape((1,180))(text_embedding)\n",
    "    temp1 = Permute([2,1])(temp1)\n",
    "    print \"temp1:\", temp1\n",
    "    \n",
    "    print \"x_lags:\", x_lags\n",
    "    temp2 = Permute([1,2])(RepeatVector(180)(BatchNormalization()(x_lags)))\n",
    "    temp2 = Dropout(0.5)(temp2)\n",
    "    print \"temp2:\", temp2\n",
    "    \n",
    "    temp = Concatenate(axis=2)([temp1, temp2])\n",
    "    print \"concatenated:\", temp\n",
    "    temp = Dense(1, activation=\"tanh\")(temp)\n",
    "    #temp = Permute([2,1])(temp)\n",
    "    print \"after tanh:\", temp\n",
    "    temp = Reshape((180,))(temp)\n",
    "    temp = BatchNormalization()(temp)\n",
    "    print temp\n",
    "    attention_probs = Activation(\"softmax\")(temp)\n",
    "    #print fail\n",
    "\n",
    "    #attention_probs = Dense(180, activation='softmax', name='attention_vec')(text_embedding)\n",
    "    attention_mul = merge([text_embedding, attention_probs], output_shape=180, name='attention_mul', mode='mul')\n",
    "    attention_mul = BatchNormalization()(attention_mul)\n",
    "    #attention_mul = Dropout(0.5)(attention_mul)\n",
    "    \n",
    "    feat = Concatenate(axis=1)([x_lags, attention_mul])\n",
    "    \n",
    "    feat = BatchNormalization()(feat)\n",
    "    #feat = Dense(units=50, activation='relu')(feat)\n",
    "    #feat = Dropout(0.5)(feat)\n",
    "    \n",
    "    preds = Dense(units=num_preds)(feat)\n",
    "    #preds = Dense(units=num_preds, kernel_regularizer=keras.regularizers.l2(0.2))(feat)\n",
    "    preds = Activation(\"linear\")(preds)\n",
    "    \n",
    "    model = Model([input_lags, input_events, sequence_input], preds)\n",
    "    \n",
    "    rmsp = keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\n",
    "    #model.compile(loss=\"mse\", optimizer=rmsp)\n",
    "    model.compile(loss=\"mse\", optimizer=\"adam\")\n",
    "    \n",
    "    return model, input_lags, preds\n",
    "\n",
    "\n",
    "print \"\\nrunning MLP with lags + weather + events + late + text...\"\n",
    "\n",
    "# checkpoint best model\n",
    "checkpoint = ModelCheckpoint(\"weights.best.hdf5\", monitor='val_loss', verbose=0, save_best_only=True, mode='min')\n",
    "\n",
    "# fit model to the mean\n",
    "model, input_lags, preds = build_model_text(1, NUM_LAGS+len(sel), 4, 1)\n",
    "model.fit(\n",
    "    [np.concatenate([lags_train, weather_feats_train[:,sel]], axis=1), \n",
    "     np.concatenate([event_feats_train[:,:]], axis=1),\n",
    "     data_train],\n",
    "    y_train,\n",
    "    batch_size=64,\n",
    "    epochs=700,\n",
    "    #validation_split=0.2,\n",
    "    validation_data=([np.concatenate([lags_val, weather_feats_val[:,sel]], axis=1), \n",
    "                      np.concatenate([event_feats_val[:,:]], axis=1),\n",
    "                      data_val], y_val),\n",
    "    callbacks=[checkpoint],\n",
    "    verbose=0)   \n",
    "\n",
    "print \"Total number of iterations:  \", len(model.history.history[\"loss\"])\n",
    "print \"Best loss at iteratation:    \", np.argmin(model.history.history[\"loss\"]), \"   Best:\", np.min(model.history.history[\"loss\"])\n",
    "print \"Best val_loss at iteratation:\", np.argmin(model.history.history[\"val_loss\"]), \"   Best:\", np.min(model.history.history[\"val_loss\"])\n",
    "\n",
    "# load weights\n",
    "model.load_weights(\"weights.best.hdf5\")\n",
    "\n",
    "print model.evaluate([np.concatenate([lags_test[:,:], weather_feats_test[:,sel]], axis=1), \n",
    "                      np.concatenate([event_feats_test[:,:]], axis=1),\n",
    "                      data_test],\n",
    "                      y_test, verbose=2)\n",
    "\n",
    "# make predictions\n",
    "preds_lstm = model.predict([np.concatenate([lags_test[:,:], weather_feats_test[:,sel]], axis=1), \n",
    "                            np.concatenate([event_feats_test[:,:]], axis=1),\n",
    "                            data_test])\n",
    "preds_lstm = preds_lstm[:,0] * std_test + trend_test\n",
    "y_true = y_test * std_test + trend_test\n",
    "corr, mae, rae, rmse, rrse, mape, r2 = compute_error(y_true, preds_lstm)\n",
    "print \"MAE:  %.3f\\tRMSE: %.3f\\tR2:   %.3f\" % (mae, rmse, r2)\n",
    "\n",
    "\n",
    "# ---------------------------------------- MLP with weather + events information (no text) + event_lags + TEXT\n",
    "\n",
    "def build_model_text_v2(num_inputs, num_lags, num_feat, num_preds):\n",
    "    input_lags = Input(shape=(num_lags,))\n",
    "    input_events = Input(shape=(num_feat,))\n",
    "    \n",
    "    x_lags = Concatenate(axis=1)([input_lags, input_events])\n",
    "    #x_lags = BatchNormalization()(x_lags)\n",
    "    \n",
    "    x = x_lags\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dense(units=100, activation=\"tanh\", kernel_regularizer=keras.regularizers.l2(0.05))(x)\n",
    "    #x = Dense(units=50, activation=\"tanh\", kernel_regularizer=keras.regularizers.l2(0.1))(x)\n",
    "    #x = BatchNormalization()(x)\n",
    "    #x = Dropout(0.5)(x)\n",
    "    #x = BatchNormalization()(x)\n",
    "    #x = Dense(units=50, activation=\"tanh\", kernel_regularizer=keras.regularizers.l2(0.1))(x)\n",
    "    #x = BatchNormalization()(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    x_lags = BatchNormalization()(x)\n",
    "    #x_lags = x\n",
    "    \n",
    "    sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "    embedding_layer = Embedding(num_words,\n",
    "                                EMBEDDING_DIM,\n",
    "                                weights=[embedding_matrix],\n",
    "                                input_length=MAX_SEQUENCE_LENGTH,\n",
    "                                trainable=True)\n",
    "    embedded_sequences = embedding_layer(sequence_input)\n",
    "    x = Conv1D(50, 3, activation='relu')(embedded_sequences)\n",
    "    x = MaxPooling1D(3)(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    x = Conv1D(30, 3, activation='relu')(x)\n",
    "    x = MaxPooling1D(3)(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    x = Conv1D(30, 5, activation='relu')(x)\n",
    "    x = MaxPooling1D(5)(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    #x = Conv1D(50, 5, activation='relu')(x)\n",
    "    #x = MaxPooling1D(5)(x)\n",
    "    text_embedding = Flatten()(x)\n",
    "    #text_embedding = Dropout(0.5)(text_embedding)\n",
    "    #text_embedding = Dense(units=100, activation='relu')(text_embedding)\n",
    "    #text_embedding = Dropout(0.5)(text_embedding)\n",
    "    \n",
    "    print \"text_embedding:\", text_embedding\n",
    "    #temp1 = Permute([1,2])(RepeatVector(180)(BatchNormalization()(text_embedding)))\n",
    "    temp1 = Reshape((1,180))(text_embedding)\n",
    "    temp1 = Permute([2,1])(temp1)\n",
    "    print \"temp1:\", temp1\n",
    "    \n",
    "    print \"x_lags:\", x_lags\n",
    "    temp2 = Permute([1,2])(RepeatVector(180)(BatchNormalization()(x_lags)))\n",
    "    temp2 = Dropout(0.5)(temp2)\n",
    "    print \"temp2:\", temp2\n",
    "    \n",
    "    temp = Concatenate(axis=2)([temp1, temp2])\n",
    "    print \"concatenated:\", temp\n",
    "    temp = Dense(1, activation=\"tanh\")(temp)\n",
    "    #temp = Permute([2,1])(temp)\n",
    "    print \"after tanh:\", temp\n",
    "    temp = Reshape((180,))(temp)\n",
    "    temp = BatchNormalization()(temp)\n",
    "    print temp\n",
    "    attention_probs = Activation(\"softmax\")(temp)\n",
    "    #print fail\n",
    "\n",
    "    #attention_probs = Dense(180, activation='softmax', name='attention_vec')(text_embedding)\n",
    "    attention_mul = merge([text_embedding, attention_probs], output_shape=180, name='attention_mul', mode='mul')\n",
    "    attention_mul = BatchNormalization()(attention_mul)\n",
    "    #attention_mul = Dropout(0.5)(attention_mul)\n",
    "    \n",
    "    feat = Concatenate(axis=1)([x_lags, attention_mul])\n",
    "    \n",
    "    feat = BatchNormalization()(feat)\n",
    "    #feat = Dense(units=50, activation='relu')(feat)\n",
    "    #feat = Dropout(0.5)(feat)\n",
    "    \n",
    "    preds = Dense(units=num_preds)(feat)\n",
    "    #preds = Dense(units=num_preds, kernel_regularizer=keras.regularizers.l2(0.2))(feat)\n",
    "    preds = Activation(\"linear\")(preds)\n",
    "    \n",
    "    model = Model([input_lags, input_events, sequence_input], preds)\n",
    "    \n",
    "    rmsp = keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\n",
    "    #model.compile(loss=\"mse\", optimizer=rmsp)\n",
    "    model.compile(loss=\"mse\", optimizer=\"adam\")\n",
    "    \n",
    "    return model, input_lags, preds\n",
    "\n",
    "\n",
    "print \"\\nrunning MLP with lags + weather + events + late + event_lags + text...\"\n",
    "\n",
    "# checkpoint best model\n",
    "checkpoint = ModelCheckpoint(\"weights.best.hdf5\", monitor='val_loss', verbose=0, save_best_only=True, mode='min')\n",
    "\n",
    "# fit model to the mean\n",
    "model, input_lags, preds = build_model_text_v2(1, NUM_LAGS+len(sel), 4+len(sel2), 1)\n",
    "model.fit(\n",
    "    [np.concatenate([lags_train, weather_feats_train[:,sel]], axis=1), \n",
    "     np.concatenate([event_feats_train[:,:], lags_event_feats_train[:,sel2]], axis=1),\n",
    "     data_train],\n",
    "    y_train,\n",
    "    batch_size=64,\n",
    "    epochs=700,\n",
    "    #validation_split=0.2,\n",
    "    validation_data=([np.concatenate([lags_val, weather_feats_val[:,sel]], axis=1), \n",
    "                      np.concatenate([event_feats_val[:,:], lags_event_feats_val[:,sel2]], axis=1),\n",
    "                      data_val], y_val),\n",
    "    callbacks=[checkpoint],\n",
    "    verbose=0)   \n",
    "\n",
    "print \"Total number of iterations:  \", len(model.history.history[\"loss\"])\n",
    "print \"Best loss at iteratation:    \", np.argmin(model.history.history[\"loss\"]), \"   Best:\", np.min(model.history.history[\"loss\"])\n",
    "print \"Best val_loss at iteratation:\", np.argmin(model.history.history[\"val_loss\"]), \"   Best:\", np.min(model.history.history[\"val_loss\"])\n",
    "\n",
    "# load weights\n",
    "model.load_weights(\"weights.best.hdf5\")\n",
    "\n",
    "print model.evaluate([np.concatenate([lags_test[:,:], weather_feats_test[:,sel]], axis=1), \n",
    "                      np.concatenate([event_feats_test[:,:], lags_event_feats_test[:,sel2]], axis=1),\n",
    "                      data_test],\n",
    "                      y_test, verbose=2)\n",
    "\n",
    "# make predictions\n",
    "preds_lstm = model.predict([np.concatenate([lags_test[:,:], weather_feats_test[:,sel]], axis=1), \n",
    "                            np.concatenate([event_feats_test[:,:], lags_event_feats_test[:,sel2]], axis=1),\n",
    "                            data_test])\n",
    "preds_lstm = preds_lstm[:,0] * std_test + trend_test\n",
    "corr, mae, rae, rmse, rrse, mape, r2 = compute_error(y_true, preds_lstm)\n",
    "print \"MAE:  %.3f\\tRMSE: %.3f\\tR2:   %.3f\" % (mae, rmse, r2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}