bi_lstm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using gpu device 0: GRID K520\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import pickle\n",
    "import itertools\n",
    "import time\n",
    "import random\n",
    "\n",
    "import theano\n",
    "from theano import tensor as T\n",
    "import lasagne\n",
    "#from recurrent import LSTMLayer, RecurrentSoftmaxLayer\n",
    "from lasagne.layers import DenseLayer, LSTMLayer, SliceLayer\n",
    "from lasagne.regularization import l2, regularize_layer_params, regularize_network_params, regularize_layer_params_weighted\n",
    "from lasagne.objectives import aggregate\n",
    "\n",
    "# have to change these hyper parameters, currently performing worse.\n",
    "NUM_EPOCHS = 200\n",
    "BATCH_SIZE = 256\n",
    "NUM_HIDDEN_UNITS = 512\n",
    "LEARNING_RATE = 0.00005\n",
    "MOMENTUM = 0.9\n",
    "REG_STRENGTH = 0.00005\n",
    "DROPOUT = 0.5\n",
    "\n",
    "# NUM_EPOCHS = 600\n",
    "# BATCH_SIZE = 256\n",
    "# NUM_HIDDEN_UNITS = 512\n",
    "# LEARNING_RATE = 0.00005\n",
    "# MOMENTUM = 0.9\n",
    "# REG_STRENGTH = 0.0001\n",
    "# DROPOUT = 0.6\n",
    "# SEED = .5\n",
    "\n",
    "DATA_PATH = '/home/ubuntu/VideoNet/cnn_out_data_fc6/'\n",
    "\n",
    "\n",
    "def get_data():\n",
    "    #get data \n",
    "    class_names = sorted(os.listdir(DATA_PATH)) \n",
    "\n",
    "    print '\\tgetting data'\n",
    "    inputs = []\n",
    "    labels = []\n",
    "    for class_index in range(len(class_names)):\n",
    "        class_name = class_names[class_index]\n",
    "        print '\\tclass_Index %s' % class_index\n",
    "        print '\\tgetting data for class %s' % class_name\n",
    "        class_dir = DATA_PATH + class_name + '/'\n",
    "        for pfile in os.listdir(class_dir):\n",
    "            f = open(class_dir + pfile, 'r')\n",
    "            features = pickle.load(f)\n",
    "            f.close()\n",
    "            inputs.append(features)\n",
    "            labels.append(class_index)\n",
    "    \n",
    "    c = list(zip(inputs, labels))\n",
    "    random.shuffle(c)\n",
    "    inputs, labels = zip(*c)\n",
    "\n",
    "#     random.shuffle(inputs, lambda: SEED)\n",
    "#     random.shuffle(labels, lambda: SEED)\n",
    "    \n",
    "    \n",
    "    assert len(inputs) == len(labels)\n",
    "    \n",
    "#     split_size = len(inputs)/10\n",
    "#     train_size = len(inputs) - 4*split_size\n",
    "#     val_ind = train_size - 1 + split_size\n",
    "#     val_ind2= val_ind + split_size\n",
    "\n",
    "#     X_train = inputs[:train_size]\n",
    "#     y_train = labels[:train_size]\n",
    "#     X_valid = inputs[train_size-1:val_ind]\n",
    "#     y_valid = labels[train_size-1:val_ind]\n",
    "#     X_test = inputs[val_ind-1:val_ind2]\n",
    "#     y_test = labels[val_ind-1:val_ind2]\n",
    "    \n",
    "    split_size = len(inputs)/10\n",
    "    train_size = len(inputs) - 2*split_size\n",
    "    val_ind = train_size - 1 + split_size\n",
    "\n",
    "    X_train = inputs[:train_size]\n",
    "    y_train = labels[:train_size]\n",
    "    X_valid = inputs[train_size-1:val_ind]\n",
    "    y_valid = labels[train_size-1:val_ind]\n",
    "    X_test = inputs[val_ind-1:]\n",
    "    y_test = labels[val_ind-1:]\n",
    "\n",
    "    print '\\tnumpying' \n",
    "    X_train = np.array(X_train)\n",
    "    y_train = np.array(y_train)\n",
    "    X_valid = np.array(X_valid)\n",
    "    y_valid = np.array(y_valid)\n",
    "    X_test = np.array(X_test)\n",
    "    y_test = np.array(y_test)\n",
    "\n",
    "    print '\\tcreating theano shared vars'\n",
    "    return dict(\n",
    "        X_train=theano.shared(lasagne.utils.floatX(X_train)),\n",
    "        y_train=T.cast(theano.shared(y_train), 'int32'),\n",
    "        X_valid=theano.shared(lasagne.utils.floatX(X_valid)),\n",
    "        y_valid=T.cast(theano.shared(y_valid), 'int32'),\n",
    "        X_test=theano.shared(lasagne.utils.floatX(X_test)),\n",
    "        y_test=T.cast(theano.shared(y_test), 'int32'),\n",
    "        num_examples_train=X_train.shape[0],\n",
    "        num_examples_valid=X_valid.shape[0],\n",
    "        num_examples_test=X_test.shape[0],\n",
    "        input_dim=X_train.shape,\n",
    "        output_dim=len(class_names),\n",
    "        )\n",
    "\n",
    "\n",
    "def build_model(input_dim, output_dim, \n",
    "                batch_size=BATCH_SIZE, num_hidden_units=NUM_HIDDEN_UNITS):\n",
    "    l_in = lasagne.layers.InputLayer(\n",
    "          shape=(batch_size, input_dim[1], input_dim[2]),\n",
    "          )\n",
    "    l_lstm_forward = LSTMLayer(\n",
    "            l_in,\n",
    "            num_units=num_hidden_units, nonlinearity=lasagne.nonlinearities.tanh\n",
    "            ) \n",
    "    l_lstm_backward = LSTMLayer(\n",
    "            l_in,\n",
    "            num_units=num_hidden_units, \n",
    "        backwards=True, nonlinearity=lasagne.nonlinearities.tanh\n",
    "            ) \n",
    "    l_lstm_combined = lasagne.layers.ElemwiseSumLayer(\n",
    "        incomings = (l_lstm_forward, l_lstm_backward)\n",
    "        )\n",
    "    l_out = DenseLayer(\n",
    "        lasagne.layers.dropout(l_lstm_combined, p=.5),\n",
    "        num_units=output_dim, nonlinearity=lasagne.nonlinearities.softmax\n",
    "        )\n",
    "        #lasagne.layers.dropout(l_rec2, p=.5)\n",
    "#     l_out = RecurrentSoftmaxLayer(\n",
    "#         l_rec1,\n",
    "#         num_units=output_dim\n",
    "#         )\n",
    "    \n",
    "    return l_out\n",
    "\n",
    "def create_iter_functions(dataset, output_layer,\n",
    "                          X_tensor_type=T.tensor3,\n",
    "                          batch_size=BATCH_SIZE,\n",
    "                          learning_rate=LEARNING_RATE,\n",
    "                          momentum=MOMENTUM,\n",
    "                          reg_strength=REG_STRENGTH):\n",
    "    batch_index = T.iscalar('batch_index')\n",
    "    X_batch = X_tensor_type('x')\n",
    "    y_batch = T.ivector('y')\n",
    "    batch_slice = slice(\n",
    "      batch_index * batch_size, (batch_index + 1) * batch_size)\n",
    "\n",
    "    output_train=lasagne.layers.get_output(output_layer,X_batch)\n",
    "\n",
    "    l2_penalty = regularize_layer_params(output_layer, l2)* REG_STRENGTH\n",
    "    \n",
    "    loss_train= aggregate(lasagne.objectives.categorical_crossentropy(output_train, y_batch)) + l2_penalty\n",
    "    \n",
    "    #loss_train = loss_train.mean() + l2_penalty#+ REG_STRENGTH*reg\n",
    "    \n",
    "    output_eval = lasagne.layers.get_output(output_layer, X_batch, deterministic=True)\n",
    "\n",
    "    loss_eval= aggregate(lasagne.objectives.categorical_crossentropy(output_eval, y_batch))\n",
    "    \n",
    "#     loss_eval = loss_eval.mean()\n",
    "    \n",
    "    pred = T.argmax(lasagne.layers.get_output(output_layer,X_batch, deterministic=True), axis=1)\n",
    "    \n",
    "    accuracy = T.mean(T.eq(pred, y_batch), dtype=theano.config.floatX)\n",
    "\n",
    "    all_params = lasagne.layers.get_all_params(output_layer, trainable=True)\n",
    "    \n",
    "    updates = lasagne.updates.rmsprop(\n",
    "      loss_train, all_params, learning_rate, momentum)\n",
    "    \n",
    "    iter_train = theano.function(\n",
    "      [batch_index], [loss_train, accuracy],\n",
    "      updates=updates,\n",
    "      givens={\n",
    "        X_batch: dataset['X_train'][batch_slice],\n",
    "        y_batch: dataset['y_train'][batch_slice],\n",
    "        },\n",
    "      )\n",
    "\n",
    "    iter_valid = theano.function(\n",
    "      [batch_index], [loss_eval, accuracy],\n",
    "      givens={\n",
    "        X_batch: dataset['X_valid'][batch_slice],\n",
    "        y_batch: dataset['y_valid'][batch_slice],\n",
    "        },\n",
    "      )\n",
    "\n",
    "    iter_test = theano.function(\n",
    "      [batch_index], [loss_eval, accuracy],\n",
    "      givens={\n",
    "        X_batch: dataset['X_test'][batch_slice],\n",
    "        y_batch: dataset['y_test'][batch_slice],\n",
    "        },\n",
    "      )\n",
    "\n",
    "    return dict(\n",
    "      train=iter_train,\n",
    "      valid=iter_valid,\n",
    "      test=iter_test,\n",
    "      )\n",
    "\n",
    "def train(iter_funcs, dataset, batch_size=BATCH_SIZE):\n",
    "    num_batches_train = dataset['num_examples_train'] // batch_size\n",
    "    num_batches_valid = dataset['num_examples_valid'] // batch_size\n",
    "\n",
    "    for epoch in itertools.count(1):\n",
    "        batch_train_losses = []\n",
    "        batch_train_accuracies = []\n",
    "        for b in range(num_batches_train):\n",
    "            #print '\\tbatch %d of %d' % (b, num_batches_train)\n",
    "            #tick = time.time()\n",
    "            batch_train_loss, batch_train_accuracy = iter_funcs['train'](b)\n",
    "            batch_train_losses.append(batch_train_loss)\n",
    "            batch_train_accuracies.append(batch_train_accuracy)\n",
    "            #toc = time.time()\n",
    "            #print '\\t\\t loss: %f' % (batch_train_loss)\n",
    "            #print '\\t\\t took %f' % (toc - tick)\n",
    "\n",
    "        avg_train_loss = np.mean(batch_train_losses)\n",
    "        avg_train_accuracy = np.mean(batch_train_accuracies)\n",
    "\n",
    "        batch_valid_losses = []\n",
    "        batch_valid_accuracies = []\n",
    "        for b in range(num_batches_valid):\n",
    "            batch_valid_loss, batch_valid_accuracy = iter_funcs['valid'](b)\n",
    "            batch_valid_losses.append(batch_valid_loss)\n",
    "            batch_valid_accuracies.append(batch_valid_accuracy)\n",
    "\n",
    "        avg_valid_loss = np.mean(batch_valid_losses)\n",
    "        avg_valid_accuracy = np.mean(batch_valid_accuracies)\n",
    "\n",
    "        yield {\n",
    "            'number': epoch,\n",
    "            'train_loss': avg_train_loss,\n",
    "            'train_accuracy': avg_train_accuracy,\n",
    "            'valid_loss': avg_valid_loss,\n",
    "            'valid_accuracy': avg_valid_accuracy\n",
    "            }\n",
    "\n",
    "def test(iter_funcs, dataset, batch_size=BATCH_SIZE):\n",
    "    num_batches_test = dataset['num_examples_test'] // batch_size\n",
    "    batch_accuracies = []\n",
    "    for b in range(num_batches_test):\n",
    "        batch_loss, batch_accuracy = iter_funcs['test'](b)\n",
    "        batch_accuracies.append(batch_accuracy)\n",
    "    avg_test_accuracy = np.mean(batch_accuracies)\n",
    "    return avg_test_accuracy\n",
    "\n",
    "def run(dataset,\n",
    "    num_epochs=NUM_EPOCHS,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    num_hidden_units=NUM_HIDDEN_UNITS,\n",
    "    learning_rate=LEARNING_RATE,\n",
    "    momentum=MOMENTUM,\n",
    "    reg_strength=REG_STRENGTH,\n",
    "    dropout=DROPOUT,\n",
    "    TESTING = False\n",
    "    ):\n",
    "    # assign global vars\n",
    "    global NUM_EPOCHS\n",
    "    global BATCH_SIZE\n",
    "    global NUM_HIDDEN_UNITS \n",
    "    global LEARNING_RATE \n",
    "    global MOMENTUM\n",
    "    global REG_STRENGTH\n",
    "    global DROPOUT\n",
    "    NUM_EPOCHS = num_epochs\n",
    "    BATCH_SIZE = batch_size\n",
    "    NUM_HIDDEN_UNITS = num_hidden_units \n",
    "    LEARNING_RATE = learning_rate \n",
    "    MOMENTUM = momentum\n",
    "    REG_STRENGTH = reg_strength\n",
    "    DROPOUT = dropout\n",
    "\n",
    "    to_return = None\n",
    "    print 'BUILDING MODEL'\n",
    "    output_layer = build_model(\n",
    "        input_dim = dataset['input_dim'],\n",
    "        output_dim = dataset['output_dim'],\n",
    "        )\n",
    "    print 'CREATING ITER FUNCS'\n",
    "    iter_funcs = create_iter_functions(dataset, output_layer)\n",
    "    results= np.zeros(shape=(num_epochs,5))\n",
    "    print 'TRAINING'\n",
    "    for epoch in train(iter_funcs, dataset):\n",
    "        print(\"Epoch %d of %d\" % (epoch['number'], num_epochs))\n",
    "        print(\"\\ttraining loss:\\t\\t%.6f\" % epoch['train_loss'])\n",
    "        print(\"\\ttraining accuracy:\\t\\t%.2f %%\" % (epoch['train_accuracy'] * 100))\n",
    "        print(\"\\tvalidation loss:\\t\\t%.6f\" % epoch['valid_loss'])\n",
    "        validation_acc = (epoch['valid_accuracy'] * 100)\n",
    "        print(\"\\tvalidation accuracy:\\t\\t%.2f %%\" % (validation_acc))\n",
    "        \n",
    "        to_return = validation_acc\n",
    "        \n",
    "        results[epoch['number']-1][0]= epoch['number']\n",
    "        results[epoch['number']-1][1]= epoch['train_loss']\n",
    "        results[epoch['number']-1][2] = epoch['valid_loss']\n",
    "        results[epoch['number']-1][3] = epoch['train_accuracy'] * 100\n",
    "        results[epoch['number']-1][4]= validation_acc\n",
    "\n",
    "        \n",
    "        \n",
    "        if epoch['number'] >= num_epochs:\n",
    "            break\n",
    "            \n",
    "    if TESTING:\n",
    "        print 'TESTING'\n",
    "        test_acc = test(iter_funcs, dataset)\n",
    "        print 'test accuracy: \\t\\t%.2f' % (test_acc*100)\n",
    "        to_return = test_acc*100\n",
    "\n",
    "    return results, to_return\n",
    "\n",
    "\n",
    "def mymain(dataset,num_epochs=NUM_EPOCHS,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    num_hidden_units=NUM_HIDDEN_UNITS,\n",
    "    learning_rate=LEARNING_RATE,\n",
    "    momentum=MOMENTUM,\n",
    "    reg_strength=REG_STRENGTH,\n",
    "    dropout=DROPOUT,\n",
    "    TESTING = True\n",
    "    ):\n",
    " \n",
    "    \n",
    "    results,test123=run(dataset,\n",
    "        num_epochs,\n",
    "        batch_size,\n",
    "        num_hidden_units,\n",
    "        learning_rate,\n",
    "        momentum,\n",
    "        reg_strength,\n",
    "        dropout,\n",
    "        TESTING)\n",
    "\n",
    "    with open ('bi_lstm_gbl.txt','a') as f:\n",
    "        f.write(str(batch_size)+'\\t'+str(num_hidden_units)+'\\t'+str(learning_rate)+'\\t'+str(reg_strength)+\n",
    "                '\\t'+str(dropout)+'\\n')\n",
    "        for i in range(len(results)):\n",
    "            f.write(str(results[i][0])+'\\t'+str(results[i][1])+'\\t'+str(results[i][2])+'\\t'+str(results[i][3])+'\\t'+str(results[i][4])+'\\n')\n",
    "        f.write(str(test123)+'\\n')    \n",
    "        f.close()\n",
    "# if __name__ == '__main__':\n",
    "#     mymain()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print 'LOADING DATA'\n",
    "dataset = get_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mymain(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}