emotion_detection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Introduction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this tutorial, we will train logistic/softmax regression model using TensorFlow, to predict if the input is neutral face or smiling face.\n",
    "About the dataset, I collected images from Google Images and cropped the mouth part is manually put them into two separate folders. \n",
    "This is developed in Python 2.7.10 and Tensorflow 1.2.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensorflow version 1.13.1\n"
     ]
    }
   ],
   "source": [
    "import cv2\n",
    "import glob,os\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "print (\"tensorflow version\",tf.__version__)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading data and resize "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reading images from the data folder\n",
    "# label, smile = 1, neutral = 0\n",
    "(img_width, img_height) = (45, 25)\n",
    "def load_images_from_folder(folder):\n",
    "    (images, lables, names, id) = ([], [], {}, 0)\n",
    "    for (subdirs, dirs, files) in os.walk(folder):\n",
    "        print (subdirs, dirs)\n",
    "        for subdir in dirs:\n",
    "            names[id] = subdir\n",
    "            subjectpath = os.path.join(folder, subdir)\n",
    "            for filename in os.listdir(subjectpath):\n",
    "                path = subjectpath + '/' + filename\n",
    "                lable = id\n",
    "                img = cv2.imread(path, 0) \n",
    "                img = cv2.resize(img,(img_width, img_height)) \n",
    "                images.append(img)\n",
    "                lables.append(int(lable))\n",
    "            id += 1\n",
    "        print(names)\n",
    "        print(lables)\n",
    "        return images, lables, names\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/ayushi/Real-time-Smile-Recognition/data ['smile', 'neutral']\n",
      "{0: 'smile', 1: 'neutral'}\n",
      "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n"
     ]
    }
   ],
   "source": [
    "X,Y,classes = load_images_from_folder(\"/home/ayushi/Real-time-Smile-Recognition/data\")\n",
    "\n",
    "(X,Y) = [np.array(lis) for lis in [X, Y]]\n",
    "Y = pd.get_dummies(Y) #converting labels to one-hot, Used Pandas for it. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dimension of our data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Height of the image:  25\n",
      "Width of the image:   45\n",
      "Total number of data: 107\n"
     ]
    }
   ],
   "source": [
    "print (\"Height of the image:  \" + str(X.shape[1]))\n",
    "print (\"Width of the image:   \"  + str(X.shape[2]))\n",
    "print (\"Total number of data: \"+ str(len(X)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Splitting the data using sklearn "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, Y, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Size of data, Train set and Test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lenght of train data: 71\n",
      "lenght of test data:  36\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, Y, test_size=0.33, random_state=42)\n",
    "print (\"lenght of train data: \"+str(len(X_train))) \n",
    "print (\"lenght of test data:  \"+str(len(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_set_x_flatten: (71, 1125)\n",
      "test_set_x_flatten : (36, 1125)\n",
      "[[159 157 166 ... 145 143 142]\n",
      " [146 149 140 ... 149 145 146]\n",
      " [204 208 210 ... 206 207 194]\n",
      " ...\n",
      " [162 164 174 ... 162 161 158]\n",
      " [170 181 179 ... 154 159 155]\n",
      " [112 116 113 ... 136 132 132]]\n"
     ]
    }
   ],
   "source": [
    "train_set_x_flatten = X_train.reshape(X_train.shape[0],-1)\n",
    "test_set_x_flatten = X_test.reshape(X_test.shape[0],-1)\n",
    "print (\"train_set_x_flatten: \"+str(train_set_x_flatten.shape))\n",
    "print (\"test_set_x_flatten : \"+str(test_set_x_flatten.shape))\n",
    "\n",
    "print (train_set_x_flatten)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of train set: 71\n",
      "length of test set: 36\n",
      "shape of train set: (71, 1125)\n",
      "shape of test set: (36, 1125)\n"
     ]
    }
   ],
   "source": [
    "#Normalizing data, conveting all pixel value in range between 0-1\n",
    "train_set_x = train_set_x_flatten/255.\n",
    "test_set_x = test_set_x_flatten/255.\n",
    "print (\"length of train set: \" + str(len(train_set_x)))\n",
    "print (\"length of test set: \" + str(len(test_set_x)))\n",
    "print (\"shape of train set: \"+ str(train_set_x.shape))\n",
    "print (\"shape of test set: \"+ str(test_set_x.shape))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Building the softmax regression using tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 1125])\n",
    "W = tf.Variable(tf.zeros([1125, 2]), dtype=tf.float32, name=\"w1\")\n",
    "b = tf.Variable(tf.zeros([2]),  dtype=tf.float32, name=\"bias\")\n",
    "y = tf.nn.softmax(tf.matmul(x, W) + b, name=\"first_operation\")\n",
    "y_ = tf.placeholder(tf.float32, [None, 2], name = \"final_output\")\n",
    "cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training with learning rate: 0.1\n",
      "losses after per 1000 iteration:  0.6931472\n",
      "losses after per 1000 iteration:  0.016667362\n",
      "losses after per 1000 iteration:  0.009934486\n",
      "losses after per 1000 iteration:  0.007109392\n",
      "losses after per 1000 iteration:  0.0055379937\n",
      "losses after per 1000 iteration:  0.0045358283\n",
      "losses after per 1000 iteration:  0.0038408486\n",
      "losses after per 1000 iteration:  0.003330561\n",
      "losses after per 1000 iteration:  0.0029399844\n",
      "losses after per 1000 iteration:  0.0026314124\n",
      "losses after per 1000 iteration:  0.0023814947\n",
      "losses after per 1000 iteration:  0.0021749528\n",
      "losses after per 1000 iteration:  0.002001402\n",
      "losses after per 1000 iteration:  0.0018535227\n",
      "losses after per 1000 iteration:  0.0017260044\n",
      "Model saved in file: ./model/emotion_model\n",
      "accuracy with learning rate:  0.1 0.5833333\n"
     ]
    }
   ],
   "source": [
    "learning_rate = 0.1\n",
    "print (\"training with learning rate: \" + str(learning_rate))\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entropy)\n",
    "\n",
    "saver = tf.train.Saver()\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    for step in range(15000):\n",
    "        trainStep,loss = sess.run([train_step, cross_entropy], feed_dict={x: train_set_x, y_: y_train})\n",
    "        if step%1000==0:\n",
    "            print (\"losses after per 1000 iteration: \",loss)\n",
    "\n",
    "    save_path = saver.save(sess, \"./model/emotion_model\")\n",
    "\n",
    "    print(\"Model saved in file: %s\" % save_path)\n",
    "\n",
    "    correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(\"accuracy with learning rate: \" ,str(learning_rate), sess.run(accuracy, feed_dict={x:test_set_x , y_: y_test}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# prediction "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./model/emotion_model\n",
      "neutral\n"
     ]
    }
   ],
   "source": [
    "saver = tf.train.Saver()\n",
    "with tf.Session() as sess:\n",
    "    #saver.restore(sess, \"./model/emotion_model\")\n",
    "    print(\"w1:\", sess.run(W))\n",
    "    #print(\"bias:\", sess.run(b))\n",
    "    img = cv2.imread(\"/home/ayushi/Real-time-Smile-Recognition/test/test1.jpg\", 0) \n",
    "    img = cv2.resize(img,(img_width, img_height)) # Resizing all the images\n",
    "    image=[]\n",
    "    image.append(img)\n",
    "    image_test = np.array(image)\n",
    "    test_image_flatten = image_test.reshape(image_test.shape[0],-1)\n",
    "    test_image_normalized = test_image_flatten/255.\n",
    "    #print (test_image_normalized.shape)\n",
    "    x_ = tf.cast(test_image_normalized, tf.float32)\n",
    "    y = tf.nn.softmax(tf.matmul(x_, W) + b)\n",
    "    #print (sess.run(y))\n",
    "    indx = sess.run(tf.argmax(y,1))\n",
    "    if indx==0:\n",
    "        print (\"smile\")\n",
    "    elif indx==1:\n",
    "        print (\"neutral\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}