Traffic_Sign_Classifier.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Self-Driving Car Engineer Nanodegree\n",
    "\n",
    "\n",
    "## Project: Build a Traffic Sign Recognition Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Load The Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load pickled data\n",
    "import pickle\n",
    "\n",
    "# TODO: Fill this in based on where you saved the training and testing data\n",
    "DATA_FOLDER = 'data/traffic-signs-data'\n",
    "\n",
    "training_file = DATA_FOLDER + '/train.p'\n",
    "validation_file= DATA_FOLDER + '/valid.p'\n",
    "testing_file = DATA_FOLDER + '/test.p'\n",
    "\n",
    "with open(training_file, mode='rb') as f:\n",
    "    train = pickle.load(f)\n",
    "with open(validation_file, mode='rb') as f:\n",
    "    valid = pickle.load(f)\n",
    "with open(testing_file, mode='rb') as f:\n",
    "    test = pickle.load(f)\n",
    "    \n",
    "X_train, y_train = train['features'], train['labels']\n",
    "X_valid, y_valid = valid['features'], valid['labels']\n",
    "X_test, y_test = test['features'], test['labels']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## Dataset Summary & Exploration "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Replace each question mark with the appropriate value. \n",
    "### Use python, pandas or numpy methods rather than hard coding the results\n",
    "import numpy as np\n",
    "\n",
    "# TODO: Number of training examples\n",
    "n_train = len(X_train)\n",
    "\n",
    "# TODO: Number of validation examples\n",
    "n_validation = len(X_valid)\n",
    "\n",
    "# TODO: Number of testing examples.\n",
    "n_test = len(X_test)\n",
    "\n",
    "# TODO: What's the shape of an traffic sign image?\n",
    "image_shape = X_train[0].shape\n",
    "\n",
    "# TODO: How many unique classes/labels there are in the dataset.\n",
    "n_classes = len(np.unique(y_train))\n",
    "\n",
    "print(\"Number of training examples =\", n_train)\n",
    "print(\"Number of testing examples =\", n_test)\n",
    "print(\"Image data shape =\", image_shape)\n",
    "print(\"Number of classes =\", n_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualizing the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Data exploration visualization function\n",
    "def display(num_row_imgs, num_col_imgs, is_grey = 'false', hspace=0.3, wspace = 0.3):\n",
    "    \n",
    "    fig, axs = plt.subplots(num_row_imgs,num_col_imgs, figsize=(15, 6))\n",
    "    fig.subplots_adjust(hspace, wspace)\n",
    "    axs = axs.ravel()\n",
    "\n",
    "    for i in range(num_row_imgs*num_col_imgs):\n",
    "\n",
    "        index = random.randint(0, len(X_train))\n",
    "        image = X_train[index].squeeze()\n",
    "        if is_grey == 'false':\n",
    "            axs[i].imshow(image)\n",
    "        else:\n",
    "            axs[i].imshow(image, cmap = 'gray')\n",
    "        axs[i].axis('off')\n",
    "        axs[i].set_title(y_train[index])\n",
    "        print(y_train[index], '-->', sign_names[y_train[index]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "# Visualizations will be shown in the notebook.\n",
    "%matplotlib inline\n",
    "\n",
    "import csv\n",
    "import random\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "sign_names = []\n",
    "with open('signnames.csv') as signname_file:\n",
    "    signname_reader = csv.DictReader(signname_file)\n",
    "    sign_names = [row['SignName'] for row in signname_reader]\n",
    "\n",
    "\n",
    "NUM_OF_COL_IMGS = 5\n",
    "NUM_OF_ROW_IMGS = 2\n",
    "\n",
    "     \n",
    "display(NUM_OF_ROW_IMGS, NUM_OF_COL_IMGS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----\n",
    "\n",
    "## Step 2: Design and Test a Model Architecture"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Data Set Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Data Analysis\n",
    "import random\n",
    "import numpy\n",
    "from matplotlib import pyplot\n",
    "import numpy as np\n",
    "\n",
    "# plotting the count of each sign\n",
    "def get_data_stats():\n",
    "\n",
    "    global y_train, y_test\n",
    "    y_pos = range(n_classes)\n",
    "    \n",
    "    \n",
    "    label_list = y_train.tolist()\n",
    "    label_test_list = y_test.tolist()\n",
    "    \n",
    "    sign_type = ([label_list.count(n_class) for n_class in range(n_classes)])\n",
    "    sign_test_type = ([label_test_list.count(n_class) for n_class in range(n_classes)])\n",
    "    \n",
    "    plt.figure(figsize=(20,10))\n",
    "    plt.bar(y_pos, sign_type, tick_label = range(43), label = 'train')\n",
    "    plt.bar(y_pos, sign_test_type, label = 'test')\n",
    "    plt.ylabel('Sample Count')\n",
    "    plt.xlabel('Sample Class')\n",
    "    plt.legend(loc='upper right')\n",
    "    plt.show()\n",
    "\n",
    "    print(\"minimum samples for any label in training data:\", min(np.bincount(y_train)))\n",
    "    print(\"maximum samples for any label in training data:\", max(np.bincount(y_train)))\n",
    "    \n",
    "    print(\"minimum samples for any label in test data:\", min(np.bincount(y_test)))\n",
    "    print(\"maximum samples for any label in test data:\", max(np.bincount(y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_data_stats()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Utility function : Update progress"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#util function -> update progress bar\n",
    "import sys\n",
    "\n",
    "def update_progress(progress, final_count):\n",
    "    barLength = 1 # Modify this to change the length of the progress bar\n",
    "    status = \"\"\n",
    "    if isinstance(progress, int):\n",
    "        progress = float(progress)\n",
    "    if not isinstance(progress, float):\n",
    "        progress = 0\n",
    "        status = \"error: progress var must be float\\r\\n\"\n",
    "    if progress < 0:\n",
    "        progress = 0\n",
    "        status = \"Halt...\\r\\n\"\n",
    "    if progress >= final_count:\n",
    "        progress = final_count\n",
    "        status = \"Done...\\r\\n\"\n",
    "        \n",
    "    \n",
    "    block = int((barLength*progress/final_count)*10)\n",
    "    #print('block->', block, ' barLength->', barLength, ' barlength-block->', (barLength-block), \n",
    "    #     ' progress %', int((progress/final_count)*100))\n",
    "    text = \"\\rPercent: [{0}] {1}% {2}\".format( \"#\"*block + \"-\"*(barLength-block), \n",
    "                                              int((progress/final_count)*100), status)\n",
    "    sys.stdout.write(text)\n",
    "    sys.stdout.flush()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Pre-Processing\n",
    "\n",
    "1. Grey Scale conversion\n",
    "2. Data set normalization\n",
    "3. Histogram Equalization\n",
    "4. Data balancing\n",
    "5. Brightness Augmentation\n",
    "6. Random translation and rotation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#data proprocessing function\n",
    "from skimage import exposure\n",
    "\n",
    "\n",
    "def augment_brightness_camera_images(image):\n",
    "    image1 = cv2.cvtColor(image,cv2.COLOR_RGB2HSV)\n",
    "    random_bright = .25+np.random.uniform()\n",
    "    #print(random_bright)\n",
    "    image1[:,:,2] = image1[:,:,2]*random_bright\n",
    "    image1 = cv2.cvtColor(image1,cv2.COLOR_HSV2RGB)\n",
    "    return image1\n",
    "\n",
    "def transform_image(img,ang_range,shear_range,trans_range,brightness=0):\n",
    "    '''\n",
    "    This function transforms images to generate new images.\n",
    "    The function takes in following arguments,\n",
    "    1- Image\n",
    "    2- ang_range: Range of angles for rotation\n",
    "    3- shear_range: Range of values to apply affine transform to\n",
    "    4- trans_range: Range of values to apply translations over.\n",
    "\n",
    "    A Random uniform distribution is used to generate different parameters for transformation\n",
    "\n",
    "    '''\n",
    "    # Rotation\n",
    "\n",
    "    ang_rot = np.random.uniform(ang_range)-ang_range/2\n",
    "    rows,cols = img.shape    \n",
    "    Rot_M = cv2.getRotationMatrix2D((cols/2,rows/2),ang_rot,1)\n",
    "\n",
    "    # Translation\n",
    "    tr_x = trans_range*np.random.uniform()-trans_range/2\n",
    "    tr_y = trans_range*np.random.uniform()-trans_range/2\n",
    "    Trans_M = np.float32([[1,0,tr_x],[0,1,tr_y]])\n",
    "\n",
    "    # Shear\n",
    "    pts1 = np.float32([[5,5],[20,5],[5,20]])\n",
    "\n",
    "    pt1 = 5+shear_range*np.random.uniform()-shear_range/2\n",
    "    pt2 = 20+shear_range*np.random.uniform()-shear_range/2\n",
    "\n",
    "    # Brightness\n",
    "\n",
    "\n",
    "    pts2 = np.float32([[pt1,5],[pt2,pt1],[5,pt2]])\n",
    "\n",
    "    shear_M = cv2.getAffineTransform(pts1,pts2)\n",
    "\n",
    "    img = cv2.warpAffine(img,Rot_M,(cols,rows))\n",
    "    img = cv2.warpAffine(img,Trans_M,(cols,rows))\n",
    "    img = cv2.warpAffine(img,shear_M,(cols,rows))\n",
    "\n",
    "    if brightness == 1:\n",
    "      img = augment_brightness_camera_images(img)\n",
    "    \n",
    "    #img = pre_process_image(img)\n",
    "\n",
    "    return img\n",
    "\n",
    "#data augmentation\n",
    "def extended_augment_limit_diff():\n",
    "    print('extended augmentation')\n",
    "    global y_train, X_train\n",
    "    \n",
    "    max_samples = max(np.bincount(y_train))\n",
    "    for class_n in range(n_classes):\n",
    "        \n",
    "        print(class_n, ': ', end='')\n",
    "        class_indices = np.where(y_train == class_n)\n",
    "        n_samples = len(class_indices[0])\n",
    "        if n_samples < 800:\n",
    "            for i in range(int((800 - n_samples)/5)):\n",
    "                \n",
    "                arr = []\n",
    "                new_img = X_train[class_indices[0][i % n_samples]]\n",
    "                #plt.imshow(new_img)\n",
    "                #plt.show()\n",
    "                for index in range(5):\n",
    "                    new_img = transform_image(new_img,20,10,10)\n",
    "                    \n",
    "                    #plt.imshow(new_img)\n",
    "                    #plt.show()\n",
    "                    arr.append(new_img)\n",
    "                X_train = np.concatenate((X_train, arr), axis=0)\n",
    "                y_train = np.concatenate((y_train, [class_n]*5), axis=0)\n",
    "                \n",
    "\n",
    "def greyscale(inp_ar):\n",
    "    inp_ar = 0.299 * inp_ar[:, :, :, 0] + 0.587 * inp_ar[:, :, :, 1] + 0.114 * inp_ar[:, :, :, 2]\n",
    "    return inp_ar\n",
    "\n",
    "def normalize(inp_ar):\n",
    "    inp_ar = (inp_ar / 255.).astype(np.float32)\n",
    "    return inp_ar\n",
    "    \n",
    "def hist_equalize(inp_ar):\n",
    "    for i in range(inp_ar.shape[0]):\n",
    "        inp_ar[i] = exposure.equalize_adapthist(inp_ar[i])\n",
    "        update_progress(i, inp_ar.shape[0])\n",
    "    return inp_ar\n",
    "\n",
    "def reshape_after_augment(inp_ar):\n",
    "    inp_ar = np.reshape(inp_ar, (inp_ar.shape[0], inp_ar.shape[2], inp_ar.shape[2], 1))\n",
    "    return inp_ar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Preprocess the data here. It is required to normalize the data. Other preprocessing steps could include \n",
    "### converting to grayscale, etc.\n",
    "import numpy as np\n",
    "import cv2\n",
    "from sklearn.utils import shuffle\n",
    "from skimage import exposure\n",
    "\n",
    "#greyscale conversion\n",
    "#print(X_train.shape)\n",
    "X_train = greyscale(X_train)\n",
    "X_test = greyscale(X_test)\n",
    "X_valid = greyscale(X_valid)\n",
    "print('greyscale conversion done...')\n",
    "\n",
    "#normalization\n",
    "X_train = normalize(X_train)\n",
    "X_test = normalize(X_test)\n",
    "X_valid = normalize(X_valid)\n",
    "print('normalization done...')\n",
    "\n",
    "\n",
    "# Apply localized histogram localization\n",
    "X_train = hist_equalize(X_train)\n",
    "print('train histogram equalization done...')\n",
    "\n",
    "X_test = hist_equalize(X_test)\n",
    "print('test histogram equalization done...')\n",
    "\n",
    "X_valid = hist_equalize(X_valid)\n",
    "print('valid histogram equalization done...')\n",
    "\n",
    "#data augmentation\n",
    "extended_augment_limit_diff()\n",
    "print('data augmentation done')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def reshape_after_augment(inp_ar):\n",
    "    inp_ar = np.reshape(inp_ar, (inp_ar.shape[0], inp_ar.shape[2], inp_ar.shape[2], 1))\n",
    "    return inp_ar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "print(X_train.shape)\n",
    "\n",
    "X_train = reshape_after_augment(X_train)\n",
    "X_test = reshape_after_augment(X_test)\n",
    "X_valid = reshape_after_augment(X_valid)\n",
    "\n",
    "print(X_train.shape)\n",
    "print(X_test.shape)\n",
    "print(X_valid.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize Balanced/Transformed Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(NUM_OF_ROW_IMGS, NUM_OF_COL_IMGS, is_grey = 'true')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Data Statistics of Updated Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_data_stats()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Architecture\n",
    "\n",
    "\n",
    "\n",
    "<img src=\"writeup-images/architecture.png\" width=\"800\" alt=\"Network Architecture\" />"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Modified LeNet Architecture\n",
    "\n",
    "#conv1->relu->pool->dropout->conv2->relu->pool->dropout->conv3->relu->pool->dropout->fc1->rel\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib.layers import flatten\n",
    "\n",
    "conv1_W = None;\n",
    "conv2_W = None;\n",
    "conv3_W = None\n",
    "fc1_W = None;\n",
    "fc2_W = None;\n",
    "\n",
    "def LeNet(x):   \n",
    "    \n",
    "    global conv1_W,conv2_W, conv3_W, fc1_W, fc2_W\n",
    "    # Arguments used for tf.truncated_normal, randomly defines variables for the weights and biases for each layer\n",
    "    mu = 0\n",
    "    sigma = 0.1\n",
    "    \n",
    "     # SOLUTION: Layer 1: Convolutional. Input = 32x32x1. Output = 32x32x16.\n",
    "    conv1_W = tf.Variable(tf.truncated_normal(shape=(5, 5, 1, 16), mean = mu, stddev = sigma))\n",
    "    conv1_b = tf.Variable(tf.zeros(16))\n",
    "    conv1   = tf.nn.conv2d(x, conv1_W, strides=[1, 1, 1, 1], padding='SAME') + conv1_b\n",
    "    print ('conv1-->', conv1)\n",
    "    tf.add_to_collection('vars', conv1_W)\n",
    "    \n",
    "    # SOLUTION: Activation.\n",
    "    conv1 = tf.nn.relu(conv1)\n",
    "    print ('conv1 activation-->', conv1)\n",
    "    # SOLUTION: Pooling. Input = 32x32x16. Output = 16x16x16.\n",
    "    conv1 = tf.nn.max_pool(conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "    print ('conv1 pooling -->', conv1)\n",
    "    \n",
    "    # Dropout\n",
    "    conv1 = tf.nn.dropout(conv1, keep_prob_conv1)\n",
    "    print ('conv1 dropout -->', conv1)\n",
    "    \n",
    "    # SOLUTION: Layer 2: Convolutional. Input = 16x16x16 Output = 16x16x32?\n",
    "    conv2_W = tf.Variable(tf.truncated_normal(shape=(5, 5, 16, 32), mean = mu, stddev = sigma))\n",
    "    conv2_b = tf.Variable(tf.zeros(32))\n",
    "    conv2   = tf.nn.conv2d(conv1, conv2_W, strides=[1, 1, 1, 1], padding='SAME') + conv2_b\n",
    "    print ('conv2-->', conv2)\n",
    "    tf.add_to_collection('vars', conv2_W)\n",
    "    \n",
    "    # SOLUTION: Activation.\n",
    "    conv2 = tf.nn.relu(conv2)\n",
    "    print ('conv2 activation-->', conv2)\n",
    "    \n",
    "    # SOLUTION: Pooling. Input = 16x16x32. Output = 8x8x32.\n",
    "    conv2 = tf.nn.max_pool(conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "    print ('conv2 pooling-->', conv2)\n",
    "    \n",
    "    # Dropout\n",
    "    conv2 = tf.nn.dropout(conv2, keep_prob_conv2)\n",
    "    print ('conv2 dropout-->', conv2)\n",
    "    \n",
    "    # SOLUTION: Layer 3: Convolutional. Input =8x8x32 Output = 8x8x64?\n",
    "    conv3_W = tf.Variable(tf.truncated_normal(shape=(5, 5, 32, 64), mean = mu, stddev = sigma))\n",
    "    conv3_b = tf.Variable(tf.zeros(64))\n",
    "    conv3   = tf.nn.conv2d(conv2, conv3_W, strides=[1, 1, 1, 1], padding='SAME') + conv3_b\n",
    "    print ('conv3-->', conv3)\n",
    "    tf.add_to_collection('vars', conv3_W)\n",
    "    \n",
    "    # SOLUTION: Activation.\n",
    "    conv3 = tf.nn.relu(conv3)\n",
    "    print ('conv3 activation-->', conv3)\n",
    "\n",
    "    # SOLUTION: Pooling. Input = 8x8x64. Output = 4x4x64.\n",
    "    conv3 = tf.nn.max_pool(conv3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "    print ('conv3 pooling-->', conv3)\n",
    "    \n",
    "    # Dropout\n",
    "    conv3 = tf.nn.dropout(conv3, keep_prob_conv3)\n",
    "    print ('conv3 pooling-->', conv3)\n",
    "\n",
    "    # SOLUTION: Flatten. Input = 4x4x64. Output = 1024.\n",
    "    fc0   = flatten(conv3)\n",
    "    print ('flattened layer shape-->', fc0)\n",
    "    \n",
    "    # SOLUTION: Layer 3: Fully Connected. Input = 1024. Output = 2048.\n",
    "    fc1_W = tf.Variable(tf.truncated_normal(shape=(1024, 84), mean = mu, stddev = sigma))\n",
    "    fc1_b = tf.Variable(tf.zeros(84))\n",
    "    fc1   = tf.matmul(fc0, fc1_W) + fc1_b\n",
    "    print ('fc1 shape-->', fc1)\n",
    "    tf.add_to_collection('vars', fc1_W)\n",
    "    \n",
    "    # SOLUTION: Activation.\n",
    "    fc1    = tf.nn.relu(fc1)\n",
    "    print ('fc1 activation-->', fc1)\n",
    "    \n",
    "    #dropout\n",
    "    fc1 = tf.nn.dropout(fc1, keep_prob_fc1)\n",
    "    print ('fc1 dropout-->', fc1)\n",
    "\n",
    "    # SOLUTION: Layer 5: Fully Connected. Input = 84. Output = 43.\n",
    "    fc2_W  = tf.Variable(tf.truncated_normal(shape=(84, 43), mean = mu, stddev = sigma))\n",
    "    fc2_b  = tf.Variable(tf.zeros(43))\n",
    "    logits = tf.matmul(fc1, fc2_W) + fc2_b\n",
    "    print ('logits-->', logits)\n",
    "    tf.add_to_collection('vars', fc2_W)\n",
    "    tf.add_to_collection('vars', logits)\n",
    "    \n",
    "    return logits\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train, Validate and Test the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Train your model here.\n",
    "### Calculate and report the accuracy on the training and validation set.\n",
    "### Once a final model architecture is selected, \n",
    "### the accuracy on the test set should be calculated and reported as well.\n",
    "### Feel free to use as many code cells as needed.\n",
    "from sklearn.utils import shuffle\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "x = tf.placeholder(tf.float32, (None, 32, 32, 1))\n",
    "y = tf.placeholder(tf.int32, (None))\n",
    "one_hot_y = tf.one_hot(y, 43)\n",
    "#keep_prob = tf.placeholder(tf.float32) # probability to keep units\n",
    "keep_prob_conv1 = tf.placeholder(tf.float32) # probability to keep units\n",
    "keep_prob_conv2 = tf.placeholder(tf.float32) # probability to keep units\n",
    "keep_prob_conv3 = tf.placeholder(tf.float32) # probability to keep units\n",
    "keep_prob_fc1 = tf.placeholder(tf.float32) # probability to keep units\n",
    "\n",
    "\n",
    "EPOCHS = 50\n",
    "BATCH_SIZE = 32\n",
    "#ARCH_NAME = '(leNet-1fc)-1-1(hst-eq)-0-4'\n",
    "ARCH_NAME = '3conv-2fc - 1-1-1-5-diff-prob-relu'\n",
    "#ARCH_NAME = 'leNet-1-1-1-5-different-prob'\n",
    "\n",
    "rate = 0.0005\n",
    "\n",
    "\n",
    "logits = LeNet(x)\n",
    "print('logits shape->', logits.get_shape())\n",
    "regularizers = tf.nn.l2_loss(conv1_W) + tf.nn.l2_loss(conv2_W) + tf.nn.l2_loss(conv3_W) + tf.nn.l2_loss(fc1_W) + tf.nn.l2_loss(fc2_W)\n",
    "cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=one_hot_y, logits=logits)\n",
    "loss_operation = tf.reduce_mean(cross_entropy + 0.00005*regularizers)\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate = rate)\n",
    "training_operation = optimizer.minimize(loss_operation)\n",
    "\n",
    "correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(one_hot_y, 1))\n",
    "accuracy_operation = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "def evaluate(X_data, y_data):\n",
    "    num_examples = len(X_data)\n",
    "    total_accuracy = 0\n",
    "    sess = tf.get_default_session()\n",
    "    for offset in range(0, num_examples, BATCH_SIZE):\n",
    "        batch_x, batch_y = X_data[offset:offset+BATCH_SIZE], y_data[offset:offset+BATCH_SIZE]\n",
    "        accuracy = sess.run(accuracy_operation, feed_dict={x: batch_x, y: batch_y, keep_prob_conv1: 1.0, keep_prob_conv2: 1.0, keep_prob_conv3: 1.0, keep_prob_fc1: 1.0})\n",
    "        total_accuracy += (accuracy * len(batch_x))\n",
    "    return total_accuracy / num_examples\n",
    "\n",
    "def plot_graph(x, y, rate, BATCH_SIZE, arch_name):\n",
    "    fig1 = plt.figure()\n",
    "    plt.plot(x, y)\n",
    "    plt.xlim(0, len(x))\n",
    "    plt.ylim(0, 100)\n",
    "    plt.xlabel('epochs')\n",
    "    plt.ylabel('validation-accuracy')\n",
    "    plt.title(arch_name + '-->Validation Accuracy - Epoch')\n",
    "    fig1.savefig('results/' + arch_name + \"__\" + str(rate) + \"_\" + str(BATCH_SIZE) + \".jpeg\", bbox_inches='tight')\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    validation_accuracy = []\n",
    "    num_examples = len(X_train)\n",
    "    \n",
    "    print(\"Training...\")\n",
    "    print()\n",
    "    for i in range(EPOCHS):\n",
    "        X_train, y_train = shuffle(X_train, y_train)\n",
    "        for offset in range(0, num_examples, BATCH_SIZE):\n",
    "            end = offset + BATCH_SIZE\n",
    "            batch_x, batch_y = X_train[offset:end], y_train[offset:end]\n",
    "            sess.run(training_operation, feed_dict={x: batch_x, y: batch_y, keep_prob_conv1: 0.9, keep_prob_conv2: 0.8, keep_prob_conv3: 0.7, keep_prob_fc1: 0.5})\n",
    "         \n",
    "        #print('x_valid shape->', X_valid.shape, ' y_valid shape->', y_valid.shape)\n",
    "        validation_accuracy.append(evaluate(X_valid, y_valid)*100)\n",
    "        print(\"EPOCH {} ...\".format(i+1))\n",
    "        print(\"Validation Accuracy = {:.3f}\".format(validation_accuracy[i]))\n",
    "        print()\n",
    "        \n",
    "    saver.save(sess, './lenet')\n",
    "    print(\"Model saved\")\n",
    "    plot_graph(list(np.arange(1,(EPOCHS+1))), validation_accuracy, rate, BATCH_SIZE, ARCH_NAME)\n",
    "   \n",
    "    \n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, tf.train.latest_checkpoint('.'))\n",
    "\n",
    "    test_accuracy = evaluate(X_test, y_test)*100\n",
    "    print(\"Test Accuracy = {:.3f}\".format(test_accuracy))\n",
    "    with open('results/result.txt', 'a') as f:\n",
    "        f.write('arch-->' + ARCH_NAME + '  epochs->' + str(EPOCHS) + ' batch size-->' + str(BATCH_SIZE) + ' accuracy->' + str(test_accuracy) + '\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    saver = tf.train.import_meta_graph('./lenet.meta')\n",
    "    saver.restore(sess, \"./lenet\")\n",
    "    print(\"Lenet Model restored...\")\n",
    "    Train_accuracy = evaluate(X_train, y_train)\n",
    "    print(\"Train Accuracy = {:.3f}\".format(Train_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## Test a Model on New Images"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and Output the Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Load the images and plot them here.\n",
    "### Feel free to use as many code cells as needed.\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "import cv2\n",
    "import os\n",
    "\n",
    "additional_data_dir = 'additional-data/'\n",
    "    \n",
    "additional_images = []    \n",
    "for filename in os.listdir(additional_data_dir):\n",
    "    if not os.path.isdir(filename):\n",
    "        image = mpimg.imread(os.path.join(additional_data_dir, filename))\n",
    "        print('image file', filename, ' image shape->', image.shape)\n",
    "        image = cv2.resize(image, (32, 32))\n",
    "        #image = np.reshape(image, (32, 32, 3))\n",
    "        plt.imshow(image)\n",
    "        print('image file', filename, ' image shape->', image.shape)\n",
    "        additional_images.append(np.array(image))\n",
    "    \n",
    "additional_images = np.asarray(additional_images)\n",
    "print (additional_images.shape)\n",
    "\n",
    "\n",
    "#greyscale conversion\n",
    "additional_images = 0.299 * additional_images[:, :, :, 0] + 0.587 * additional_images[:, :, :, 1] + 0.114 * additional_images[:, :, :, 2]\n",
    "print('greyscale conversion done...')\n",
    "\n",
    "#normalization\n",
    "additional_images = (additional_images / 255.).astype(np.float32)\n",
    "print('normalization done...')\n",
    "\n",
    "# Apply localized histogram localization  \n",
    "for i in range(additional_images.shape[0]):\n",
    "        additional_images[i] = exposure.equalize_adapthist(additional_images[i])\n",
    "        update_progress(i, additional_images.shape[0])\n",
    "print('train histogram equalization done...')\n",
    "\n",
    "\n",
    "\n",
    "additional_images = np.reshape(additional_images, (additional_images.shape[0], additional_images.shape[2], \n",
    "                                                   additional_images.shape[2], 1))\n",
    "\n",
    "\n",
    "y_additional_images = [18, 1, 34, 14, 2]\n",
    "y_additional_images = np.asarray(y_additional_images)\n",
    "\n",
    "sample_frame_2 = plt.figure(figsize=(6,6))\n",
    "axes = []\n",
    "for i in range(0, len(additional_images)):\n",
    "    axis_2 = sample_frame_2.add_subplot(2,5,i+1)\n",
    "    axis_2.set_xlabel(y_additional_images[i])\n",
    "    plt.xticks(np.array([]))\n",
    "    plt.yticks(np.array([]))\n",
    "    axis_2.imshow(additional_images[i].squeeze(), cmap = 'gray')\n",
    "plt.tight_layout(h_pad=0, w_pad=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict the Sign Type for Each Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Run the predictions here and use the model to output the prediction for each image.\n",
    "### Make sure to pre-process the images with the same pre-processing pipeline used earlier.\n",
    "### Feel free to use as many code cells as needed.\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    saver3 = tf.train.import_meta_graph('./lenet.meta')\n",
    "    saver3.restore(sess, \"./lenet\")\n",
    "    internet_accuracy = evaluate(additional_images, y_additional_images)\n",
    "    print(\"Additional Image Test Accuracy = {:.3f}\".format(internet_accuracy))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Analyze Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Calculate the accuracy for these 5 new images. \n",
    "### For example, if the model predicted 1 out of 5 signs correctly, it's 20% accurate on these new images.\n",
    "\n",
    "### Calculate the accuracy for these 5 new images. \n",
    "### For example, if the model predicted 1 out of 5 signs correctly, it's 20% accurate on these new images.\n",
    "\n",
    "Internet_results = []\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    saver3 = tf.train.import_meta_graph('./lenet.meta')\n",
    "    saver3.restore(sess, \"./lenet\")\n",
    "    prediction = tf.argmax(logits, 1)\n",
    "    result = sess.run(prediction, feed_dict={x: additional_images, y: y_additional_images, keep_prob_conv1: 1.0, keep_prob_conv2: 1.0, keep_prob_conv3: 1.0, keep_prob_fc1: 1.0})\n",
    "    Internet_results = result\n",
    "    print(result)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Output Top 5 Softmax Probabilities For Each Image Found on the Web"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Print out the top five softmax probabilities for the predictions on the German traffic sign images found on the web. \n",
    "### Feel free to use as many code cells as needed.\n",
    "\n",
    "### Print out the top five softmax probabilities for the predictions on the German traffic sign images found on the web. \n",
    "### Feel free to use as many code cells as needed.\n",
    "\n",
    "softmax = tf.nn.softmax(logits)\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    saver3 = tf.train.import_meta_graph('./lenet.meta')\n",
    "    saver3.restore(sess, \"./lenet\")\n",
    "    probabilities = sess.run(softmax, feed_dict={x: additional_images, y: y_additional_images, keep_prob_conv1: 1.0, keep_prob_conv2: 1.0, keep_prob_conv3: 1.0, keep_prob_fc1: 1.0})\n",
    "    result_top_five = sess.run(tf.nn.top_k(probabilities, k=5))\n",
    "    print (result_top_five)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## Step : Visualize the Neural Network's State with Test Images\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Visualize your network's feature maps here.\n",
    "### Feel free to use as many code cells as needed.\n",
    "\n",
    "# image_input: the test image being fed into the network to produce the feature maps\n",
    "# tf_activation: should be a tf variable name used during your training procedure that represents the calculated state of a specific weight layer\n",
    "# activation_min/max: can be used to view the activation contrast in more detail, by default matplot sets min and max to the actual min and max values of the output\n",
    "# plt_num: used to plot out multiple different weight feature map sets on the same block, just extend the plt number for each new feature map entry\n",
    "\n",
    "def outputFeatureMap(image_input, tf_activation, activation_min=-1, activation_max=-1 ,plt_num=1):\n",
    "    # Here make sure to preprocess your image_input in a way your network expects\n",
    "    # with size, normalization, ect if needed\n",
    "    # image_input =\n",
    "    # Note: x should be the same name as your network's tensorflow data placeholder variable\n",
    "    # If you get an error tf_activation is not defined it may be having trouble accessing the variable from inside a function\n",
    "    activation = tf_activation.eval(session=sess,feed_dict={x : image_input, keep_prob_conv1:1.0, keep_prob_conv2:1.0, keep_prob_conv3:1.0, keep_prob_fc1:1.0})\n",
    "    featuremaps = activation.shape[3]\n",
    "    print('activation', featuremaps)\n",
    "    plt.figure(plt_num, figsize=(15,15))\n",
    "    for featuremap in range(featuremaps):\n",
    "        plt.subplot(8,8, featuremap+1) # sets the number of feature maps to show on each row and column\n",
    "        plt.title('FeatureMap ' + str(featuremap)) # displays the feature map number\n",
    "        plt.axis('off')\n",
    "        if activation_min != -1 & activation_max != -1:\n",
    "            plt.imshow(activation[0,:,:, featuremap], interpolation=\"nearest\", vmin =activation_min, vmax=activation_max)#, cmap=\"gray\")\n",
    "        elif activation_max != -1:\n",
    "            plt.imshow(activation[0,:,:, featuremap], interpolation=\"nearest\", vmax=activation_max)#, cmap=\"gray\")\n",
    "        elif activation_min !=-1:\n",
    "            plt.imshow(activation[0,:,:, featuremap], interpolation=\"nearest\", vmin=activation_min)#, cmap=\"gray\")\n",
    "        else:\n",
    "            plt.imshow(activation[0,:,:, featuremap], interpolation=\"nearest\")#, cmap=\"gray\")\n",
    "            \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "additional_data_dir = 'additional-data/'\n",
    "    \n",
    "#additional_images = []    \n",
    "for filename in os.listdir(additional_data_dir):\n",
    "    if not os.path.isdir(filename):\n",
    "        image = mpimg.imread(os.path.join(additional_data_dir, filename))\n",
    "        print('image file', filename, ' image shape->', image.shape)\n",
    "        \n",
    "        plt.imshow(image)\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "l_names = ['add_0:0', 'add_1:0', 'add_2:0']\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    saver3 = tf.train.import_meta_graph('./lenet.meta')\n",
    "    saver3.restore(sess, \"./lenet\")\n",
    "    layer = sess.graph.get_tensor_by_name(l_names[1])\n",
    "    outputFeatureMap(image_input=[additional_images[0]], tf_activation=layer)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}