Development-CNN_Backend.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "This jupyter notebook was used to develop the convolutional neural network model, which will be used to make predictions.  Fitting the model to 60,000 images, was prohibitive on my local machine, so it this was converted to 'training_app.py', which was processed in FloydHub.  The model was saved to disk to be used for future predictions. \n",
    "\n",
    "Handwritten Digit Recognition\n",
    "\n",
    "Train model in Keras & Save it\n",
    "Flask Backend Python Micro Framework\n",
    "Deploy code to Google Cloud\n",
    "\n",
    "#dense means fully connected layers, dropout is a technique to improve convergence, flatten to reshape our matrices for feeding into respective layers\n",
    "#for convolution (images) and pooling is a technique to help choose the most relevant features in an image\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/Users/ohm/anaconda3/envs/deeplearning/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/Users/ohm/anaconda3/envs/deeplearning/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/Users/ohm/anaconda3/envs/deeplearning/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/Users/ohm/anaconda3/envs/deeplearning/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/Users/ohm/anaconda3/envs/deeplearning/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/Users/ohm/anaconda3/envs/deeplearning/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/Users/ohm/anaconda3/envs/deeplearning/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/Users/ohm/anaconda3/envs/deeplearning/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/Users/ohm/anaconda3/envs/deeplearning/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/Users/ohm/anaconda3/envs/deeplearning/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/Users/ohm/anaconda3/envs/deeplearning/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/Users/ohm/anaconda3/envs/deeplearning/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "from keras import backend as K\n",
    "from keras.datasets import mnist\n",
    "from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D\n",
    "from keras.models import Sequential\n",
    "from keras.utils import to_categorical\n",
    "from keras.losses import categorical_crossentropy\n",
    "from keras.optimizers import Adadelta\n",
    "import matplotlib.pyplot as pyplot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "epochs = 12\n",
    "img_rows, img_cols = 28, 28\n",
    "image_height = image_width = 28\n",
    "square = (2, 2)\n",
    "cube = (3, 3)\n",
    "drpout_rt_1 = 0.2\n",
    "drpout_rt_2 = 0.5\n",
    "activation = 'relu'\n",
    "loss='categorical_crossentropy'\n",
    "optimizer=Adadelta(1.0, 0.95)\n",
    "metrics=['accuracy']\n",
    "activation_dense='softmax'\n",
    "hl='–'*25\n",
    "num_classes = 10 #y_train.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-- From Constantin on stackoverflow; print variable name\n",
    "```\n",
    "import inspect, re\n",
    "\n",
    "def varname(p):\n",
    "  for line in inspect.getframeinfo(inspect.currentframe().f_back)[3]:\n",
    "    m = re.search(r'\\bvarname\\s*\\(\\s*([A-Za-z_][A-Za-z0-9_]*)\\s*\\)', line)\n",
    "    if m:\n",
    "      return m.group(1)\n",
    "      ```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The Keras data format is \t'channels_last'\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––\n",
      "The shape of x_train is \t(60000, 28, 28)\n",
      "The shape of y_train is \t(60000,)\n",
      "The shape of x_test is \t\t(10000, 28, 28)\n",
      "The shape of y_test is \t\t(10000,)\n",
      "––––––––––––––––––––––––––––––––––––––––––––––––––––\n"
     ]
    }
   ],
   "source": [
    "print(f'The Keras data format is \\t\\'{K.image_data_format()}\\'')\n",
    "print('––––––––––––––––––––––––––––––––––––––––––––––––––––')\n",
    "print(f'The shape of x_train is \\t{x_train.shape}')\n",
    "print(f'The shape of y_train is \\t{y_train.shape}')\n",
    "print(f'The shape of x_test is \\t\\t{x_test.shape}')\n",
    "print(f'The shape of y_test is \\t\\t{y_test.shape}')\n",
    "print('––––––––––––––––––––––––––––––––––––––––––––––––––––')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "–––––––––––––––––––––––––\n",
      "reshaping succesful... \n",
      "–––––––––––––––––––––––––\n",
      "The shape of x_train is (60000, 28, 28, 1)\n",
      "            The shape of y_train is (60000,)\n",
      "            The shape of x_test is (10000, 28, 28, 1)\n",
      "            The shape of y_test is (10000,)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#this assumes our data format\n",
    "#For 3D data, \"channels_last\" assumes (conv_dim1, conv_dim2, conv_dim3, channels) while \n",
    "#\"channels_first\" assumes (channels, conv_dim1, conv_dim2, conv_dim3).\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)\n",
    "    x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "    print(hl)\n",
    "    print('reshaping succesful... ')\n",
    "    print(f'The shape of x_train is {x_train.shape}')\n",
    "    print(f'The shape of y_train is {y_train.shape}')\n",
    "    print(f'The shape of x_test is {x_test.shape}')\n",
    "    print(f'The shape of y_test is {y_test.shape}')\n",
    "elif K.image_data_format() == 'channels_last':\n",
    "    x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)\n",
    "    x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)\n",
    "    print(hl)\n",
    "    print('reshaping succesful... ')\n",
    "    print(hl)\n",
    "    print(f'The shape of x_train is {x_train.shape}\\n\\\n",
    "            The shape of y_train is {y_train.shape}\\n\\\n",
    "            The shape of x_test is {x_test.shape}\\n\\\n",
    "            The shape of y_test is {y_test.shape}\\n')\n",
    "else: print('Invalid Format Submitted')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#more reshaping\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert class vectors to binary class matrices\n",
    "y_train = to_categorical(y_train, num_classes)\n",
    "y_test = to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 10 distinct categories\n"
     ]
    }
   ],
   "source": [
    "if y_train.shape[1] == y_test.shape[1]:\n",
    "    print(f'There are {y_train.shape[1]} distinct categories')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define Convolutional Neural Network Using keras.Sequential\n",
    "def define_model():\n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(32, \n",
    "                     kernel_size=cube, \n",
    "                     activation='relu', \n",
    "                     kernel_initializer='he_uniform', \n",
    "                     padding='same', \n",
    "                     input_shape=(image_height, image_width, 1)\n",
    "                    )\n",
    "             )\n",
    "    model.add(MaxPooling2D(square))\n",
    "    model.add(Dropout(drpout_rt_1))\n",
    "    model.add(Conv2D(64, \n",
    "                     cube, \n",
    "                     activation='relu', \n",
    "                     kernel_initializer='he_uniform', \n",
    "                     padding='same'\n",
    "                    )\n",
    "             )\n",
    "    model.add(MaxPooling2D(square))\n",
    "    model.add(Dropout(drpout_rt_1))\n",
    "    model.add(Conv2D(128, \n",
    "                     cube, \n",
    "                     activation='relu', \n",
    "                     kernel_initializer='he_uniform', \n",
    "                     padding='same'\n",
    "                    )\n",
    "             )\n",
    "    model.add(MaxPooling2D(square))\n",
    "    model.add(Dropout(drpout_rt_1))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(128, \n",
    "                    activation='relu', \n",
    "                    kernel_initializer='he_uniform'\n",
    "                   )\n",
    "             )\n",
    "    model.add(Dropout(drpout_rt_2))\n",
    "    model.add(Dense(num_classes, activation=activation_dense))\n",
    "    \n",
    "    # compile model\n",
    "    model.compile(optimizer=optimizer, \n",
    "                  loss=loss, \n",
    "                  metrics=metrics)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# plot diagnostic learning curves\n",
    "def summarize_diagnostics(history):\n",
    "    # plot loss\n",
    "    pyplot.subplot(211)\n",
    "    pyplot.title('Cross Entropy Loss')\n",
    "    pyplot.plot(history.history['loss'], color='blue', label='train')\n",
    "    pyplot.plot(history.history['val_loss'], color='orange', label='test')\n",
    "    # plot accuracy\n",
    "    pyplot.subplot(212)\n",
    "    pyplot.title('Classification Accuracy')\n",
    "    pyplot.plot(history.history['accuracy'], color='blue', label='train')\n",
    "    pyplot.plot(history.history['val_accuracy'], color='orange', label='test')\n",
    "    # save plot to file\n",
    "    filename = sys.argv[0].split('/')[-1]\n",
    "    pyplot.savefig(filename + '_plot.png')\n",
    "    pyplot.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/ohm/anaconda3/envs/deeplearning/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:4070: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model = define_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model is fit on 1 epoch here, but fit on 25 epochs on FloydHub.com\n",
    "\n",
    "epochs = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/ohm/anaconda3/envs/deeplearning/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/1\n",
      "60000/60000 [==============================] - 1893s 32ms/step - loss: 0.4483 - accuracy: 0.8541 - val_loss: 0.0692 - val_accuracy: 0.9774\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x640f02650>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.06924215244501829\n",
      "Test accuracy: 0.977400004863739\n"
     ]
    }
   ],
   "source": [
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model saved to disk\n"
     ]
    }
   ],
   "source": [
    "model.save('MNIST_CNN.h5')\n",
    "del model\n",
    "print('model saved to disk')\n",
    "summarize_diagnostics(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Deep Learning",
   "language": "python",
   "name": "deeplearning"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}