cifar-model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from matplotlib.pyplot import imshow\n",
    "from tensorflow.contrib.layers.python.layers import batch_norm\n",
    "from tensorflow.python.framework import ops\n",
    "import os\n",
    "TRAIN = False\n",
    "INFER = True\n",
    "BATCH_SIZE = 128\n",
    "SAVE = True\n",
    "global_step = tf.contrib.framework.get_or_create_global_step()\n",
    "import urllib, StringIO\n",
    "from PIL import Image\n",
    "\n",
    "class_names = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "INFER_URLS = [\n",
    "    ('deer', 'http://gfp.sd.gov/hunting/big-game/images/deer1-01.jpg'),\n",
    "    ('frog', 'http://www.gaylordfunkyfish.com/images/Red-Eye-Tree-Frog-300x244.png'),\n",
    "    ('airplane', 'https://media.licdn.com/mpr/mpr/jc/AAEAAQAAAAAAAAMiAAAAJGVlYTU5Y2YyLWQwMzYtNDlmZS04MDdlLWI0ZjJjZWRhYjk4ZQ.jpg'),\n",
    "    ('ship', 'http://www.vships.com/media/92241/passenger-vessel.jpg'),\n",
    "    ('dog', 'https://images-na.ssl-images-amazon.com/images/G/01/img15/pet-products/small-tiles/23695_pets_vertical_store_dogs_small_tile_8._CB312176604_.jpg'),\n",
    "    ('bird', 'http://www.audubon.org/sites/default/files/styles/engagement_card/public/sfw_apa_2013_28342_232388_briankushner_blue_jay_kk_high.jpg')\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def read_and_decode_single_example(filename, epochs=None):\n",
    "    # first construct a queue containing a list of filenames.\n",
    "    # this lets a user split up there dataset in multiple files to keep\n",
    "    # size down\n",
    "    filename_queue = tf.train.string_input_producer([filename],\n",
    "                                                    num_epochs=epochs)\n",
    "    # Unlike the TFRecordWriter, the TFRecordReader is symbolic\n",
    "    reader = tf.TFRecordReader()\n",
    "    # One can read a single serialized example from a filename\n",
    "    # serialized_example is a Tensor of type string.\n",
    "    _, serialized_example = reader.read(filename_queue)\n",
    "    # The serialized example is converted back to actual values.\n",
    "    # One needs to describe the format of the objects to be returned\n",
    "    features = tf.parse_single_example(\n",
    "        serialized_example,\n",
    "        features={\n",
    "            # We know the length of both fields. If not the\n",
    "            # tf.VarLenFeature could be used\n",
    "            'label': tf.FixedLenFeature([], tf.int64),\n",
    "            'image': tf.FixedLenFeature([], 'string')\n",
    "        })\n",
    "    # now return the converted data\n",
    "    label = features['label']\n",
    "    image_raw = features['image']\n",
    "    image = tf.decode_raw(image_raw, tf.uint8)\n",
    "    image = tf.reshape(image, [32,32,3])\n",
    "    image = tf.cast(image, tf.float32) / 255\n",
    "    return label, image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def distort_image(image):\n",
    "    distorted_image = tf.random_crop(image, [24, 24, 3])\n",
    "    distorted_image = tf.image.random_flip_left_right(distorted_image)\n",
    "    distorted_image = tf.image.random_brightness(distorted_image, max_delta=0.2)\n",
    "    distorted_image = tf.image.random_contrast(distorted_image, lower=0.7, upper=1.3)\n",
    "    # distorted_image = tf.image.per_image_whitening(distorted_image) # renamed to per_image_standardization in latest release\n",
    "    distorted_image = tf.clip_by_value(distorted_image, 0, 1)\n",
    "    return distorted_image\n",
    "\n",
    "def simple_crop(image):\n",
    "    return tf.random_crop(image, [24, 24, 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import urllib2\n",
    "import matplotlib.pyplot as plt\n",
    "import PIL\n",
    "def load_image(url):\n",
    "    file = StringIO.StringIO(urllib.urlopen(url).read())\n",
    "    img = Image.open(file)\n",
    "    img = img.resize((24,24), PIL.Image.BICUBIC)\n",
    "    img = img.convert('RGB')\n",
    "    # plt.imshow(img)\n",
    "    img = (np.array(img.getdata()) / 255.0).reshape((24,24,3))\n",
    "    # drop last dimension (alpha):\n",
    "    return img\n",
    "img = load_image('https://images-na.ssl-images-amazon.com/images/G/01/img15/pet-products/small-tiles/23695_pets_vertical_store_dogs_small_tile_8._CB312176604_.jpg')\n",
    "# 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "# create image batches:\n",
    "\n",
    "if INFER:\n",
    "    queue1 = tf.FIFOQueue(capacity=10, dtypes=[tf.float32], shapes=[(24,24,3)])\n",
    "    enq = queue1.enqueue_many(np.array([load_image(url) for _, url in INFER_URLS]))\n",
    "    close_input_q = queue1.close()\n",
    "    images_batch = tf.train.batch([queue1.dequeue()], batch_size=BATCH_SIZE, allow_smaller_final_batch=True)\n",
    "    \n",
    "#     sess = tf.Session()\n",
    "#     tf.train.start_queue_runners(sess=sess)\n",
    "#     sess.run(enq)\n",
    "#     sess.run(close)\n",
    "#     img1 = sess.run(images_batch)\n",
    "    labels_batch = tf.placeholder(tf.int64, [None])\n",
    "else:\n",
    "    filename = \"cifar-train.tfrecords\" if TRAIN else \"cifar-test.tfrecords\"\n",
    "    label, image = read_and_decode_single_example(filename, epochs=(None if TRAIN else 1))\n",
    "    if TRAIN:\n",
    "        images_batch, labels_batch = tf.train.shuffle_batch([distort_image(image), label], batch_size=BATCH_SIZE, capacity=1000, min_after_dequeue=500)\n",
    "    else:\n",
    "        images_batch, labels_batch = tf.train.batch([simple_crop(image), label], batch_size=BATCH_SIZE, allow_smaller_final_batch=True)\n",
    "\n",
    "# print 'ya'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dropout_keep_prob = tf.placeholder(tf.float32, name='dropout_keep_prob')        \n",
    "\n",
    "def weight_var(shape, stddev=0.1, weight_decay=0):\n",
    "    initial = tf.truncated_normal(shape, stddev=stddev)\n",
    "    v = tf.Variable(initial)\n",
    "    if weight_decay > 0:\n",
    "        l2 = tf.nn.l2_loss(v) * weight_decay\n",
    "        tf.add_to_collection('losses', l2)\n",
    "    return v\n",
    "\n",
    "def create_fc(input, out_size):\n",
    "    # input_dropped = tf.nn.dropout(input, dropout_keep_prob)\n",
    "    in_size = input.get_shape()[-1].value\n",
    "    w = weight_var([in_size, out_size], weight_decay=0.004)\n",
    "    b = weight_var([out_size], weight_decay=0.004)\n",
    "    x = tf.matmul(input, w)\n",
    "    return tf.nn.relu(x + b)\n",
    "\n",
    "def create_conv(input, out_channels, patch_size=3, stride=1, batch_norm=False, dropout=False):\n",
    "    in_channels = input.get_shape()[-1].value\n",
    "    w = weight_var([patch_size, patch_size, in_channels, out_channels])\n",
    "    b = weight_var([out_channels], stddev=0)\n",
    "    conv = tf.nn.conv2d(input, w, strides=[1,stride,stride,1], padding='SAME')\n",
    "    if batch_norm: conv = create_batch_norm(conv)\n",
    "    activation = tf.nn.relu(conv + b)\n",
    "    if dropout: activation = create_dropout(activation)\n",
    "    return activation\n",
    "\n",
    "def create_max_pool(inputs, ksize=2, stride=2):\n",
    "    return tf.nn.max_pool(inputs, ksize=[1, ksize, ksize, 1], strides=[1, stride, stride, 1], padding='SAME')\n",
    "\n",
    "def create_batch_norm(inputs):\n",
    "    return batch_norm(inputs, is_training=TRAIN)\n",
    "\n",
    "def create_dropout(inputs):\n",
    "    return tf.nn.dropout(inputs, dropout_keep_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def forward(images):\n",
    "    # images are 24x24x3\n",
    "    for size in [32, 64, 64]:\n",
    "        images = create_conv(images, size, dropout=True, batch_norm=False)\n",
    "        images = create_max_pool(images)\n",
    "    # now images are 6x6x32 = 1176:\n",
    "    n_channels = images.get_shape()[-1].value\n",
    "    vecs = tf.reshape(images, [-1, 3*3*n_channels])\n",
    "    # vecs = tf.reshape(images, [-1, 24*24*3])\n",
    "    vecs = create_fc(vecs, 512)\n",
    "    vecs = batch_norm(vecs)\n",
    "    logits = create_fc(vecs, 10)\n",
    "    return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "NUM_EPOCHS_PER_DECAY = 300.0      # Epochs after which learning rate decays.\n",
    "LEARNING_RATE_DECAY_FACTOR = 0.1  # Learning rate decay factor.\n",
    "INITIAL_LEARNING_RATE = 0.001       # Initial learning rate.\n",
    "EXAMPLES_PER_EPOCH = 60000\n",
    "\n",
    "learn_rate = tf.train.exponential_decay(INITIAL_LEARNING_RATE,\n",
    "                                  global_step,\n",
    "                                  EXAMPLES_PER_EPOCH * NUM_EPOCHS_PER_DECAY / BATCH_SIZE,\n",
    "                                  LEARNING_RATE_DECAY_FACTOR,\n",
    "                                  staircase=False)\n",
    "\n",
    "# create optimizer:\n",
    "logits = forward(images_batch)\n",
    "cross_entropy = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits, labels_batch))\n",
    "loss = cross_entropy # + tf.add_n(tf.get_collection('losses'))\n",
    "optimizer = tf.train.AdamOptimizer(learn_rate)\n",
    "train_step = optimizer.minimize(loss, global_step=global_step)\n",
    "\n",
    "probs = tf.nn.softmax(logits)\n",
    "predictions = tf.argmax(probs, 1)\n",
    "n_correct = tf.reduce_sum(tf.cast(tf.equal(predictions, labels_batch), tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Restored from checkpoint models/1/model.ckpt-9900\n"
     ]
    }
   ],
   "source": [
    "# 16 is a good model\n",
    "save_path = 'models/1'\n",
    "if not os.path.exists(save_path):\n",
    "    os.mkdir(save_path)\n",
    "\n",
    "session = tf.Session()\n",
    "init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())\n",
    "session.run(init_op)\n",
    "    \n",
    "saver = tf.train.Saver()\n",
    "ckpt = tf.train.get_checkpoint_state(save_path)\n",
    "if ckpt and ckpt.model_checkpoint_path:\n",
    "    saver.restore(session, ckpt.model_checkpoint_path)\n",
    "    print 'Restored from checkpoint', ckpt.model_checkpoint_path\n",
    "else:\n",
    "    print 'Did not restore from checkpoint'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "It's actually a ('deer', 'http://gfp.sd.gov/hunting/big-game/images/deer1-01.jpg')\n",
      "# airplane\n",
      "# automobile\n",
      "# bird\n",
      "# cat\n",
      "######### deer\n",
      "# dog\n",
      "# frog\n",
      "########## horse\n",
      "# ship\n",
      "# truck\n",
      "\n",
      "\n",
      "\n",
      "It's actually a ('frog', 'http://www.gaylordfunkyfish.com/images/Red-Eye-Tree-Frog-300x244.png')\n",
      "# airplane\n",
      "# automobile\n",
      "# bird\n",
      "# cat\n",
      "# deer\n",
      "# dog\n",
      "################### frog\n",
      "# horse\n",
      "# ship\n",
      "# truck\n",
      "\n",
      "\n",
      "\n",
      "It's actually a ('airplane', 'https://media.licdn.com/mpr/mpr/jc/AAEAAQAAAAAAAAMiAAAAJGVlYTU5Y2YyLWQwMzYtNDlmZS04MDdlLWI0ZjJjZWRhYjk4ZQ.jpg')\n",
      "########### airplane\n",
      "# automobile\n",
      "# bird\n",
      "# cat\n",
      "# deer\n",
      "# dog\n",
      "# frog\n",
      "# horse\n",
      "###### ship\n",
      "## truck\n",
      "\n",
      "\n",
      "\n",
      "It's actually a ('ship', 'http://www.vships.com/media/92241/passenger-vessel.jpg')\n",
      "# airplane\n",
      "# automobile\n",
      "# bird\n",
      "# cat\n",
      "# deer\n",
      "# dog\n",
      "# frog\n",
      "# horse\n",
      "################## ship\n",
      "## truck\n",
      "\n",
      "\n",
      "\n",
      "It's actually a ('dog', 'https://images-na.ssl-images-amazon.com/images/G/01/img15/pet-products/small-tiles/23695_pets_vertical_store_dogs_small_tile_8._CB312176604_.jpg')\n",
      "# airplane\n",
      "# automobile\n",
      "# bird\n",
      "### cat\n",
      "###### deer\n",
      "######### dog\n",
      "# frog\n",
      "### horse\n",
      "# ship\n",
      "# truck\n",
      "\n",
      "\n",
      "\n",
      "It's actually a ('bird', 'http://www.audubon.org/sites/default/files/styles/engagement_card/public/sfw_apa_2013_28342_232388_briankushner_blue_jay_kk_high.jpg')\n",
      "# airplane\n",
      "# automobile\n",
      "####### bird\n",
      "##### cat\n",
      "# deer\n",
      "##### dog\n",
      "# frog\n",
      "## horse\n",
      "# ship\n",
      "# truck\n",
      "\n",
      "\n",
      "\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "tf.train.start_queue_runners(sess=session)\n",
    "if TRAIN:\n",
    "    while True:\n",
    "        feed_dict = {dropout_keep_prob: 0.7}\n",
    "        _, cur_loss, step, pred_labels, lr = session.run([train_step, loss, global_step, predictions, learn_rate], feed_dict=feed_dict)\n",
    "        if step % 50 == 1:\n",
    "            print \"Step: {0}; Loss: {1}; learn rate: {2}\".format(step, cur_loss, lr)\n",
    "        if step % 150 == 0:\n",
    "            if SAVE:\n",
    "                saver.save(session, save_path + '/model.ckpt', global_step=step)\n",
    "                print 'saved'\n",
    "            print pred_labels\n",
    "elif INFER:\n",
    "    i = 0\n",
    "    session.run(enq)\n",
    "    session.run(close_input_q)\n",
    "    while True:\n",
    "        try:\n",
    "            feed_dict = {dropout_keep_prob: 1}\n",
    "            all_scores, preds = session.run([probs, predictions], feed_dict=feed_dict)\n",
    "            for scores in all_scores:\n",
    "                print \"It's actually a\", INFER_URLS[i]\n",
    "                for cls, score in zip(class_names, scores):\n",
    "                    print '#' * int(1 + 20 * score), cls\n",
    "                i += 1\n",
    "                print '\\n\\n'\n",
    "        except tf.errors.OutOfRangeError:\n",
    "            print 'Done!'\n",
    "            break\n",
    "else: # evaluate:\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    i = 0\n",
    "    while True:\n",
    "        try:\n",
    "            feed_dict = {dropout_keep_prob: 1}\n",
    "            nc, pred = session.run([n_correct, predictions], feed_dict=feed_dict)\n",
    "            correct += nc\n",
    "            # if i % 10 == 0:\n",
    "            #     print pred\n",
    "            # this assumes all batches are BATCH_SIZE (the last one might be smaller) but that's okay\n",
    "            total += BATCH_SIZE\n",
    "            i += 1\n",
    "            if i % 10 == 0:\n",
    "                print \"Accuracy so far: {0}\".format(correct * 1.0 / total)\n",
    "        except tf.errors.OutOfRangeError:\n",
    "            print 'Done!'\n",
    "            print \"Final accuracy: {0}\".format(correct * 1.0 / total)\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}