Best Student.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def cnn_model_fn(features, labels, mode):\n",
    "  \"\"\"Model function for CNN.\"\"\"\n",
    "  # Input Layer\n",
    "  # Reshape X to 4-D tensor: [batch_size, width, height, channels]\n",
    "  # MNIST images are 28x28 pixels, and have one color channel\n",
    "  input_layer = tf.reshape(features[\"x\"], [-1, 28, 28, 1])\n",
    "  print(\"_THIS_\",type(features))\n",
    "  # Convolutional Layer #1\n",
    "  # Computes 32 features using a 5x5 filter with ReLU activation.\n",
    "  # Padding is added to preserve width and height.\n",
    "  # Input Tensor Shape: [batch_size, 28, 28, 1]\n",
    "  # Output Tensor Shape: [batch_size, 28, 28, 32]\n",
    "  conv1 = tf.layers.conv2d(\n",
    "      inputs=input_layer,\n",
    "      filters=32,\n",
    "      kernel_size=[5, 5],\n",
    "      padding=\"same\",\n",
    "      activation=tf.nn.relu)\n",
    "\n",
    "  # Pooling Layer #1\n",
    "  # First max pooling layer with a 2x2 filter and stride of 2\n",
    "  # Input Tensor Shape: [batch_size, 28, 28, 32]\n",
    "  # Output Tensor Shape: [batch_size, 14, 14, 32]\n",
    "  pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2)\n",
    "\n",
    "  # Convolutional Layer #2\n",
    "  # Computes 64 features using a 5x5 filter.\n",
    "  # Padding is added to preserve width and height.\n",
    "  # Input Tensor Shape: [batch_size, 14, 14, 32]\n",
    "  # Output Tensor Shape: [batch_size, 14, 14, 64]\n",
    "  conv2 = tf.layers.conv2d(\n",
    "      inputs=pool1,\n",
    "      filters=64,\n",
    "      kernel_size=[5, 5],\n",
    "      padding=\"same\",\n",
    "      activation=tf.nn.relu)\n",
    "\n",
    "  # Pooling Layer #2\n",
    "  # Second max pooling layer with a 2x2 filter and stride of 2\n",
    "  # Input Tensor Shape: [batch_size, 14, 14, 64]\n",
    "  # Output Tensor Shape: [batch_size, 7, 7, 64]\n",
    "  pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2)\n",
    "\n",
    "  # Flatten tensor into a batch of vectors\n",
    "  # Input Tensor Shape: [batch_size, 7, 7, 64]\n",
    "  # Output Tensor Shape: [batch_size, 7 * 7 * 64]\n",
    "  pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64])\n",
    "\n",
    "  # Dense Layer\n",
    "  # Densely connected layer with 1024 neurons\n",
    "  # Input Tensor Shape: [batch_size, 7 * 7 * 64]\n",
    "  # Output Tensor Shape: [batch_size, 1024]\n",
    "  dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu)\n",
    "\n",
    "  # Add dropout operation; 0.6 probability that element will be kept\n",
    "  dropout = tf.layers.dropout(\n",
    "      inputs=dense, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN)\n",
    "\n",
    "  # Logits layer\n",
    "  # Input Tensor Shape: [batch_size, 1024]\n",
    "  # Output Tensor Shape: [batch_size, 10]\n",
    "  logits = tf.layers.dense(inputs=dropout, units=10)\n",
    "\n",
    "  predictions = {\n",
    "      # Generate predictions (for PREDICT and EVAL mode)\n",
    "      \"classes\": tf.argmax(input=logits, axis=1),\n",
    "      # Add `softmax_tensor` to the graph. It is used for PREDICT and by the\n",
    "      # `logging_hook`.\n",
    "      \"probabilities\": tf.nn.softmax(logits/temprature, name=\"softmax_tensor\")\n",
    "  }\n",
    "  if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "    return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)\n",
    "\n",
    "  # Calculate Loss (for both TRAIN and EVAL modes)\n",
    "  loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)\n",
    "\n",
    "  # Configure the Training Op (for TRAIN mode)\n",
    "  if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)\n",
    "    train_op = optimizer.minimize(\n",
    "        loss=loss,\n",
    "        global_step=tf.train.get_global_step())\n",
    "    return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)\n",
    "\n",
    "  # Add evaluation metrics (for EVAL mode)\n",
    "  eval_metric_ops = {\n",
    "      \"accuracy\": tf.metrics.accuracy(\n",
    "          labels=labels, predictions=predictions[\"classes\"]),\n",
    "  }\n",
    "  return tf.estimator.EstimatorSpec(\n",
    "      mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#cross entropy between two logits with temperature applied\n",
    "#returns a 1d array with loss for each row in the inputs\n",
    "def cross_entropy2(tar_soft_t, pred_logits_t):\n",
    "   pred_soft = tf.nn.softmax(pred_logits_t)\n",
    "   pred_log = tf.log(pred_soft)\n",
    "   product = -1 * tar_soft_t * pred_log\n",
    "   return tf.reduce_mean(product)\n",
    "    \n",
    "def custom_loss(y_true, pred_logits, tar_soft_t, pred_logits_t):\n",
    "#    print(tf.losses.sparse_softmax_cross_entropy(labels = y_true, logits = pred_logits))\n",
    "#    print(l * cross_entropy2(tar_soft_t, pred_logits_t))\n",
    "   return loss_weight * tf.losses.sparse_softmax_cross_entropy(labels = y_true, logits = pred_logits) + (1-loss_weight) * cross_entropy2(tar_soft_t, pred_logits_t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getdata():\n",
    "    return np.random.randn(100,784)\n",
    "\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "tf.logging.set_verbosity(tf.logging.INFO)\n",
    "\n",
    "\n",
    "def student_model_fn(features, labels, mode):\n",
    "  sess=tf.InteractiveSession()\n",
    "  tf.train.start_queue_runners(sess)\n",
    "  data=features['x'].eval()\n",
    "  out=labels.eval()\n",
    "  sess.close()\n",
    "#   print(\"swapnil\",features['x'],labels)\n",
    "#   print(labels)\n",
    "#   tf.Session().run(features['x']\n",
    "#   inp=tf.reshape(features[\"x\"], [-1, 28, 28, 1])\n",
    "  \"\"\"Model function for CNN.\"\"\"\n",
    "  # Input Layer\n",
    "  # Reshape X to 4-D tensor: [batch_size, width, height, channels]\n",
    "  # MNIST images are 28x28 pixels, and have one color channel\n",
    "  input_layer = tf.reshape(features[\"x\"], [-1, 28, 28, 1])\n",
    "  # Convolutional Layer #1\n",
    "  # Computes 32 features using a 5x5 filter with ReLU activation.\n",
    "  # Padding is added to preserve width and height.\n",
    "  # Input Tensor Shape: [batch_size, 28, 28, 1]\n",
    "  # Output Tensor Shape: [batch_size, 28, 28, 32]\n",
    "#   data=tf.reshape(features[\"x\"], [100, 784])\n",
    "#   data=getdata()\n",
    "#   data=np.random.random((100,784)).astype(np.float32)\n",
    "#   out=np.random.random(100).astype(np.float32)\n",
    "#   print(data.shape,out.shape)\n",
    "  eval_teacher_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "      x={\"x\":data},\n",
    "      y=out,\n",
    "      num_epochs=1,\n",
    "      shuffle=False)\n",
    "\n",
    "  predictions=mnist_classifier.predict(input_fn=eval_teacher_fn)\n",
    "  teacher_pred=list(predictions)\n",
    "  teacher_soft=[ p['probabilities'] for p in teacher_pred]\n",
    "  temp=np.array(teacher_soft)\n",
    "  teacher_soft_t=tf.convert_to_tensor(temp)\n",
    "  print(teacher_soft_t)\n",
    "#   for i in predictions:\n",
    "#     print(i)\n",
    "#   output=list(predictions)\n",
    "\n",
    "\n",
    "  conv1 = tf.layers.conv2d(\n",
    "      inputs=input_layer,\n",
    "      filters=8,\n",
    "      kernel_size=[5, 5],\n",
    "      padding=\"same\",\n",
    "      activation=tf.nn.relu)\n",
    "\n",
    "  # Pooling Layer #1\n",
    "  # First max pooling layer with a 2x2 filter and stride of 2\n",
    "  # Input Tensor Shape: [batch_size, 28, 28, 32]\n",
    "  # Output Tensor Shape: [batch_size, 14, 14, 32]\n",
    "  pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2)\n",
    "\n",
    "  # Convolutional Layer #2\n",
    "  # Computes 64 features using a 5x5 filter.\n",
    "  # Padding is added to preserve width and height.\n",
    "  # Input Tensor Shape: [batch_size, 14, 14, 32]\n",
    "  # Output Tensor Shape: [batch_size, 14, 14, 64]\n",
    "  conv2 = tf.layers.conv2d(\n",
    "      inputs=pool1,\n",
    "      filters=8,\n",
    "      kernel_size=[5, 5],\n",
    "      padding=\"same\",\n",
    "      activation=tf.nn.relu)\n",
    "\n",
    "  # Pooling Layer #2\n",
    "  # Second max pooling layer with a 2x2 filter and stride of 2\n",
    "  # Input Tensor Shape: [batch_size, 14, 14, 64]\n",
    "  # Output Tensor Shape: [batch_size, 7, 7, 64]\n",
    "  pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2)\n",
    "\n",
    "  # Flatten tensor into a batch of vectors\n",
    "  # Input Tensor Shape: [batch_size, 7, 7, 64]\n",
    "  # Output Tensor Shape: [batch_size, 7 * 7 * 64]\n",
    "  pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 8])\n",
    "\n",
    "  # Dense Layer\n",
    "  # Densely connected layer with 1024 neurons\n",
    "  # Input Tensor Shape: [batch_size, 7 * 7 * 64]\n",
    "  # Output Tensor Shape: [batch_size, 1024]\n",
    "  dense = tf.layers.dense(inputs=pool2_flat, units=256, activation=tf.nn.relu)\n",
    "\n",
    "  # Add dropout operation; 0.6 probability that element will be kept\n",
    "  dropout = tf.layers.dropout(\n",
    "      inputs=dense, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN)\n",
    "\n",
    "  # Logits layer\n",
    "  # Input Tensor Shape: [batch_size, 1024]\n",
    "  # Output Tensor Shape: [batch_size, 10]\n",
    "  logits = tf.layers.dense(inputs=dropout, units=10)\n",
    "\n",
    "  predictions = {\n",
    "      # Generate predictions (for PREDICT and EVAL mode)\n",
    "      \"classes\": tf.argmax(input=logits, axis=1),\n",
    "      # Add `softmax_tensor` to the graph. It is used for PREDICT and by the\n",
    "      # `logging_hook`.\n",
    "      \"probabilities\": tf.nn.softmax(logits, name=\"softmax_tensor\")\n",
    "  }\n",
    "  if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "    return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)\n",
    "\n",
    "  # Calculate Loss (for both TRAIN and EVAL modes)\n",
    "  #   loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)\n",
    "  \n",
    "    \n",
    "#   student_pred=predictions['probabilities']\n",
    "#   sess1=tf.InteractiveSession()\n",
    "#   tf.train.start_queue_runners(sess1)\n",
    "#   student_soft=student_pred.eval()\n",
    "#   sess1.close()  \n",
    "  if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "    pred_logits_t=tf.nn.softmax(logits/temprature, name=\"softmax_tensor\")\n",
    "#     print('a',labels,'b', logits,'c', teacher_soft_t ,'d', pred_logits_t)\n",
    "    loss=custom_loss(labels, logits,teacher_soft_t , pred_logits_t)\n",
    "  else:\n",
    "    loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)\n",
    "  \n",
    "    \n",
    "  # Configure the Training Op (for TRAIN mode)\n",
    "  if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)\n",
    "    train_op = optimizer.minimize(\n",
    "        loss=loss,\n",
    "        global_step=tf.train.get_global_step())\n",
    "    return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)\n",
    "\n",
    "  # Add evaluation metrics (for EVAL mode)\n",
    "  eval_metric_ops = {\n",
    "      \"accuracy\": tf.metrics.accuracy(\n",
    "          labels=labels, predictions=predictions[\"classes\"]),\n",
    "  }\n",
    "  return tf.estimator.EstimatorSpec(\n",
    "      mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST-data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST-data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST-data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST-data/t10k-labels-idx1-ubyte.gz\n",
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f47334ff190>, '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': '/tmp/mnist_convnet_model_test1', '_save_summary_steps': 100}\n",
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f47334ff0d0>, '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': './mnist_convnet_student_10_0.5', '_save_summary_steps': 100}\n",
      "[[0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]] [7 3 4 ... 5 6 8] [[0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]] [7 2 1 ... 4 5 6]\n",
      "_THIS_ <type 'dict'>\n",
      "INFO:tensorflow:Restoring parameters from /tmp/mnist_convnet_model_test1/model.ckpt-20000\n",
      "Tensor(\"Const:0\", shape=(100, 10), dtype=float32)\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Restoring parameters from ./mnist_convnet_student_10_0.5/model.ckpt-1000\n",
      "INFO:tensorflow:Saving checkpoints for 1001 into ./mnist_convnet_student_10_0.5/model.ckpt.\n",
      "INFO:tensorflow:loss = 1.256669, step = 1001\n",
      "INFO:tensorflow:global_step/sec: 145.151\n",
      "INFO:tensorflow:loss = 1.2390691, step = 1101 (0.691 sec)\n",
      "INFO:tensorflow:global_step/sec: 149.363\n",
      "INFO:tensorflow:loss = 1.2498308, step = 1201 (0.670 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.433\n",
      "INFO:tensorflow:loss = 1.2554646, step = 1301 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.992\n",
      "INFO:tensorflow:loss = 1.2470624, step = 1401 (0.628 sec)\n",
      "INFO:tensorflow:global_step/sec: 142.761\n",
      "INFO:tensorflow:loss = 1.2506614, step = 1501 (0.699 sec)\n",
      "INFO:tensorflow:global_step/sec: 134.215\n",
      "INFO:tensorflow:loss = 1.2477077, step = 1601 (0.745 sec)\n",
      "INFO:tensorflow:global_step/sec: 150.413\n",
      "INFO:tensorflow:loss = 1.2530544, step = 1701 (0.665 sec)\n",
      "INFO:tensorflow:global_step/sec: 143.224\n",
      "INFO:tensorflow:loss = 1.2425301, step = 1801 (0.703 sec)\n",
      "INFO:tensorflow:global_step/sec: 145.957\n",
      "INFO:tensorflow:loss = 1.2415042, step = 1901 (0.681 sec)\n",
      "INFO:tensorflow:global_step/sec: 152.79\n",
      "INFO:tensorflow:loss = 1.2306874, step = 2001 (0.652 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.644\n",
      "INFO:tensorflow:loss = 1.2310292, step = 2101 (0.623 sec)\n",
      "INFO:tensorflow:global_step/sec: 156.461\n",
      "INFO:tensorflow:loss = 1.229805, step = 2201 (0.639 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.409\n",
      "INFO:tensorflow:loss = 1.2365018, step = 2301 (0.608 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.76\n",
      "INFO:tensorflow:loss = 1.2200845, step = 2401 (0.622 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.459\n",
      "INFO:tensorflow:loss = 1.2144636, step = 2501 (0.608 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.215\n",
      "INFO:tensorflow:loss = 1.2247934, step = 2601 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 151.574\n",
      "INFO:tensorflow:loss = 1.2201006, step = 2701 (0.660 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.07\n",
      "INFO:tensorflow:loss = 1.22324, step = 2801 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 151.379\n",
      "INFO:tensorflow:loss = 1.2037266, step = 2901 (0.661 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.602\n",
      "INFO:tensorflow:loss = 1.2039969, step = 3001 (0.608 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.874\n",
      "INFO:tensorflow:loss = 1.1734505, step = 3101 (0.626 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.036\n",
      "INFO:tensorflow:loss = 1.1758661, step = 3201 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.357\n",
      "INFO:tensorflow:loss = 1.1561085, step = 3301 (0.609 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.597\n",
      "INFO:tensorflow:loss = 1.1599016, step = 3401 (0.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.942\n",
      "INFO:tensorflow:loss = 1.1494657, step = 3501 (0.610 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.818\n",
      "INFO:tensorflow:loss = 1.141269, step = 3601 (0.619 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.234\n",
      "INFO:tensorflow:loss = 1.1739326, step = 3701 (0.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.437\n",
      "INFO:tensorflow:loss = 1.1401727, step = 3801 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 165.714\n",
      "INFO:tensorflow:loss = 1.0781933, step = 3901 (0.604 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.291\n",
      "INFO:tensorflow:loss = 1.1283482, step = 4001 (0.608 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.937\n",
      "INFO:tensorflow:loss = 1.0467358, step = 4101 (0.606 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.387\n",
      "INFO:tensorflow:loss = 1.0345374, step = 4201 (0.628 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.923\n",
      "INFO:tensorflow:loss = 1.0627869, step = 4301 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.873\n",
      "INFO:tensorflow:loss = 1.0111632, step = 4401 (0.622 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.029\n",
      "INFO:tensorflow:loss = 0.9089061, step = 4501 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.343\n",
      "INFO:tensorflow:loss = 0.9886848, step = 4601 (0.627 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.157\n",
      "INFO:tensorflow:loss = 0.94591105, step = 4701 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.229\n",
      "INFO:tensorflow:loss = 0.8803824, step = 4801 (0.620 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.967\n",
      "INFO:tensorflow:loss = 0.83437824, step = 4901 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 150.124\n",
      "INFO:tensorflow:loss = 0.7971532, step = 5001 (0.668 sec)\n",
      "INFO:tensorflow:global_step/sec: 149.857\n",
      "INFO:tensorflow:loss = 0.8362492, step = 5101 (0.667 sec)\n",
      "INFO:tensorflow:global_step/sec: 152.515\n",
      "INFO:tensorflow:loss = 0.8316386, step = 5201 (0.655 sec)\n",
      "INFO:tensorflow:global_step/sec: 137.038\n",
      "INFO:tensorflow:loss = 0.78967357, step = 5301 (0.728 sec)\n",
      "INFO:tensorflow:global_step/sec: 150.941\n",
      "INFO:tensorflow:loss = 0.79551834, step = 5401 (0.663 sec)\n",
      "INFO:tensorflow:global_step/sec: 145.839\n",
      "INFO:tensorflow:loss = 0.70374656, step = 5501 (0.687 sec)\n",
      "INFO:tensorflow:global_step/sec: 149.682\n",
      "INFO:tensorflow:loss = 0.72389674, step = 5601 (0.668 sec)\n",
      "INFO:tensorflow:global_step/sec: 151.522\n",
      "INFO:tensorflow:loss = 0.6154312, step = 5701 (0.658 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.376\n",
      "INFO:tensorflow:loss = 0.6432276, step = 5801 (0.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.53\n",
      "INFO:tensorflow:loss = 0.7550691, step = 5901 (0.632 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.27\n",
      "INFO:tensorflow:loss = 0.58095706, step = 6001 (0.627 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.413\n",
      "INFO:tensorflow:loss = 0.6107256, step = 6101 (0.633 sec)\n",
      "INFO:tensorflow:global_step/sec: 138.889\n",
      "INFO:tensorflow:loss = 0.54566044, step = 6201 (0.720 sec)\n",
      "INFO:tensorflow:global_step/sec: 154.909\n",
      "INFO:tensorflow:loss = 0.59627175, step = 6301 (0.644 sec)\n",
      "INFO:tensorflow:global_step/sec: 144.431\n",
      "INFO:tensorflow:loss = 0.5557511, step = 6401 (0.691 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.078\n",
      "INFO:tensorflow:loss = 0.55277914, step = 6501 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.77\n",
      "INFO:tensorflow:loss = 0.62058616, step = 6601 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.643\n",
      "INFO:tensorflow:loss = 0.5643281, step = 6701 (0.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.475\n",
      "INFO:tensorflow:loss = 0.58641016, step = 6801 (0.608 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.245\n",
      "INFO:tensorflow:loss = 0.52596486, step = 6901 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.638\n",
      "INFO:tensorflow:loss = 0.5648979, step = 7001 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.193\n",
      "INFO:tensorflow:loss = 0.49651268, step = 7101 (0.624 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 160.646\n",
      "INFO:tensorflow:loss = 0.53383994, step = 7201 (0.619 sec)\n",
      "INFO:tensorflow:global_step/sec: 165.171\n",
      "INFO:tensorflow:loss = 0.5698982, step = 7301 (0.605 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.777\n",
      "INFO:tensorflow:loss = 0.625221, step = 7401 (0.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.103\n",
      "INFO:tensorflow:loss = 0.56378466, step = 7501 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.65\n",
      "INFO:tensorflow:loss = 0.37583143, step = 7601 (0.619 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.032\n",
      "INFO:tensorflow:loss = 0.44402978, step = 7701 (0.610 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.714\n",
      "INFO:tensorflow:loss = 0.42619115, step = 7801 (0.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.631\n",
      "INFO:tensorflow:loss = 0.4181462, step = 7901 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.354\n",
      "INFO:tensorflow:loss = 0.44282153, step = 8001 (0.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.214\n",
      "INFO:tensorflow:loss = 0.5182487, step = 8101 (0.627 sec)\n",
      "INFO:tensorflow:global_step/sec: 149.201\n",
      "INFO:tensorflow:loss = 0.4113545, step = 8201 (0.669 sec)\n",
      "INFO:tensorflow:global_step/sec: 139.27\n",
      "INFO:tensorflow:loss = 0.49873573, step = 8301 (0.717 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.526\n",
      "INFO:tensorflow:loss = 0.41721213, step = 8401 (0.626 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.517\n",
      "INFO:tensorflow:loss = 0.46350083, step = 8501 (0.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.602\n",
      "INFO:tensorflow:loss = 0.47756535, step = 8601 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.037\n",
      "INFO:tensorflow:loss = 0.37117088, step = 8701 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.188\n",
      "INFO:tensorflow:loss = 0.4652388, step = 8801 (0.609 sec)\n",
      "INFO:tensorflow:global_step/sec: 157.79\n",
      "INFO:tensorflow:loss = 0.36625353, step = 8901 (0.634 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.848\n",
      "INFO:tensorflow:loss = 0.49655566, step = 9001 (0.610 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.577\n",
      "INFO:tensorflow:loss = 0.37649286, step = 9101 (0.631 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.696\n",
      "INFO:tensorflow:loss = 0.4138747, step = 9201 (0.622 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.573\n",
      "INFO:tensorflow:loss = 0.40961587, step = 9301 (0.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.405\n",
      "INFO:tensorflow:loss = 0.3327838, step = 9401 (0.616 sec)\n",
      "INFO:tensorflow:global_step/sec: 157.26\n",
      "INFO:tensorflow:loss = 0.40790713, step = 9501 (0.636 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.49\n",
      "INFO:tensorflow:loss = 0.43020278, step = 9601 (0.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.679\n",
      "INFO:tensorflow:loss = 0.38565066, step = 9701 (0.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.048\n",
      "INFO:tensorflow:loss = 0.5039674, step = 9801 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 156.316\n",
      "INFO:tensorflow:loss = 0.51203823, step = 9901 (0.641 sec)\n",
      "INFO:tensorflow:global_step/sec: 157.045\n",
      "INFO:tensorflow:loss = 0.3632871, step = 10001 (0.636 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.286\n",
      "INFO:tensorflow:loss = 0.38355786, step = 10101 (0.620 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.693\n",
      "INFO:tensorflow:loss = 0.3411672, step = 10201 (0.622 sec)\n",
      "INFO:tensorflow:global_step/sec: 156.969\n",
      "INFO:tensorflow:loss = 0.49883056, step = 10301 (0.637 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.807\n",
      "INFO:tensorflow:loss = 0.37371737, step = 10401 (0.625 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.131\n",
      "INFO:tensorflow:loss = 0.393039, step = 10501 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.092\n",
      "INFO:tensorflow:loss = 0.4100281, step = 10601 (0.609 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.449\n",
      "INFO:tensorflow:loss = 0.37898448, step = 10701 (0.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.281\n",
      "INFO:tensorflow:loss = 0.3850323, step = 10801 (0.624 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.608\n",
      "INFO:tensorflow:loss = 0.3734457, step = 10901 (0.618 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.617\n",
      "INFO:tensorflow:loss = 0.38915077, step = 11001 (0.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.857\n",
      "INFO:tensorflow:loss = 0.36604616, step = 11101 (0.610 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.651\n",
      "INFO:tensorflow:loss = 0.3796252, step = 11201 (0.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.992\n",
      "INFO:tensorflow:loss = 0.2765839, step = 11301 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.205\n",
      "INFO:tensorflow:loss = 0.345964, step = 11401 (0.627 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.302\n",
      "INFO:tensorflow:loss = 0.39286804, step = 11501 (0.626 sec)\n",
      "INFO:tensorflow:global_step/sec: 154.654\n",
      "INFO:tensorflow:loss = 0.30543423, step = 11601 (0.645 sec)\n",
      "INFO:tensorflow:global_step/sec: 153.905\n",
      "INFO:tensorflow:loss = 0.37126195, step = 11701 (0.650 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.145\n",
      "INFO:tensorflow:loss = 0.42369318, step = 11801 (0.609 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.977\n",
      "INFO:tensorflow:loss = 0.32215965, step = 11901 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.544\n",
      "INFO:tensorflow:loss = 0.33218145, step = 12001 (0.622 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.932\n",
      "INFO:tensorflow:loss = 0.33065853, step = 12101 (0.621 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.357\n",
      "INFO:tensorflow:loss = 0.35572106, step = 12201 (0.620 sec)\n",
      "INFO:tensorflow:global_step/sec: 156.423\n",
      "INFO:tensorflow:loss = 0.35218498, step = 12301 (0.639 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.924\n",
      "INFO:tensorflow:loss = 0.2699247, step = 12401 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.115\n",
      "INFO:tensorflow:loss = 0.3966166, step = 12501 (0.619 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.495\n",
      "INFO:tensorflow:loss = 0.3750869, step = 12601 (0.626 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.046\n",
      "INFO:tensorflow:loss = 0.28486058, step = 12701 (0.625 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.21\n",
      "INFO:tensorflow:loss = 0.31418934, step = 12801 (0.628 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.608\n",
      "INFO:tensorflow:loss = 0.31499118, step = 12901 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.249\n",
      "INFO:tensorflow:loss = 0.31803122, step = 13001 (0.620 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.585\n",
      "INFO:tensorflow:loss = 0.35135636, step = 13101 (0.623 sec)\n",
      "INFO:tensorflow:global_step/sec: 154.904\n",
      "INFO:tensorflow:loss = 0.37459704, step = 13201 (0.645 sec)\n",
      "INFO:tensorflow:global_step/sec: 157.999\n",
      "INFO:tensorflow:loss = 0.2829515, step = 13301 (0.633 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.66\n",
      "INFO:tensorflow:loss = 0.28374687, step = 13401 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 149.365\n",
      "INFO:tensorflow:loss = 0.25923604, step = 13501 (0.671 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.152\n",
      "INFO:tensorflow:loss = 0.40797475, step = 13601 (0.623 sec)\n",
      "INFO:tensorflow:global_step/sec: 157.889\n",
      "INFO:tensorflow:loss = 0.25720263, step = 13701 (0.633 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.175\n",
      "INFO:tensorflow:loss = 0.3567122, step = 13801 (0.620 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.929\n",
      "INFO:tensorflow:loss = 0.29433843, step = 13901 (0.626 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.408\n",
      "INFO:tensorflow:loss = 0.29807898, step = 14001 (0.619 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.181\n",
      "INFO:tensorflow:loss = 0.36206433, step = 14101 (0.620 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.1\n",
      "INFO:tensorflow:loss = 0.2849944, step = 14201 (0.629 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.552\n",
      "INFO:tensorflow:loss = 0.37920898, step = 14301 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.197\n",
      "INFO:tensorflow:loss = 0.28267092, step = 14401 (0.609 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.858\n",
      "INFO:tensorflow:loss = 0.26536506, step = 14501 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.559\n",
      "INFO:tensorflow:loss = 0.37283295, step = 14601 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.144\n",
      "INFO:tensorflow:loss = 0.33073708, step = 14701 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.2\n",
      "INFO:tensorflow:loss = 0.35969096, step = 14801 (0.620 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.565\n",
      "INFO:tensorflow:loss = 0.39224112, step = 14901 (0.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.623\n",
      "INFO:tensorflow:loss = 0.3953805, step = 15001 (0.607 sec)\n",
      "INFO:tensorflow:global_step/sec: 156.958\n",
      "INFO:tensorflow:loss = 0.25106895, step = 15101 (0.638 sec)\n",
      "INFO:tensorflow:global_step/sec: 152.715\n",
      "INFO:tensorflow:loss = 0.34201115, step = 15201 (0.656 sec)\n",
      "INFO:tensorflow:global_step/sec: 157.907\n",
      "INFO:tensorflow:loss = 0.28753135, step = 15301 (0.632 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 160.838\n",
      "INFO:tensorflow:loss = 0.33610064, step = 15401 (0.622 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.247\n",
      "INFO:tensorflow:loss = 0.25754973, step = 15501 (0.629 sec)\n",
      "INFO:tensorflow:global_step/sec: 150.534\n",
      "INFO:tensorflow:loss = 0.31534183, step = 15601 (0.663 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.788\n",
      "INFO:tensorflow:loss = 0.32577515, step = 15701 (0.630 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.278\n",
      "INFO:tensorflow:loss = 0.3666944, step = 15801 (0.620 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.476\n",
      "INFO:tensorflow:loss = 0.3353055, step = 15901 (0.627 sec)\n",
      "INFO:tensorflow:global_step/sec: 155.685\n",
      "INFO:tensorflow:loss = 0.26557043, step = 16001 (0.644 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.82\n",
      "INFO:tensorflow:loss = 0.37922457, step = 16101 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 165.017\n",
      "INFO:tensorflow:loss = 0.2517677, step = 16201 (0.606 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.301\n",
      "INFO:tensorflow:loss = 0.32865846, step = 16301 (0.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.086\n",
      "INFO:tensorflow:loss = 0.31878966, step = 16401 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.963\n",
      "INFO:tensorflow:loss = 0.26444608, step = 16501 (0.610 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.04\n",
      "INFO:tensorflow:loss = 0.31836137, step = 16601 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.147\n",
      "INFO:tensorflow:loss = 0.24284136, step = 16701 (0.618 sec)\n",
      "INFO:tensorflow:global_step/sec: 156.518\n",
      "INFO:tensorflow:loss = 0.2877335, step = 16801 (0.638 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.231\n",
      "INFO:tensorflow:loss = 0.33397797, step = 16901 (0.628 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.802\n",
      "INFO:tensorflow:loss = 0.3165435, step = 17001 (0.618 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.725\n",
      "INFO:tensorflow:loss = 0.32305342, step = 17101 (0.618 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.483\n",
      "INFO:tensorflow:loss = 0.374399, step = 17201 (0.628 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.167\n",
      "INFO:tensorflow:loss = 0.2912918, step = 17301 (0.609 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.312\n",
      "INFO:tensorflow:loss = 0.27083078, step = 17401 (0.628 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.898\n",
      "INFO:tensorflow:loss = 0.35351217, step = 17501 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.262\n",
      "INFO:tensorflow:loss = 0.25950563, step = 17601 (0.624 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.868\n",
      "INFO:tensorflow:loss = 0.2450084, step = 17701 (0.618 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.82\n",
      "INFO:tensorflow:loss = 0.3359076, step = 17801 (0.607 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.62\n",
      "INFO:tensorflow:loss = 0.23453346, step = 17901 (0.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.14\n",
      "INFO:tensorflow:loss = 0.39379025, step = 18001 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.748\n",
      "INFO:tensorflow:loss = 0.29251486, step = 18101 (0.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.715\n",
      "INFO:tensorflow:loss = 0.21102586, step = 18201 (0.622 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.633\n",
      "INFO:tensorflow:loss = 0.31338444, step = 18301 (0.619 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.614\n",
      "INFO:tensorflow:loss = 0.23557536, step = 18401 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.116\n",
      "INFO:tensorflow:loss = 0.22796334, step = 18501 (0.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.62\n",
      "INFO:tensorflow:loss = 0.25168204, step = 18601 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.157\n",
      "INFO:tensorflow:loss = 0.25430328, step = 18701 (0.610 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.408\n",
      "INFO:tensorflow:loss = 0.20659779, step = 18801 (0.626 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.422\n",
      "INFO:tensorflow:loss = 0.26457936, step = 18901 (0.631 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.482\n",
      "INFO:tensorflow:loss = 0.30566058, step = 19001 (0.619 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.387\n",
      "INFO:tensorflow:loss = 0.31217176, step = 19101 (0.608 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.551\n",
      "INFO:tensorflow:loss = 0.33642983, step = 19201 (0.608 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.004\n",
      "INFO:tensorflow:loss = 0.29218444, step = 19301 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.917\n",
      "INFO:tensorflow:loss = 0.290297, step = 19401 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.619\n",
      "INFO:tensorflow:loss = 0.2889679, step = 19501 (0.622 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.335\n",
      "INFO:tensorflow:loss = 0.27272326, step = 19601 (0.616 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.086\n",
      "INFO:tensorflow:loss = 0.21692073, step = 19701 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.847\n",
      "INFO:tensorflow:loss = 0.20460191, step = 19801 (0.623 sec)\n",
      "INFO:tensorflow:global_step/sec: 151.879\n",
      "INFO:tensorflow:loss = 0.23882645, step = 19901 (0.657 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.807\n",
      "INFO:tensorflow:loss = 0.335661, step = 20001 (0.610 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.091\n",
      "INFO:tensorflow:loss = 0.25107664, step = 20101 (0.621 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.532\n",
      "INFO:tensorflow:loss = 0.2740075, step = 20201 (0.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.103\n",
      "INFO:tensorflow:loss = 0.26983893, step = 20301 (0.609 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.367\n",
      "INFO:tensorflow:loss = 0.27943617, step = 20401 (0.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.796\n",
      "INFO:tensorflow:loss = 0.2834943, step = 20501 (0.618 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.798\n",
      "INFO:tensorflow:loss = 0.23944378, step = 20601 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.433\n",
      "INFO:tensorflow:loss = 0.22899461, step = 20701 (0.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.296\n",
      "INFO:tensorflow:loss = 0.26289588, step = 20801 (0.620 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.176\n",
      "INFO:tensorflow:loss = 0.31973204, step = 20901 (0.609 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 21000 into ./mnist_convnet_student_10_0.5/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.24388236.\n",
      "ERROR:tensorflow:Exception in QueueRunner: Attempted to use a closed Session.\n",
      "_THIS_ <type 'dict'>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-82:\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/sanjay/anaconda2/lib/python2.7/threading.py\", line 801, in __bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/home/sanjay/anaconda2/lib/python2.7/threading.py\", line 754, in run\n",
      "    self.__target(*self.__args, **self.__kwargs)\n",
      "  File \"/home/sanjay/anaconda2/lib/python2.7/site-packages/tensorflow/python/estimator/inputs/queues/feeding_queue_runner.py\", line 94, in _run\n",
      "    sess.run(enqueue_op, feed_dict=feed_dict)\n",
      "  File \"/home/sanjay/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py\", line 895, in run\n",
      "    run_metadata_ptr)\n",
      "  File \"/home/sanjay/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py\", line 1053, in _run\n",
      "    raise RuntimeError('Attempted to use a closed Session.')\n",
      "RuntimeError: Attempted to use a closed Session.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /tmp/mnist_convnet_model_test1/model.ckpt-20000\n",
      "Tensor(\"Const:0\", shape=(128, 10), dtype=float32)\n",
      "INFO:tensorflow:Starting evaluation at 2018-05-06-11:00:58\n",
      "INFO:tensorflow:Restoring parameters from ./mnist_convnet_student_10_0.5/model.ckpt-21000\n",
      "INFO:tensorflow:Finished evaluation at 2018-05-06-11:00:58\n",
      "INFO:tensorflow:Saving dict for global step 21000: accuracy = 0.93677205, global_step = 21000, loss = 0.21718322\n",
      "{'loss': 0.21718322, 'global_step': 21000, 'accuracy': 0.93677205}\n"
     ]
    }
   ],
   "source": [
    "# Load training and eval data\n",
    "mnist = tf.contrib.learn.datasets.load_dataset(\"mnist\")\n",
    "train_data = mnist.train.images  # Returns np.array\n",
    "train_labels = np.asarray(mnist.train.labels, dtype=np.int32)\n",
    "eval_data = mnist.test.images  # Returns np.array\n",
    "eval_labels = np.asarray(mnist.test.labels, dtype=np.int32)\n",
    "iters=0\n",
    "\n",
    "mnist_classifier = tf.estimator.Estimator(\n",
    "  model_fn=cnn_model_fn, model_dir=\"./models/mnist_convnet_model_test1\")\n",
    "\n",
    "\n",
    "# Create the Estimator\n",
    "student_classifier = tf.estimator.Estimator(\n",
    "  model_fn=student_model_fn, model_dir=\"./models/mnist_convnet_student_10_0.5\")\n",
    "\n",
    "\n",
    "# Set up logging for predictions\n",
    "# Log the values in the \"Softmax\" tensor with label \"probabilities\"\n",
    "# tensors_to_log = {\"probabilities\": \"softmax_tensor\"}\n",
    "# logging_hook = tf.train.LoggingTensorHook(\n",
    "#   tensors=tensors_to_log, every_n_iter=50)\n",
    "\n",
    "temprature=10\n",
    "loss_weight=0.5\n",
    "\n",
    "# Train the model\n",
    "train_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "  x={\"x\": train_data},\n",
    "  y=train_labels,\n",
    "  batch_size=100,\n",
    "  num_epochs=None,\n",
    "  shuffle=True)\n",
    "print(train_data,train_labels,eval_data,eval_labels)\n",
    "\n",
    "student_classifier.train(\n",
    "  input_fn=train_input_fn,\n",
    "  steps=20000,\n",
    "  hooks=None)\n",
    "# print(\"eva\",type(eval_data))\n",
    "\n",
    "# Evaluate the model and print results\n",
    "eval_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "  x={\"x\": eval_data},\n",
    "  y=eval_labels,\n",
    "  num_epochs=1,\n",
    "  shuffle=False)\n",
    "eval_result=student_classifier.evaluate(input_fn=eval_input_fn)\n",
    "# output=list(predictions)\n",
    "print(eval_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f46a4778950>, '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': '/tmp/mnist_convnet_model_test1', '_save_summary_steps': 100}\n",
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f4720739cd0>, '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': './mnist_convnet_student_10_0.75', '_save_summary_steps': 100}\n",
      "[[0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]] [7 3 4 ... 5 6 8] [[0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]] [7 2 1 ... 4 5 6]\n",
      "_THIS_ <type 'dict'>\n",
      "INFO:tensorflow:Restoring parameters from /tmp/mnist_convnet_model_test1/model.ckpt-20000\n",
      "Tensor(\"Const:0\", shape=(100, 10), dtype=float32)\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into ./mnist_convnet_student_10_0.75/model.ckpt.\n",
      "INFO:tensorflow:loss = 1.8317287, step = 1\n",
      "INFO:tensorflow:global_step/sec: 121.61\n",
      "INFO:tensorflow:loss = 1.7985559, step = 101 (0.825 sec)\n",
      "INFO:tensorflow:global_step/sec: 152.15\n",
      "INFO:tensorflow:loss = 1.7935694, step = 201 (0.657 sec)\n",
      "INFO:tensorflow:global_step/sec: 151.11\n",
      "INFO:tensorflow:loss = 1.7808467, step = 301 (0.662 sec)\n",
      "INFO:tensorflow:global_step/sec: 139.42\n",
      "INFO:tensorflow:loss = 1.7622614, step = 401 (0.717 sec)\n",
      "INFO:tensorflow:global_step/sec: 149.33\n",
      "INFO:tensorflow:loss = 1.7179695, step = 501 (0.670 sec)\n",
      "INFO:tensorflow:global_step/sec: 132.064\n",
      "INFO:tensorflow:loss = 1.6953411, step = 601 (0.757 sec)\n",
      "INFO:tensorflow:global_step/sec: 150.159\n",
      "INFO:tensorflow:loss = 1.7261053, step = 701 (0.664 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.994\n",
      "INFO:tensorflow:loss = 1.6844974, step = 801 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.271\n",
      "INFO:tensorflow:loss = 1.6779234, step = 901 (0.628 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.581\n",
      "INFO:tensorflow:loss = 1.6470399, step = 1001 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.319\n",
      "INFO:tensorflow:loss = 1.606811, step = 1101 (0.616 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.324\n",
      "INFO:tensorflow:loss = 1.5142317, step = 1201 (0.609 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.804\n",
      "INFO:tensorflow:loss = 1.5469601, step = 1301 (0.622 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.239\n",
      "INFO:tensorflow:loss = 1.4808378, step = 1401 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.967\n",
      "INFO:tensorflow:loss = 1.3487233, step = 1501 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.012\n",
      "INFO:tensorflow:loss = 1.3270409, step = 1601 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.821\n",
      "INFO:tensorflow:loss = 1.3826653, step = 1701 (0.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 156.29\n",
      "INFO:tensorflow:loss = 1.3139018, step = 1801 (0.640 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.358\n",
      "INFO:tensorflow:loss = 1.2210026, step = 1901 (0.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.558\n",
      "INFO:tensorflow:loss = 1.1096628, step = 2001 (0.623 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.141\n",
      "INFO:tensorflow:loss = 1.077775, step = 2101 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.514\n",
      "INFO:tensorflow:loss = 1.0523957, step = 2201 (0.624 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.806\n",
      "INFO:tensorflow:loss = 0.8091905, step = 2301 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.996\n",
      "INFO:tensorflow:loss = 0.93348193, step = 2401 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.354\n",
      "INFO:tensorflow:loss = 0.96027136, step = 2501 (0.609 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.392\n",
      "INFO:tensorflow:loss = 0.9078164, step = 2601 (0.632 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.447\n",
      "INFO:tensorflow:loss = 0.6241035, step = 2701 (0.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.402\n",
      "INFO:tensorflow:loss = 0.8726523, step = 2801 (0.620 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.368\n",
      "INFO:tensorflow:loss = 0.8015868, step = 2901 (0.616 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.568\n",
      "INFO:tensorflow:loss = 0.6673333, step = 3001 (0.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.994\n",
      "INFO:tensorflow:loss = 0.7384268, step = 3101 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.2\n",
      "INFO:tensorflow:loss = 0.65662557, step = 3201 (0.609 sec)\n",
      "INFO:tensorflow:global_step/sec: 157.178\n",
      "INFO:tensorflow:loss = 0.7324132, step = 3301 (0.636 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.805\n",
      "INFO:tensorflow:loss = 0.7467292, step = 3401 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.493\n",
      "INFO:tensorflow:loss = 0.72704804, step = 3501 (0.619 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.011\n",
      "INFO:tensorflow:loss = 0.60588044, step = 3601 (0.621 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.015\n",
      "INFO:tensorflow:loss = 0.66948354, step = 3701 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.954\n",
      "INFO:tensorflow:loss = 0.6819389, step = 3801 (0.621 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.608\n",
      "INFO:tensorflow:loss = 0.45769218, step = 3901 (0.631 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.525\n",
      "INFO:tensorflow:loss = 0.6403791, step = 4001 (0.607 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.46\n",
      "INFO:tensorflow:loss = 0.47729144, step = 4101 (0.624 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.136\n",
      "INFO:tensorflow:loss = 0.556227, step = 4201 (0.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.248\n",
      "INFO:tensorflow:loss = 0.47348815, step = 4301 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.38\n",
      "INFO:tensorflow:loss = 0.48737693, step = 4401 (0.619 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.898\n",
      "INFO:tensorflow:loss = 0.44369802, step = 4501 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.814\n",
      "INFO:tensorflow:loss = 0.60766035, step = 4601 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 165.356\n",
      "INFO:tensorflow:loss = 0.52551025, step = 4701 (0.605 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.149\n",
      "INFO:tensorflow:loss = 0.44767773, step = 4801 (0.624 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.708\n",
      "INFO:tensorflow:loss = 0.48195216, step = 4901 (0.627 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.67\n",
      "INFO:tensorflow:loss = 0.4313252, step = 5001 (0.607 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.176\n",
      "INFO:tensorflow:loss = 0.44006887, step = 5101 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.402\n",
      "INFO:tensorflow:loss = 0.37411726, step = 5201 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.465\n",
      "INFO:tensorflow:loss = 0.54664856, step = 5301 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.316\n",
      "INFO:tensorflow:loss = 0.30697864, step = 5401 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.644\n",
      "INFO:tensorflow:loss = 0.39873567, step = 5501 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.405\n",
      "INFO:tensorflow:loss = 0.3257864, step = 5601 (0.620 sec)\n",
      "INFO:tensorflow:global_step/sec: 165.454\n",
      "INFO:tensorflow:loss = 0.44924322, step = 5701 (0.604 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.17\n",
      "INFO:tensorflow:loss = 0.32555795, step = 5801 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.418\n",
      "INFO:tensorflow:loss = 0.364525, step = 5901 (0.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.29\n",
      "INFO:tensorflow:loss = 0.4779319, step = 6001 (0.608 sec)\n",
      "INFO:tensorflow:global_step/sec: 156.643\n",
      "INFO:tensorflow:loss = 0.5166335, step = 6101 (0.639 sec)\n",
      "INFO:tensorflow:global_step/sec: 163\n",
      "INFO:tensorflow:loss = 0.42212483, step = 6201 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.705\n",
      "INFO:tensorflow:loss = 0.4055148, step = 6301 (0.607 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.677\n",
      "INFO:tensorflow:loss = 0.41738123, step = 6401 (0.622 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 160.9\n",
      "INFO:tensorflow:loss = 0.34445223, step = 6501 (0.621 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.082\n",
      "INFO:tensorflow:loss = 0.38167787, step = 6601 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 165.268\n",
      "INFO:tensorflow:loss = 0.38531262, step = 6701 (0.605 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.999\n",
      "INFO:tensorflow:loss = 0.3637136, step = 6801 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.062\n",
      "INFO:tensorflow:loss = 0.46060556, step = 6901 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 157.993\n",
      "INFO:tensorflow:loss = 0.37778184, step = 7001 (0.633 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.891\n",
      "INFO:tensorflow:loss = 0.3413535, step = 7101 (0.625 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.879\n",
      "INFO:tensorflow:loss = 0.34768385, step = 7201 (0.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.258\n",
      "INFO:tensorflow:loss = 0.30511826, step = 7301 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.726\n",
      "INFO:tensorflow:loss = 0.38251308, step = 7401 (0.629 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.587\n",
      "INFO:tensorflow:loss = 0.3573645, step = 7501 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.13\n",
      "INFO:tensorflow:loss = 0.24525596, step = 7601 (0.625 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.67\n",
      "INFO:tensorflow:loss = 0.32534626, step = 7701 (0.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.404\n",
      "INFO:tensorflow:loss = 0.27361882, step = 7801 (0.623 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.687\n",
      "INFO:tensorflow:loss = 0.3783896, step = 7901 (0.626 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.429\n",
      "INFO:tensorflow:loss = 0.34550482, step = 8001 (0.623 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.157\n",
      "INFO:tensorflow:loss = 0.3440552, step = 8101 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 156.215\n",
      "INFO:tensorflow:loss = 0.35205752, step = 8201 (0.640 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.137\n",
      "INFO:tensorflow:loss = 0.34439227, step = 8301 (0.621 sec)\n",
      "INFO:tensorflow:global_step/sec: 160\n",
      "INFO:tensorflow:loss = 0.42701423, step = 8401 (0.625 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.315\n",
      "INFO:tensorflow:loss = 0.32406196, step = 8501 (0.630 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.432\n",
      "INFO:tensorflow:loss = 0.4604652, step = 8601 (0.609 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.924\n",
      "INFO:tensorflow:loss = 0.2362198, step = 8701 (0.622 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.524\n",
      "INFO:tensorflow:loss = 0.22299363, step = 8801 (0.622 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.252\n",
      "INFO:tensorflow:loss = 0.23151737, step = 8901 (0.616 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.498\n",
      "INFO:tensorflow:loss = 0.31834656, step = 9001 (0.619 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.55\n",
      "INFO:tensorflow:loss = 0.2418308, step = 9101 (0.608 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.168\n",
      "INFO:tensorflow:loss = 0.27447596, step = 9201 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.718\n",
      "INFO:tensorflow:loss = 0.37872693, step = 9301 (0.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.706\n",
      "INFO:tensorflow:loss = 0.31555858, step = 9401 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.814\n",
      "INFO:tensorflow:loss = 0.26010638, step = 9501 (0.630 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.548\n",
      "INFO:tensorflow:loss = 0.22284448, step = 9601 (0.623 sec)\n",
      "INFO:tensorflow:global_step/sec: 154.619\n",
      "INFO:tensorflow:loss = 0.37548098, step = 9701 (0.647 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.357\n",
      "INFO:tensorflow:loss = 0.2498078, step = 9801 (0.631 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.941\n",
      "INFO:tensorflow:loss = 0.31337646, step = 9901 (0.618 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.609\n",
      "INFO:tensorflow:loss = 0.23251423, step = 10001 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.799\n",
      "INFO:tensorflow:loss = 0.32185334, step = 10101 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.623\n",
      "INFO:tensorflow:loss = 0.33217162, step = 10201 (0.618 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.849\n",
      "INFO:tensorflow:loss = 0.38487566, step = 10301 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.092\n",
      "INFO:tensorflow:loss = 0.3367776, step = 10401 (0.633 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.095\n",
      "INFO:tensorflow:loss = 0.28821295, step = 10501 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.723\n",
      "INFO:tensorflow:loss = 0.32652196, step = 10601 (0.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.789\n",
      "INFO:tensorflow:loss = 0.28084788, step = 10701 (0.618 sec)\n",
      "INFO:tensorflow:global_step/sec: 154.262\n",
      "INFO:tensorflow:loss = 0.35655552, step = 10801 (0.648 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.1\n",
      "INFO:tensorflow:loss = 0.32979116, step = 10901 (0.621 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.959\n",
      "INFO:tensorflow:loss = 0.2181124, step = 11001 (0.623 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.492\n",
      "INFO:tensorflow:loss = 0.42705396, step = 11101 (0.618 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.113\n",
      "INFO:tensorflow:loss = 0.32555407, step = 11201 (0.621 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.586\n",
      "INFO:tensorflow:loss = 0.24108252, step = 11301 (0.607 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.662\n",
      "INFO:tensorflow:loss = 0.30449802, step = 11401 (0.619 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.532\n",
      "INFO:tensorflow:loss = 0.21315584, step = 11501 (0.627 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.964\n",
      "INFO:tensorflow:loss = 0.37962055, step = 11601 (0.621 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.543\n",
      "INFO:tensorflow:loss = 0.22800288, step = 11701 (0.619 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.911\n",
      "INFO:tensorflow:loss = 0.32048905, step = 11801 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.03\n",
      "INFO:tensorflow:loss = 0.24463642, step = 11901 (0.625 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.002\n",
      "INFO:tensorflow:loss = 0.21897388, step = 12001 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.623\n",
      "INFO:tensorflow:loss = 0.25743213, step = 12101 (0.623 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.394\n",
      "INFO:tensorflow:loss = 0.16765395, step = 12201 (0.616 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.877\n",
      "INFO:tensorflow:loss = 0.2756594, step = 12301 (0.622 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.613\n",
      "INFO:tensorflow:loss = 0.20342322, step = 12401 (0.619 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.419\n",
      "INFO:tensorflow:loss = 0.27501708, step = 12501 (0.616 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.856\n",
      "INFO:tensorflow:loss = 0.17921548, step = 12601 (0.618 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.706\n",
      "INFO:tensorflow:loss = 0.3113202, step = 12701 (0.622 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.726\n",
      "INFO:tensorflow:loss = 0.29392543, step = 12801 (0.607 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.558\n",
      "INFO:tensorflow:loss = 0.29256707, step = 12901 (0.608 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.72\n",
      "INFO:tensorflow:loss = 0.321392, step = 13001 (0.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.604\n",
      "INFO:tensorflow:loss = 0.31286448, step = 13101 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.271\n",
      "INFO:tensorflow:loss = 0.38443947, step = 13201 (0.609 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.347\n",
      "INFO:tensorflow:loss = 0.19897097, step = 13301 (0.620 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.63\n",
      "INFO:tensorflow:loss = 0.21761656, step = 13401 (0.631 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.998\n",
      "INFO:tensorflow:loss = 0.29781526, step = 13501 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 157.881\n",
      "INFO:tensorflow:loss = 0.3165941, step = 13601 (0.634 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.982\n",
      "INFO:tensorflow:loss = 0.28044188, step = 13701 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.497\n",
      "INFO:tensorflow:loss = 0.26600534, step = 13801 (0.631 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.031\n",
      "INFO:tensorflow:loss = 0.32444945, step = 13901 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.681\n",
      "INFO:tensorflow:loss = 0.25875133, step = 14001 (0.622 sec)\n",
      "INFO:tensorflow:global_step/sec: 165.474\n",
      "INFO:tensorflow:loss = 0.30273432, step = 14101 (0.604 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.717\n",
      "INFO:tensorflow:loss = 0.24003302, step = 14201 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.623\n",
      "INFO:tensorflow:loss = 0.1961348, step = 14301 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.182\n",
      "INFO:tensorflow:loss = 0.34680107, step = 14401 (0.609 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.709\n",
      "INFO:tensorflow:loss = 0.3561062, step = 14501 (0.619 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.977\n",
      "INFO:tensorflow:loss = 0.2786942, step = 14601 (0.628 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 160.663\n",
      "INFO:tensorflow:loss = 0.22872373, step = 14701 (0.622 sec)\n",
      "INFO:tensorflow:global_step/sec: 164\n",
      "INFO:tensorflow:loss = 0.25569543, step = 14801 (0.610 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.752\n",
      "INFO:tensorflow:loss = 0.3066243, step = 14901 (0.626 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.8\n",
      "INFO:tensorflow:loss = 0.2129665, step = 15001 (0.610 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.432\n",
      "INFO:tensorflow:loss = 0.20290525, step = 15101 (0.616 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.208\n",
      "INFO:tensorflow:loss = 0.31257102, step = 15201 (0.613 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.742\n",
      "INFO:tensorflow:loss = 0.28284156, step = 15301 (0.618 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.596\n",
      "INFO:tensorflow:loss = 0.26390344, step = 15401 (0.608 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.148\n",
      "INFO:tensorflow:loss = 0.24010615, step = 15501 (0.624 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.533\n",
      "INFO:tensorflow:loss = 0.27849808, step = 15601 (0.608 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.824\n",
      "INFO:tensorflow:loss = 0.20507227, step = 15701 (0.619 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.964\n",
      "INFO:tensorflow:loss = 0.35499918, step = 15801 (0.620 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.768\n",
      "INFO:tensorflow:loss = 0.21655506, step = 15901 (0.622 sec)\n",
      "INFO:tensorflow:global_step/sec: 157.08\n",
      "INFO:tensorflow:loss = 0.25396648, step = 16001 (0.637 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.943\n",
      "INFO:tensorflow:loss = 0.33471963, step = 16101 (0.611 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.969\n",
      "INFO:tensorflow:loss = 0.23255183, step = 16201 (0.616 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.116\n",
      "INFO:tensorflow:loss = 0.13808987, step = 16301 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.046\n",
      "INFO:tensorflow:loss = 0.21619016, step = 16401 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.533\n",
      "INFO:tensorflow:loss = 0.22907746, step = 16501 (0.607 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.224\n",
      "INFO:tensorflow:loss = 0.22134039, step = 16601 (0.628 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.641\n",
      "INFO:tensorflow:loss = 0.2560616, step = 16701 (0.607 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.931\n",
      "INFO:tensorflow:loss = 0.25101143, step = 16801 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.129\n",
      "INFO:tensorflow:loss = 0.3353793, step = 16901 (0.621 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.744\n",
      "INFO:tensorflow:loss = 0.29647094, step = 17001 (0.618 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.286\n",
      "INFO:tensorflow:loss = 0.26361045, step = 17101 (0.624 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.089\n",
      "INFO:tensorflow:loss = 0.20988977, step = 17201 (0.633 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.972\n",
      "INFO:tensorflow:loss = 0.21563974, step = 17301 (0.606 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.994\n",
      "INFO:tensorflow:loss = 0.21461567, step = 17401 (0.626 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.03\n",
      "INFO:tensorflow:loss = 0.21187477, step = 17501 (0.621 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.978\n",
      "INFO:tensorflow:loss = 0.21727398, step = 17601 (0.616 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.556\n",
      "INFO:tensorflow:loss = 0.22727951, step = 17701 (0.627 sec)\n",
      "INFO:tensorflow:global_step/sec: 163.579\n",
      "INFO:tensorflow:loss = 0.18230766, step = 17801 (0.612 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.67\n",
      "INFO:tensorflow:loss = 0.23635797, step = 17901 (0.621 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.761\n",
      "INFO:tensorflow:loss = 0.17438987, step = 18001 (0.607 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.919\n",
      "INFO:tensorflow:loss = 0.37694708, step = 18101 (0.617 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.184\n",
      "INFO:tensorflow:loss = 0.18793851, step = 18201 (0.628 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.389\n",
      "INFO:tensorflow:loss = 0.32464847, step = 18301 (0.628 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.403\n",
      "INFO:tensorflow:loss = 0.25268903, step = 18401 (0.616 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.315\n",
      "INFO:tensorflow:loss = 0.2748974, step = 18501 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.897\n",
      "INFO:tensorflow:loss = 0.18862587, step = 18601 (0.618 sec)\n",
      "INFO:tensorflow:global_step/sec: 158.748\n",
      "INFO:tensorflow:loss = 0.20510553, step = 18701 (0.630 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.19\n",
      "INFO:tensorflow:loss = 0.23256302, step = 18801 (0.616 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.868\n",
      "INFO:tensorflow:loss = 0.30329007, step = 18901 (0.622 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.185\n",
      "INFO:tensorflow:loss = 0.24137267, step = 19001 (0.624 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.914\n",
      "INFO:tensorflow:loss = 0.25696272, step = 19101 (0.625 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.968\n",
      "INFO:tensorflow:loss = 0.29134586, step = 19201 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.572\n",
      "INFO:tensorflow:loss = 0.31439272, step = 19301 (0.615 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.444\n",
      "INFO:tensorflow:loss = 0.22030121, step = 19401 (0.619 sec)\n",
      "INFO:tensorflow:global_step/sec: 160.041\n",
      "INFO:tensorflow:loss = 0.21812448, step = 19501 (0.626 sec)\n",
      "INFO:tensorflow:global_step/sec: 162.762\n",
      "INFO:tensorflow:loss = 0.3038418, step = 19601 (0.614 sec)\n",
      "INFO:tensorflow:global_step/sec: 164.175\n",
      "INFO:tensorflow:loss = 0.19754526, step = 19701 (0.609 sec)\n",
      "INFO:tensorflow:global_step/sec: 159.99\n",
      "INFO:tensorflow:loss = 0.18294753, step = 19801 (0.625 sec)\n",
      "INFO:tensorflow:global_step/sec: 161.516\n",
      "INFO:tensorflow:loss = 0.28398716, step = 19901 (0.619 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 20000 into ./mnist_convnet_student_10_0.75/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.31288576.\n",
      "_THIS_ <type 'dict'>\n",
      "INFO:tensorflow:Restoring parameters from /tmp/mnist_convnet_model_test1/model.ckpt-20000\n",
      "Tensor(\"Const:0\", shape=(128, 10), dtype=float32)\n",
      "INFO:tensorflow:Starting evaluation at 2018-05-06-11:04:11\n",
      "INFO:tensorflow:Restoring parameters from ./mnist_convnet_student_10_0.75/model.ckpt-20000\n",
      "INFO:tensorflow:Finished evaluation at 2018-05-06-11:04:11\n",
      "INFO:tensorflow:Saving dict for global step 20000: accuracy = 0.9516431, global_step = 20000, loss = 0.16218857\n",
      "{'loss': 0.16218857, 'global_step': 20000, 'accuracy': 0.9516431}\n"
     ]
    }
   ],
   "source": [
    "temprature=10\n",
    "loss_weight=0.75\n",
    "\n",
    "mnist_classifier = tf.estimator.Estimator(\n",
    "  model_fn=cnn_model_fn, model_dir=\"./models/mnist_convnet_model_test1\")\n",
    "\n",
    "# Create the Estimator\n",
    "student_classifier = tf.estimator.Estimator(\n",
    "  model_fn=student_model_fn, model_dir=\"./models/mnist_convnet_student_10_0.75\")\n",
    "\n",
    "# Train the model\n",
    "train_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "  x={\"x\": train_data},\n",
    "  y=train_labels,\n",
    "  batch_size=100,\n",
    "  num_epochs=None,\n",
    "  shuffle=True)\n",
    "print(train_data,train_labels,eval_data,eval_labels)\n",
    "\n",
    "student_classifier.train(\n",
    "  input_fn=train_input_fn,\n",
    "  steps=20000,\n",
    "  hooks=None)\n",
    "# print(\"eva\",type(eval_data))\n",
    "\n",
    "# Evaluate the model and print results\n",
    "eval_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "  x={\"x\": eval_data},\n",
    "  y=eval_labels,\n",
    "  num_epochs=1,\n",
    "  shuffle=False)\n",
    "eval_result=student_classifier.evaluate(input_fn=eval_input_fn)\n",
    "# output=list(predictions)\n",
    "print(eval_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}