char-model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import json\n",
    "import random\n",
    "from tensorflow.contrib.layers.python.layers import batch_norm\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "msgs = json.load(open('../data/my_messages.json'))\n",
    "chars = 'abcdefghijklmnopqrstuvwxyz'\n",
    "chars += chars.upper()\n",
    "chars += '1234567890,;:?!./ _-\\'\"\\n'\n",
    "PAD = 0\n",
    "START = 1\n",
    "END = 2\n",
    "chars = [PAD, START, END] + list(chars)\n",
    "\n",
    "def strip_msg(m):\n",
    "    return ''.join(c for c in m if c in chars)\n",
    "\n",
    "msgs = [strip_msg(m) for m in msgs]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "is this a chat about going back to the stairs and buying 4loko\n",
      "or am I misinterpreting it\n"
     ]
    }
   ],
   "source": [
    "print msgs[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def ints_from_text(text):\n",
    "    return np.array([START] + [chars.index(c) for c in text] + [END], dtype=np.int)\n",
    "\n",
    "def ints_to_text(ints):\n",
    "    return ''.join([chars[i] for i in ints if i not in [PAD, START, END]])\n",
    "\n",
    "int_msgs = [ints_from_text(text) for text in msgs if len(text.strip()) > 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# def rand_sample():\n",
    "#     msg = random.choice(int_msgs)\n",
    "#     msg = [BLANK_CHAR] * (BLOCK_LEN-1) + msg\n",
    "#     i = random.randint(BLOCK_LEN, len(msg)-1)\n",
    "#     char = msg[i]\n",
    "#     prev = msg[i-BLOCK_LEN:i]\n",
    "#     return prev, char\n",
    "\n",
    "# print rand_sample()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "def causal_conv_1d(input, dilation, out_channels, is_skip_connection=False, name='causal_conv_1d'):\n",
    "    # takes a 1-dimensional tensor! no batches here...\n",
    "    filter_size = 3\n",
    "    \n",
    "    assert dilation >= 0\n",
    "    \n",
    "    conv_init = tf.truncated_normal_initializer(0.0, 0.1)\n",
    "    bias_init = tf.truncated_normal_initializer(0.1, 0.0)\n",
    "    \n",
    "    with tf.variable_scope(name):\n",
    "        in_channels = input.get_shape()[-1].value\n",
    "                \n",
    "        # reshape to 2d so we can use atrous_conv2d:\n",
    "        # input = tf.reshape(input, [1, -1, 1, in_channels])\n",
    "        input = tf.reshape(input, [1, -1, 1, in_channels])\n",
    "        \n",
    "        w = tf.get_variable('w', \n",
    "                        shape=[filter_size/2+1, 1, in_channels, out_channels], \n",
    "                        initializer=conv_init)\n",
    "        filter = tf.pad(w, [[0, filter_size/2], [0,0], [0,0], [0,0]])\n",
    "        \n",
    "        output = tf.nn.atrous_conv2d(input, filter, dilation, 'SAME')\n",
    "        \n",
    "        b = tf.get_variable('b', shape=[out_channels], initializer=bias_init)\n",
    "        output = output + b\n",
    "        \n",
    "        # reshape back to 1d:\n",
    "        output = tf.reshape(output, [-1, out_channels])\n",
    "        return output\n",
    "\n",
    "def causal_conv_1d_padding_method(input, dilation, out_channels, is_skip_connection=False, name='causal_conv_1d'):\n",
    "    # takes a 1-dimensional tensor! no batches here...\n",
    "    filter_size = 2\n",
    "    \n",
    "    assert dilation >= 0\n",
    "    \n",
    "    conv_init = tf.truncated_normal_initializer(0.0, 0.1)\n",
    "    bias_init = tf.truncated_normal_initializer(0.1, 0.0)\n",
    "    \n",
    "    with tf.variable_scope(name):\n",
    "        in_channels = input.get_shape()[-1].value\n",
    "                \n",
    "        # reshape to 2d so we can use atrous_conv2d:\n",
    "        input = tf.reshape(input, [1, -1, 1, in_channels])\n",
    "        \n",
    "        w = tf.get_variable('w', \n",
    "                        shape=[filter_size, 1, in_channels, out_channels], \n",
    "                        initializer=conv_init)\n",
    "                \n",
    "        output = tf.nn.atrous_conv2d(input, w, dilation, 'SAME')\n",
    "        \n",
    "        b = tf.get_variable('b', shape=[out_channels], initializer=bias_init)\n",
    "        output = output + b\n",
    "        \n",
    "        # reshape back to 1d:\n",
    "        output = tf.reshape(output, [-1, out_channels])\n",
    "        return output\n",
    "\n",
    "def create_fc(input, out_channels, name='fc'):\n",
    "    with tf.variable_scope(name):\n",
    "        in_channels = input.get_shape()[-1].value\n",
    "        w = tf.get_variable('w', shape=[in_channels, out_channels], initializer=xavier_initializer())\n",
    "        b = tf.get_variable('b', shape=[out_channels], initializer=bias_init)\n",
    "        return tf.matmul(input, w) + b\n",
    "\n",
    "def create_conv_1d(input, filter_size, out_channels, name='conv1d'):\n",
    "    conv_init = tf.truncated_normal_initializer(0.0, 0.1)\n",
    "    bias_init = tf.truncated_normal_initializer(0.1, 0.0)\n",
    "    \n",
    "    with tf.variable_scope(name):\n",
    "        # turn input into a batch:\n",
    "        in_channels = input.get_shape()[-1].value\n",
    "        input = tf.reshape(input, [1, -1, in_channels])\n",
    "        w = tf.get_variable('w', shape=[filter_size, in_channels, out_channels], initializer=conv_init)\n",
    "        b = tf.get_variable('b', shape=[out_channels], initializer=bias_init)\n",
    "        output = tf.nn.conv1d(input, w, 1, 'SAME') + b\n",
    "        return tf.reshape(output, [-1, out_channels])\n",
    "    \n",
    "def create_batch_norm(inputs, name='bn'):\n",
    "    with tf.variable_scope(name):\n",
    "        return batch_norm(inputs, is_training=True, updates_collections=None)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "def char_model(seq):\n",
    "    # sequence: [None]\n",
    "    \n",
    "    char_dimension = len(chars)\n",
    "    layers = [\n",
    "        128,\n",
    "        128,\n",
    "        128,\n",
    "        128,\n",
    "        128\n",
    "    ]\n",
    "    receptive_field = 2 ** len(layers)\n",
    "        \n",
    "    seq = tf.one_hot(seq, char_dimension)\n",
    "    \n",
    "    seq = create_conv_1d(seq, 1, 64, name='char_conv')\n",
    "    seq = tf.nn.elu(seq)\n",
    "    \n",
    "    for i, channel_count in enumerate(layers):\n",
    "        dilation = 2 ** i\n",
    "                \n",
    "        seq = causal_conv_1d(seq, dilation, channel_count, name='dilated_conv' + str(i))\n",
    "        seq = tf.nn.elu(seq)\n",
    "        \n",
    "        # seq = create_batch_norm(seq, name='bn' + str(i))\n",
    "        \n",
    "        # seq = create_conv_1d(seq, 1, channel_count, name='conv' + str(i))\n",
    "        \n",
    "        # skip = seq\n",
    "        \n",
    "        # if i + 1 < len(layers):\n",
    "            # seq = tf.nn.elu(seq)\n",
    "            # seq = tf.nn.dropout(seq, 0.75)\n",
    "            # seq = create_batch_norm(seq, name='bn' + str(i))\n",
    "    \n",
    "    seq = create_conv_1d(seq, 1, char_dimension)\n",
    "    \n",
    "    return seq\n",
    "\n",
    "with tf.variable_scope('char25'):\n",
    "    input = tf.placeholder(tf.int32, [None], name='input')\n",
    "    output = char_model(input)\n",
    "    loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(output[:-1], input[1:]))\n",
    "    global_step = tf.contrib.framework.get_or_create_global_step()\n",
    "    lr = tf.placeholder(tf.float32, [], name='lr')\n",
    "    train_step = tf.train.AdamOptimizer(lr).minimize(loss, global_step=global_step)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Will not save progress\n"
     ]
    }
   ],
   "source": [
    "save_path = None # 'models/acgan3-20'\n",
    "\n",
    "session = tf.InteractiveSession()\n",
    "init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())\n",
    "session.run(init_op)\n",
    "tf.train.start_queue_runners(sess=session)\n",
    "\n",
    "import os\n",
    "saver = None\n",
    "if save_path:\n",
    "    if not os.path.exists(save_path):\n",
    "        os.mkdir(save_path)\n",
    "    saver = tf.train.Saver()\n",
    "    ckpt = tf.train.get_checkpoint_state(save_path)\n",
    "    if ckpt and ckpt.model_checkpoint_path:\n",
    "        saver.restore(session, ckpt.model_checkpoint_path)\n",
    "        print 'Restored from checkpoint', ckpt.model_checkpoint_path\n",
    "    else:\n",
    "        print 'Did not restore from checkpoint'\n",
    "else:\n",
    "    print 'Will not save progress'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def predict(max_len=20):\n",
    "    prev_chars = [START]\n",
    "    while len(prev_chars) < max_len and prev_chars[-1] != END:\n",
    "        op_ = session.run(output, feed_dict={input: prev_chars})\n",
    "        # print op_.shape\n",
    "        character_probs = session.run(tf.nn.softmax(output[-1]), feed_dict={input: prev_chars})\n",
    "        # print character_probs\n",
    "        character = np.random.choice(len(character_probs), p=character_probs)\n",
    "        # character = np.argmax(character_probs)\n",
    "        prev_chars.append(character)\n",
    "    return ints_to_text(prev_chars)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 19000, loss: 2.39981365204\n",
      "theth\n",
      "fo tha you bg\n",
      "Step: 19500, loss: 2.41388511658\n",
      "et\n",
      "Step: 20000, loss: 1.75678217411\n",
      "onn\n",
      "Step: 20500, loss: 2.40287494659\n",
      "no\n",
      "Step: 21000, loss: 2.52069711685\n",
      "yight himi\n",
      "Step: 21500, loss: 2.83501195908\n",
      "wh!\n",
      "Step: 22000, loss: 2.57978916168\n",
      "it'th  and\n",
      "got\n",
      "Step: 22500, loss: 1.48049283028\n",
      "i'm\n",
      "Step: 23000, loss: 2.28475642204\n",
      "kave bo cm to gacip\n",
      "Step: 23500, loss: 1.52788722515\n",
      "wort?\n",
      "Step: 24000, loss: 2.0639359951\n",
      "AB Braminb's ut ham\n",
      "Step: 24500, loss: 2.95376300812\n",
      "it thed get\n",
      "Step: 25000, loss: 3.11235833168\n",
      "buck\n",
      "Step: 25500, loss: 2.56164550781\n",
      "lloz yon ton lo\n",
      "Step: 26000, loss: 2.45597982407\n",
      "is OPN FZRKUGOMa\n",
      "ha\n",
      "Step: 26500, loss: 1.38520336151\n",
      "chere bere\n",
      "Step: 27000, loss: 2.40951418877\n",
      "son\n",
      "Step: 27500, loss: 4.43081331253\n",
      "OjE\n",
      "Step: 28000, loss: 1.68124842644\n",
      "came you loh ticand\n",
      "Step: 28500, loss: 2.17301344872\n",
      "INLA\n",
      "Step: 29000, loss: 3.43153810501\n",
      "hoh 1:\n",
      "Step: 29500, loss: 2.32773900032\n",
      "waite it sore tid '\n",
      "Step: 30000, loss: 2.54777312279\n",
      "chat ey\n",
      "Step: 30500, loss: 3.06897497177\n",
      "AHY\n",
      "Step: 31000, loss: 2.6062130928\n",
      "kk 500m heilmo\n",
      "a fi\n",
      "Step: 31500, loss: 1.86755859852\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-c5e469956625>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mstep_\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m500\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0;32mprint\u001b[0m \u001b[0;34m\"Step: {}, loss: {}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlosses\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlosses\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m         \u001b[0;32mprint\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m         \u001b[0mlosses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-9-78e5d7205db4>\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(max_len)\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0mop_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mprev_chars\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0;31m# print op_.shape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m         \u001b[0mcharacter_probs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mprev_chars\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m         \u001b[0;31m# print character_probs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0mcharacter\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchoice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcharacter_probs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcharacter_probs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/nn_ops.pyc\u001b[0m in \u001b[0;36msoftmax\u001b[0;34m(logits, dim, name)\u001b[0m\n\u001b[1;32m   1353\u001b[0m       \u001b[0mdimension\u001b[0m \u001b[0mof\u001b[0m \u001b[0;34m`\u001b[0m\u001b[0mlogits\u001b[0m\u001b[0;34m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1354\u001b[0m   \"\"\"\n\u001b[0;32m-> 1355\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0m_softmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogits\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgen_nn_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_softmax\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1356\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1357\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/nn_ops.pyc\u001b[0m in \u001b[0;36m_softmax\u001b[0;34m(logits, compute_op, dim, name)\u001b[0m\n\u001b[1;32m   1303\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mdim\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1304\u001b[0m     \u001b[0minput_shape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogits\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1305\u001b[0;31m     \u001b[0mlogits\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_flatten_outer_dims\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogits\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1306\u001b[0m     \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompute_op\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogits\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1307\u001b[0m     \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/nn_ops.pyc\u001b[0m in \u001b[0;36m_flatten_outer_dims\u001b[0;34m(logits)\u001b[0m\n\u001b[1;32m   1244\u001b[0m   last_dim_size = array_ops.slice(\n\u001b[1;32m   1245\u001b[0m       array_ops.shape(logits), [math_ops.sub(rank, 1)], [1])\n\u001b[0;32m-> 1246\u001b[0;31m   \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogits\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlast_dim_size\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1247\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1248\u001b[0m   \u001b[0;31m# Set output shape if known.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/array_ops.pyc\u001b[0m in \u001b[0;36mconcat\u001b[0;34m(concat_dim, values, name)\u001b[0m\n\u001b[1;32m   1078\u001b[0m   return gen_array_ops._concat(concat_dim=concat_dim,\n\u001b[1;32m   1079\u001b[0m                                \u001b[0mvalues\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1080\u001b[0;31m                                name=name)\n\u001b[0m\u001b[1;32m   1081\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1082\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_array_ops.pyc\u001b[0m in \u001b[0;36m_concat\u001b[0;34m(concat_dim, values, name)\u001b[0m\n\u001b[1;32m    436\u001b[0m   \"\"\"\n\u001b[1;32m    437\u001b[0m   result = _op_def_lib.apply_op(\"Concat\", concat_dim=concat_dim,\n\u001b[0;32m--> 438\u001b[0;31m                                 values=values, name=name)\n\u001b[0m\u001b[1;32m    439\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    440\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.pyc\u001b[0m in \u001b[0;36mapply_op\u001b[0;34m(self, op_type_name, name, **keywords)\u001b[0m\n\u001b[1;32m    757\u001b[0m         op = g.create_op(op_type_name, inputs, output_types, name=scope,\n\u001b[1;32m    758\u001b[0m                          \u001b[0minput_types\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minput_types\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattr_protos\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 759\u001b[0;31m                          op_def=op_def)\n\u001b[0m\u001b[1;32m    760\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0moutput_structure\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    761\u001b[0m           \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.pyc\u001b[0m in \u001b[0;36mcreate_op\u001b[0;34m(self, op_type, inputs, dtypes, input_types, name, attrs, op_def, compute_shapes, compute_device)\u001b[0m\n\u001b[1;32m   2240\u001b[0m                     original_op=self._default_original_op, op_def=op_def)\n\u001b[1;32m   2241\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcompute_shapes\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2242\u001b[0;31m       \u001b[0mset_shapes_for_outputs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mret\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2243\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_add_op\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mret\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2244\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_record_op_seen_by_control_dependencies\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mret\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.pyc\u001b[0m in \u001b[0;36mset_shapes_for_outputs\u001b[0;34m(op)\u001b[0m\n\u001b[1;32m   1615\u001b[0m       \u001b[0mshape_func\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_call_cpp_shape_fn_and_require_op\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1616\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1617\u001b[0;31m   \u001b[0mshapes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mshape_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1618\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mshapes\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1619\u001b[0m     raise RuntimeError(\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.pyc\u001b[0m in \u001b[0;36mcall_with_requiring\u001b[0;34m(op)\u001b[0m\n\u001b[1;32m   1566\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1567\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mcall_with_requiring\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1568\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mcall_cpp_shape_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequire_shape_fn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1569\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1570\u001b[0m   \u001b[0m_call_cpp_shape_fn_and_require_op\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_with_requiring\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/common_shapes.pyc\u001b[0m in \u001b[0;36mcall_cpp_shape_fn\u001b[0;34m(op, input_tensors_needed, input_tensors_as_shapes_needed, debug_python_shape_fn, require_shape_fn)\u001b[0m\n\u001b[1;32m    608\u001b[0m     res = _call_cpp_shape_fn_impl(op, input_tensors_needed,\n\u001b[1;32m    609\u001b[0m                                   \u001b[0minput_tensors_as_shapes_needed\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 610\u001b[0;31m                                   debug_python_shape_fn, require_shape_fn)\n\u001b[0m\u001b[1;32m    611\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mres\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    612\u001b[0m       \u001b[0;31m# Handles the case where _call_cpp_shape_fn_impl calls unknown_shape(op).\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/common_shapes.pyc\u001b[0m in \u001b[0;36m_call_cpp_shape_fn_impl\u001b[0;34m(op, input_tensors_needed, input_tensors_as_shapes_needed, debug_python_shape_fn, require_shape_fn)\u001b[0m\n\u001b[1;32m    646\u001b[0m     \u001b[0;31m# pylint: enable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    647\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSerializeToString\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 648\u001b[0;31m   \u001b[0minput_shapes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtensor_to_inference_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    649\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    650\u001b[0m   \u001b[0minput_tensors\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mNone\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minput_shapes\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/common_shapes.pyc\u001b[0m in \u001b[0;36mtensor_to_inference_result\u001b[0;34m(t)\u001b[0m\n\u001b[1;32m    645\u001b[0m     \u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandle_dtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle_dtype\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    646\u001b[0m     \u001b[0;31m# pylint: enable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 647\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSerializeToString\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    648\u001b[0m   \u001b[0minput_shapes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtensor_to_inference_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    649\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/google/protobuf/internal/python_message.pyc\u001b[0m in \u001b[0;36mSerializeToString\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1054\u001b[0m     \u001b[0;31m# Check if the message has all of its required fields set.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1055\u001b[0m     \u001b[0merrors\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1056\u001b[0;31m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIsInitialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1057\u001b[0m       raise message_mod.EncodeError(\n\u001b[1;32m   1058\u001b[0m           'Message %s is missing required fields: %s' % (\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/google/protobuf/internal/python_message.pyc\u001b[0m in \u001b[0;36mIsInitialized\u001b[0;34m(self, errors)\u001b[0m\n\u001b[1;32m   1167\u001b[0m                 \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFindInitializationErrors\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1168\u001b[0m               \u001b[0;32mreturn\u001b[0m \u001b[0mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1169\u001b[0;31m         \u001b[0;32melif\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_present_in_parent\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIsInitialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1170\u001b[0m           \u001b[0;32mif\u001b[0m \u001b[0merrors\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1171\u001b[0m             \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFindInitializationErrors\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/google/protobuf/internal/python_message.pyc\u001b[0m in \u001b[0;36mIsInitialized\u001b[0;34m(self, errors)\u001b[0m\n\u001b[1;32m   1157\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1158\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mfield\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fields\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# dict can change size!\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1159\u001b[0;31m       \u001b[0;32mif\u001b[0m \u001b[0mfield\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpp_type\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_FieldDescriptor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCPPTYPE_MESSAGE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1160\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mfield\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabel\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_FieldDescriptor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLABEL_REPEATED\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1161\u001b[0m           if (field.message_type.has_options and\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "def get_sample():\n",
    "    return random.choice(int_msgs) # [START] + random.choice(int_msgs) + [END]\n",
    "\n",
    "while True:\n",
    "    losses = []\n",
    "    feed = {\n",
    "        input: get_sample(),\n",
    "        lr: 0.0001\n",
    "    }\n",
    "    step_, loss_, _ = session.run([global_step, loss, train_step], feed_dict=feed)\n",
    "    losses.append(loss_)\n",
    "    if step_ % 500 == 0:\n",
    "        print \"Step: {}, loss: {}\".format(step_, sum(losses) / len(losses))\n",
    "        print predict()\n",
    "        losses = []\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "print predict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}