names-variational-autoencoder.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "names = set()\n",
    "for filename in ['male.txt', 'female.txt']:\n",
    "    for line in open(os.path.join('../data/names', filename)):\n",
    "        if len(line.strip()):\n",
    "            names.add(line.strip().lower())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7583 names\n",
      "longest: 54\n",
      "99th percentile longest: 9\n"
     ]
    }
   ],
   "source": [
    "print len(names), 'names'\n",
    "print 'longest:', max(map(len, names))\n",
    "by_len = sorted(names, key=len)\n",
    "print '99th percentile longest:', len(by_len[int(len(names) * 0.95)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[13  0 19  4 26 27 27 27 27 27]\n"
     ]
    }
   ],
   "source": [
    "chars = list('abcdefghijklmnopqrstuvwxyz') + ['<END>', '<NULL>']\n",
    "indices_for_chars = {c: i for i, c in enumerate(chars)}\n",
    "\n",
    "NAME_MAX_LEN = 10 # include the <END> char\n",
    "\n",
    "def name_to_vec(name, maxlen=NAME_MAX_LEN):\n",
    "    v = np.zeros(maxlen, dtype=int)\n",
    "    null_idx = indices_for_chars['<NULL>']\n",
    "    v.fill(null_idx)\n",
    "    for i, c in enumerate(name):\n",
    "        if i >= maxlen: break\n",
    "        n = indices_for_chars.get(c, null_idx)\n",
    "        v[i] = n\n",
    "    v[min(len(name), maxlen-1)] = indices_for_chars['<END>']\n",
    "    return v\n",
    "\n",
    "def vec_to_name(vec):\n",
    "    name = ''\n",
    "    for x in vec:\n",
    "        char = chars[x]\n",
    "        if len(char) == 1:\n",
    "            name += char\n",
    "        elif char == '<END>':\n",
    "            return name\n",
    "    return name\n",
    "\n",
    "print name_to_vec('nate')\n",
    "assert vec_to_name(name_to_vec('nate')) == 'nate'\n",
    "assert vec_to_name(name_to_vec('aaaaaaaaaaaa')) == 'aaaaaaaaa'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7583, 10)\n"
     ]
    }
   ],
   "source": [
    "name_vecs = np.array([name_to_vec(n) for n in names])\n",
    "print name_vecs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weight_var(shape, stddev=0.1, weight_decay=0, name=None):\n",
    "    initial = tf.truncated_normal(shape, stddev=stddev)\n",
    "    v = tf.Variable(initial, name=name)\n",
    "    if weight_decay > 0:\n",
    "        l2 = tf.nn.l2_loss(v) * weight_decay\n",
    "        tf.add_to_collection('losses', l2)\n",
    "    return v\n",
    "\n",
    "def leaky_relu(x, leak=0.2, name=\"lrelu\"):\n",
    "    with tf.variable_scope(name):\n",
    "        f1 = 0.5 * (1 + leak)\n",
    "        f2 = 0.5 * (1 - leak)\n",
    "        return f1 * x + f2 * abs(x)\n",
    "\n",
    "def relu(x):\n",
    "    # return tf.nn.relu(x)\n",
    "    return leaky_relu(x)\n",
    "\n",
    "def create_conv(input, out_channels, patch_size=5, stride=1, batch_norm=False, dropout=False):\n",
    "    in_channels = input.get_shape()[-1].value\n",
    "    w = weight_var([patch_size, patch_size, in_channels, out_channels])\n",
    "    b = weight_var([out_channels], stddev=0)\n",
    "    conv = tf.nn.conv2d(input, w, strides=[1,stride,stride,1], padding='SAME')\n",
    "    if batch_norm: conv = create_batch_norm(conv)\n",
    "    activation = relu(conv + b)\n",
    "    if dropout: activation = create_dropout(activation)\n",
    "    return activation\n",
    "    \n",
    "def text_conv(input, out_channels, patch_size=5, stride=1, dropout=False, pool_size=1):\n",
    "    in_channels = input.get_shape()[-1].value\n",
    "    w = weight_var([patch_size, in_channels, out_channels])\n",
    "    b = weight_var([out_channels], stddev=0)\n",
    "    conv = tf.nn.conv1d(input, w, stride=stride, padding='SAME')\n",
    "    activation = relu(conv + b)\n",
    "    # TODO: max_pooling\n",
    "    if dropout: activation = create_dropout(activation)\n",
    "    return activation\n",
    "\n",
    "def create_dropout(units):\n",
    "    return tf.nn.dropout(units, dropout)\n",
    "\n",
    "def create_fc(input, out_size):\n",
    "    # input_dropped = tf.nn.dropout(input, dropout_keep_prob)\n",
    "    in_size = input.get_shape()[-1].value\n",
    "    w = weight_var([in_size, out_size], weight_decay=0.004)\n",
    "    b = weight_var([out_size], weight_decay=0.004)\n",
    "    x = tf.matmul(input, w)\n",
    "    return relu(x + b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "name_placeholder = tf.placeholder(shape=[None, NAME_MAX_LEN], dtype=tf.int32, name='names')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Z_SIZE = 64\n",
    "\n",
    "def encoder_lstm(names):\n",
    "    with tf.variable_scope('encoder'):\n",
    "        cells = [tf.nn.rnn_cell.LSTMCell(size, state_is_tuple=True) for size in [len(chars), 64]]\n",
    "        lstm = tf.nn.rnn_cell.MultiRNNCell(cells, state_is_tuple=True)\n",
    "        one_hot = tf.one_hot(names, len(chars), dtype=tf.float32)\n",
    "        outputs, state = tf.nn.dynamic_rnn(lstm, one_hot, dtype=tf.float32)\n",
    "        outputs_flat = tf.reshape(outputs, [-1, 64 * NAME_MAX_LEN])\n",
    "        z_mean = create_fc(outputs_flat, Z_SIZE)\n",
    "        z_stddev = create_fc(outputs_flat, Z_SIZE)\n",
    "        return z_mean, z_stddev\n",
    "\n",
    "def encoder_conv(names):\n",
    "    with tf.variable_scope('encoder'):\n",
    "        one_hot = tf.one_hot(names, len(chars), dtype=tf.float32)\n",
    "        conv1 = text_conv(one_hot, 64)\n",
    "        conv2 = text_conv(one_hot, 64)\n",
    "        fc1 = create_fc(tf.reshape(conv2, [-1, NAME_MAX_LEN * 64]), 128)\n",
    "        z_mean = create_fc(fc1, Z_SIZE)\n",
    "        z_stddev = create_fc(fc1, Z_SIZE)\n",
    "        return z_mean, z_stddev\n",
    "    \n",
    "# def generator(noise, name='generator'):\n",
    "#     with tf.variable_scope(name, reuse=None):\n",
    "#         cells = [tf.nn.rnn_cell.LSTMCell(size, state_is_tuple=True) for size in [NOISE_SIZE, 256, len(chars)]]\n",
    "#         lstm = tf.nn.rnn_cell.MultiRNNCell(cells, state_is_tuple=True)\n",
    "#         noise_repeated_over_time = tf.tile(tf.reshape(noise, [-1, 1, NOISE_SIZE]), [1, NAME_MAX_LEN, 1])\n",
    "#         outputs, state = tf.nn.dynamic_rnn(lstm, noise_repeated_over_time, dtype=tf.float32)\n",
    "#         output_chars = tf.reshape(tf.argmax(tf.nn.softmax(outputs), axis=2), [-1, NAME_MAX_LEN])\n",
    "#         output_chars = tf.cast(output_chars, tf.int32)\n",
    "#     return output_chars\n",
    "\n",
    "# generated_names = generator(noise)\n",
    "\n",
    "z_mean, z_stddev = encoder_lstm(name_placeholder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "session = tf.Session()\n",
    "session.run(tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.00513439 -0.02931023 -0.03145247 -0.0263676   0.17818263 -0.02248107\n",
      "   0.05441747 -0.01409492  0.00079724  0.06089506  0.12747321 -0.00688516\n",
      "   0.03540077  0.02594603  0.01325507  0.06816807 -0.00771422  0.08031344\n",
      "   0.08042179  0.1554797   0.06638941  0.03745911 -0.01567956 -0.01505864\n",
      "   0.14918545 -0.01923505  0.09962676  0.08324118  0.03816357  0.08563396\n",
      "  -0.0082801  -0.01891115 -0.01132262 -0.01889286  0.09922332  0.13186967\n",
      "  -0.02898554 -0.02655983 -0.03781658  0.02988116  0.00581382  0.10335773\n",
      "  -0.03385977  0.11658278 -0.0145012  -0.0057638   0.08082868  0.09749778\n",
      "   0.13155389 -0.00898156 -0.00842716  0.12816691 -0.03120333  0.07986332\n",
      "  -0.02080249  0.11234313 -0.00056841 -0.00056067 -0.02656401 -0.00772686\n",
      "   0.04402544 -0.01465668  0.01160432  0.07347233]]\n"
     ]
    }
   ],
   "source": [
    "print session.run(z_mean, feed_dict={name_placeholder: [name_to_vec('nate')]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sample_z(z_mean, z_stddev):\n",
    "    samples = tf.random_normal(tf.shape(z_stddev), 0, 1, dtype=tf.float32)\n",
    "    return z_mean + samples * z_stddev\n",
    "\n",
    "z_vals = sample_z(z_mean, z_stddev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def decoder_lstm(z):\n",
    "    z_repeated_over_time = tf.tile(tf.reshape(z, [-1, 1, Z_SIZE]), [1, NAME_MAX_LEN, 1])\n",
    "    cells = [tf.nn.rnn_cell.LSTMCell(size, state_is_tuple=True) for size in [Z_SIZE, 256, len(chars)]]\n",
    "    lstm = tf.nn.rnn_cell.MultiRNNCell(cells, state_is_tuple=True)\n",
    "    outputs, state = tf.nn.dynamic_rnn(lstm, z_repeated_over_time, dtype=tf.float32)\n",
    "    return outputs\n",
    "\n",
    "z_input = tf.placeholder(tf.float32, [None, Z_SIZE], name='z_input')\n",
    "use_z_input = tf.placeholder(tf.int32, shape=[], name=\"use_z_input_condition\")\n",
    "decoder_input = tf.cond(use_z_input > 0, lambda: z_input, lambda: z_vals)\n",
    "\n",
    "decoded = decoder_lstm(decoder_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "diff_loss = tf.reduce_sum(tf.nn.sparse_softmax_cross_entropy_with_logits(decoded, name_placeholder))\n",
    "kl_divergence = tf.reduce_mean(0.5 * tf.reduce_sum(tf.square(z_mean) + tf.square(z_stddev) - tf.log(tf.square(z_stddev)) - 1, 1))\n",
    "loss = diff_loss + kl_divergence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "decoded_vecs = tf.argmax(decoded, axis=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "learn_rate = tf.placeholder(tf.float32, name='learning_rate')\n",
    "optimizer = tf.train.AdamOptimizer(learn_rate)\n",
    "global_step = tf.contrib.framework.get_or_create_global_step()\n",
    "train_step = optimizer.minimize(loss, global_step=global_step)\n",
    "session.run(tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Restored from checkpoint models/nva2/model.ckpt-60600\n"
     ]
    }
   ],
   "source": [
    "save_path = 'models/nva2'\n",
    "\n",
    "session = tf.Session()\n",
    "init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())\n",
    "session.run(init_op)\n",
    "\n",
    "saver = None\n",
    "if save_path:\n",
    "    if not os.path.exists(save_path):\n",
    "        os.mkdir(save_path)\n",
    "    saver = tf.train.Saver()\n",
    "    ckpt = tf.train.get_checkpoint_state(save_path)\n",
    "    if ckpt and ckpt.model_checkpoint_path:\n",
    "        saver.restore(session, ckpt.model_checkpoint_path)\n",
    "        print 'Restored from checkpoint', ckpt.model_checkpoint_path\n",
    "    else:\n",
    "        print 'Did not restore from checkpoint'\n",
    "else:\n",
    "    print 'Will not save progress'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train = False\n",
    "while train:\n",
    "    names = name_vecs[np.random.randint(name_vecs.shape[0], size=64), :]\n",
    "    feed_dict = {\n",
    "        name_placeholder: names,\n",
    "        z_input: np.zeros((64, Z_SIZE)),\n",
    "        use_z_input: 0,\n",
    "        learn_rate: 0.001\n",
    "    }\n",
    "    _, loss_, step_ = session.run([train_step, loss, global_step], feed_dict=feed_dict)\n",
    "    if step_ % 200 == 0:\n",
    "        output_ = session.run(decoded_vecs, feed_dict=feed_dict)\n",
    "        print \"Step: {0}; loss: {1}\".format(step_, loss_)\n",
    "        # print names[0]\n",
    "        # print output_[0]\n",
    "        print \" example encoding: {} -> {}\".format(vec_to_name(names[0]), vec_to_name(output_[0]))\n",
    "        if step_ % 600 == 0:\n",
    "            saver.save(session, save_path + '/model.ckpt', global_step=step_)\n",
    "            print 'Saved'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " nate -> nate\n"
     ]
    }
   ],
   "source": [
    "def reconstruct(name):\n",
    "    feed_dict = {\n",
    "        name_placeholder: np.array([name_to_vec(name)]),\n",
    "        z_input: np.zeros((64, Z_SIZE)),\n",
    "        use_z_input: 0,\n",
    "        learn_rate: 0.01\n",
    "    }\n",
    "    output_ = session.run(decoded_vecs, feed_dict=feed_dict)\n",
    "    return vec_to_name(output_[0])\n",
    "\n",
    "for name in ['nate']:\n",
    "    print name, '->', reconstruct(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " nate -> nate\n",
      "will -> will\n",
      "chen -> chen\n",
      "atty -> atty\n",
      "arielle -> arielle\n",
      "nathaniel -> nathaniel\n",
      "kimberly -> kimberly\n",
      "erica -> eriaa\n",
      "zoe -> ooe\n"
     ]
    }
   ],
   "source": [
    "# reconstruct is pretty good at reconstructing american names, even long ones:\n",
    "for name in ['nate', 'will', 'chen', 'atty', 'arielle', 'nathaniel', 'kimberly', 'erica', 'zoe']:\n",
    "    print name, '->', reconstruct(name)\n",
    "\n",
    "# although notably, it's bad at reconstructing names with less-frequent letters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "word -> word\n",
      "happy -> pappy\n",
      "winter -> winter\n",
      "candle -> canele\n",
      "cherish -> cherish\n"
     ]
    }
   ],
   "source": [
    "# it's decent at some english words that 'sound' like ames:\n",
    "for name in ['word', 'happy', 'winter', 'candle', 'cherish']:\n",
    "    print name, '->', reconstruct(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "embedding -> meeeeline\n",
      "automobile -> tetooorrd\n",
      "air -> aer\n",
      "larynx -> larnne\n"
     ]
    }
   ],
   "source": [
    "# it's worse at more longer, more \"wordy\" names:\n",
    "for name in ['embedding', 'automobile', 'air', 'larynx']:\n",
    "    print name, '->', reconstruct(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ufhoe -> geooo\n",
      "xyzy -> lyey\n",
      "ihwrfoecoei -> sierrnnod\n"
     ]
    }
   ],
   "source": [
    "# predictably, it's terrible at things that aren't even pronouncable strings of letters:\n",
    "for name in ['ufhoe', 'xyzy', 'ihwrfoecoei']:\n",
    "    print name, '->', reconstruct(name)\n",
    "# it even seems to try to turn some into slightly more name-like strings:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nate : 1.0\n",
      "july : 0.75\n",
      "fridge : 0.833333333333\n",
      "gienigoe : 0.625\n",
      "chzsiucf : 0.375\n",
      "xyxyzzy : 0.0\n"
     ]
    }
   ],
   "source": [
    "# so reconstruction quality seems like a pretty good of how \"name-ish\" a word is\n",
    "# want to give your kid a cool, ~original~ name no one has, but that sounds good?\n",
    "# what good english words sound like names, but aren't?\n",
    "\n",
    "# first, let's build a 'nameliness' score:\n",
    "def nameliness(word):\n",
    "    r = reconstruct(word)\n",
    "    return sum([1 if a == b else 0 for a, b in zip(word, r)]) / float(len(word))\n",
    "\n",
    "for name in ['nate', 'july', 'fridge', 'gienigoe', 'chzsiucf', 'xyxyzzy']:\n",
    "    print name, ':', nameliness(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9268\n",
      "['the', 'and', 'to', 'a', 'in', 'for', 'is', 'that', 'with', 'i', 'not', 'or', 'be', 'are', 'at', 'all', 'more', 'an', 'was', 'can', 'us', 'my', 'free', 'but', 'do', 'no', 'time', 'site', 'he', 'any', 'there', 'so', 'his', 'here', 'also', 'get', 'am', 'were', 'me', 's', 'date', 'had', 'name', 'day', 're', 'go', 'b', 'most', 'her', 't', 'add', 'best', 'then', 'good', 'well', 'm', 'she', 'order', 'r', 'read', 'many', 'de', 'set', 'full', 'games', 'p', 'part', 'real', 'must', 'store', 'made', 'line', 'did', 'those', 'car', 'area', 'want', 'phone', 'code', 'o', 'file', 'same', 'both', 'photo', 'game', 'care', 'end', 'per', 'north', 'posts', 'water', 'shop', 'rate', 'usa', 'main', 'call', 'non', 'shall', 'class', 'still', 'money', 'man', 'card', 'j', 'food', 'press', 'u', 'sale', 'print', 'room', 'join', 'men', 'sales', 'note', 'gallery', 'table', 'model', 'plan', 'hot', 'cost', 'better', 'july', 'cart', 'san', 'play', 'person', 'less', 'let', 'side', 'story', 'sell', 'body', 'ii', 'road', 'ca', 'hard', 'pay', 'seller', 'write', 'war', 'files', 'hand', 'sun', 'cards', 'id', 'enter', 'porn', 'share', 'garden', 'added', 'baby', 'run', 'net', 'term', 'put', 'co', 'try', 'god', 'radio', 'cell', 'sure', 'cars', 'tell', 'green', 'close', 'common', 'gold', 'feb', 'short', 'arts', 'lot', 'daily', 'past', 'due', 'et', 'mar', 'land', 'done', 'pro', 'st', 'costs', 'url', 'parts', 'early', 'either', 'word', 'rules', 'nude', 'third', 'bad', 'fast', 'far', 'en', 'feel', 'jul', 'girl', 'loan', 'wide', 'sort', 'later', 'sony', 'chat', 'loss', 'brand', 'oil', 'bit', 'stay', 'turn', 'mean', 'copy', 'cash', 'ad', 'seen', 'port', 'bar']\n"
     ]
    }
   ],
   "source": [
    "# let's grab the top 10k english words, remove the things that are names, and see which can be reconstructed best:\n",
    "# source: https://github.com/first20hours/google-10000-english\n",
    "\n",
    "top_words = list(word.strip() for word in open('../data/google-10000-english.txt'))\n",
    "top_words = list(word for word in top_words if word not in names)\n",
    "print len(top_words)\n",
    "top_words = top_words[:1000] # this is actually kinda slow, so let's stick with the top 1k\n",
    "nameliness_scores = {word: nameliness(word) for word in top_words}\n",
    "print [w for w in top_words if nameliness_scores[w] == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# let's build a big lookup table of all the names and their embeddings:\n",
    "def make_batches(list, size=128):\n",
    "    batches = []\n",
    "    while len(list):\n",
    "        batches.append(list[:min(len(list), size)])\n",
    "        list = list[len(batches[-1]):]\n",
    "    return batches\n",
    "\n",
    "embeddings = {}\n",
    "\n",
    "for batch in make_batches(list(names)):\n",
    "    feed_dict = {\n",
    "        name_placeholder: np.array([name_to_vec(name) for name in batch]),\n",
    "        z_input: np.zeros((len(batch), Z_SIZE)),\n",
    "        use_z_input: 0\n",
    "    }\n",
    "    output_ = session.run(z_mean, feed_dict=feed_dict)\n",
    "    for name, vec in zip(batch, output_):\n",
    "        embeddings[name] = vec\n",
    "    # print 'processed {}/{}'.format(len(embeddings), len(names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nate is closest to nate\n",
      "yikes is closest to giles\n",
      "panda is closest to hanna\n",
      "ixzhxzi is closest to desirae\n",
      "justxn is closest to justin\n"
     ]
    }
   ],
   "source": [
    "def embed(name):\n",
    "    feed_dict = {\n",
    "        name_placeholder: np.array([name_to_vec(name)]),\n",
    "        z_input: np.zeros((1, Z_SIZE)),\n",
    "        use_z_input: 0\n",
    "    }\n",
    "    output_ = session.run(z_mean, feed_dict=feed_dict)\n",
    "    return output_[0]\n",
    "\n",
    "def nearest(embedding):\n",
    "    def distance(name):\n",
    "        return np.linalg.norm(embedding - embeddings[name])\n",
    "    return min(embeddings.iterkeys(), key=distance)\n",
    "\n",
    "def unembed(embedding):\n",
    "    feed_dict = {\n",
    "        name_placeholder: np.zeros((1, NAME_MAX_LEN)),\n",
    "        z_input: np.array([embedding]),\n",
    "        use_z_input: 1\n",
    "    }\n",
    "    output_ = session.run(decoded_vecs, feed_dict=feed_dict)\n",
    "    return vec_to_name(output_[0])\n",
    "\n",
    "assert unembed(embed('nate')) == 'nate'\n",
    "\n",
    "# print nearest(embed('nate'))\n",
    "for name in ['nate', 'yikes', 'panda', 'ixzhxzi', 'justxn']:\n",
    "    print name, 'is closest to', nearest(embed(name))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "amy\n",
      "amy\n",
      "ammy\n",
      "ammy\n",
      "aarey\n",
      "nareey\n",
      "nariee\n",
      "faanies\n",
      "franiiss\n",
      "fransisoo\n",
      "fransisoo\n"
     ]
    }
   ],
   "source": [
    "# what happens if we try to interpolate names?\n",
    "def blend_names(name1, name2):\n",
    "    e1 = embed(name1)\n",
    "    e2 = embed(name2)\n",
    "    for i in range(11):\n",
    "        blend = i / 10.0\n",
    "        print unembed(e1 * (1 - blend) + e2 * blend)\n",
    "\n",
    "blend_names('amy', 'francisco')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nathaniel\n",
      "nathaniel\n",
      "nathaniel\n",
      "kathnnie\n",
      "cathnnee\n",
      "cathnee\n",
      "hathney\n",
      "phanne\n",
      "chene\n",
      "chene\n",
      "chen\n"
     ]
    }
   ],
   "source": [
    "blend_names('nathaniel', 'chen')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "will\n",
      "will\n",
      "will\n",
      "wille\n",
      "wille\n",
      "wille\n",
      "willie\n",
      "willie\n",
      "willie\n",
      "willia\n",
      "william\n"
     ]
    }
   ],
   "source": [
    "blend_names('will', 'william')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nathaniel\n",
      "nathaniel\n",
      "nathaniel\n",
      "nathaniel\n",
      "nnthaniil\n",
      "nnthaniie\n",
      "nntaaniia\n",
      "innaaniia\n",
      "iinaaatin\n",
      "linntttnn\n",
      "leinnttnn\n"
     ]
    }
   ],
   "source": [
    "blend_names('nathaniel', 'leinahtan')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "beatriee\n",
      "beatriee\n",
      "mettriee\n",
      "meitriee\n",
      "mittrie\n",
      "timmree\n",
      "tierree\n",
      "sierlee\n",
      "shirley\n",
      "shirley\n",
      "shirley\n"
     ]
    }
   ],
   "source": [
    "blend_names('beatrice', 'shirley')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "selia\n",
      "seeera\n"
     ]
    }
   ],
   "source": [
    "print nearest(np.zeros(Z_SIZE))\n",
    "print unembed(np.zeros(Z_SIZE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nate * 2 = naty\n",
      "willy * 2 = willy\n",
      "sam * 2 = sam\n",
      "polly * 2 = pool\n",
      "jacob * 2 = jaoo\n"
     ]
    }
   ],
   "source": [
    "# what if we multiply names?:\n",
    "for name in ['nate', 'willy', 'sam', 'polly', 'jacob']:\n",
    "    print name, '* 2 =', unembed(embed(name) * 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-nancy = hhhctttsss\n",
      "-barry = jsuu\n",
      "-chance = ddhhdddsrs\n",
      "-rachel = hhhtrrrrrr\n",
      "-gloria = sselaa\n"
     ]
    }
   ],
   "source": [
    "# what's the opposite of a name?\n",
    "for name in ['nancy', 'barry', 'chance', 'rachel', 'gloria']:\n",
    "    print '-' + name, '=', unembed(-embed(name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "roberta\n",
      "justina\n",
      "josepha\n",
      "nanee\n"
     ]
    }
   ],
   "source": [
    "# can we do addition and subtraction?\n",
    "print unembed(embed('alberta') - embed('albert') + embed('robert'))\n",
    "print unembed(embed('alberta') - embed('albert') + embed('justin'))\n",
    "print unembed(embed('alberta') - embed('albert') + embed('joseph'))\n",
    "\n",
    "print unembed(embed('alberta') - embed('albert') + embed('nate')) # doesn't work so well with names ending in vowels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rddelahh\n",
      "ruoo\n",
      "phril\n",
      "lalelee\n",
      "saliaa\n",
      "hallio\n",
      "aldii\n",
      "fulls\n",
      "ddritt\n",
      "iiccccccct\n",
      "rrnn\n",
      "sime\n",
      "tiilsna\n",
      "jnnntthyhs\n",
      "sherlo\n",
      "palloe\n",
      "heulita\n",
      "harltt\n",
      "malmaa\n",
      "jelle\n",
      "achcssssss\n",
      "burdaa\n",
      "eobertt\n",
      "saaann\n",
      "caooyla\n",
      "oeonnn\n",
      "ttttto\n",
      "acchhhtsss\n",
      "wisla\n",
      "shttt\n"
     ]
    }
   ],
   "source": [
    "# let's generate some random names:\n",
    "def generate():\n",
    "    return unembed(np.random.normal(size=Z_SIZE))\n",
    "for _ in range(30):\n",
    "    print generate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# what if we train a GAN to mimick the latent vectors produced by real names?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def unembed_multi(embeddings):\n",
    "    feed_dict = {\n",
    "        name_placeholder: np.zeros((1, NAME_MAX_LEN)),\n",
    "        z_input: np.array(embeddings),\n",
    "        use_z_input: 1\n",
    "    }\n",
    "    output_ = session.run(decoded_vecs, feed_dict=feed_dict)\n",
    "    return [vec_to_name(op) for op in output_]\n",
    "\n",
    "if False:\n",
    "    names = open('names.txt', 'w')\n",
    "    for i in xrange(10000):\n",
    "        count = 32\n",
    "        embeddings = [np.random.normal(size=Z_SIZE) for _ in xrange(count)]\n",
    "        for name in unembed_multi(embeddings):\n",
    "            names.write(name + '\\n')\n",
    "        if i % 100 == 0: print i / 10000.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nate\n",
      "iatey\n",
      "iihey\n",
      "iihhey\n",
      "iihhey\n",
      "iiehey\n",
      "iieheey\n",
      "iieheey\n",
      "iiehee\n",
      "iiehee\n",
      "iieheey\n",
      "iieheey\n",
      "iieheey\n",
      "iieeeey\n",
      "iieeeey\n",
      "iieeeey\n",
      "iieieey\n",
      "iieieey\n",
      "iieieey\n",
      "iiiiiey\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def variations_on(name):\n",
    "    z = embed(name)\n",
    "    for i in xrange(20):\n",
    "        noise = np.random.normal(z.shape)\n",
    "        print unembed(z + noise * i * 0.01)\n",
    "\n",
    "variations_on('nate')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}