text-to-image-training.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating Synthetic Images from Textual Description using GANs\n",
    "\n",
    "### Requirements\n",
    "1. Tensorflow\n",
    "2. numpy\n",
    "3. scipy\n",
    "4. pickle\n",
    "5. json\n",
    "6. opencv\n",
    "7. h5py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import argparse\n",
    "import pickle\n",
    "import h5py\n",
    "import scipy\n",
    "import random\n",
    "import json\n",
    "import os\n",
    "import shutil\n",
    "from util import tf_ops as ops\n",
    "import cv2 as opencv\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyper-parameters\n",
    "z_dim = 100\n",
    "t_dim = 256\n",
    "batch_size = 64\n",
    "image_size = 64\n",
    "gf_dim = 64\n",
    "df_dim = 64\n",
    "gfc_dim = 1024\n",
    "caption_vector_length = 2400\n",
    "data_dir = \"Data\"\n",
    "learning_rate = 0.0002\n",
    "beta1 = 0.5\n",
    "epochs = 600\n",
    "save_every = 30\n",
    "resume_model = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv2d(_input, output_dim, k_h=5, k_w=5, d_h=2, d_w=2, stddev=0.02, name=\"conv2d\"):\n",
    "    with tf.variable_scope(name):\n",
    "        w = tf.get_variable('w', [k_h, k_w, _input.get_shape()[-1], output_dim],\n",
    "                            initializer=tf.truncated_normal_initializer(stddev=stddev))\n",
    "        _conv = tf.nn.conv2d(_input, w, strides=[1, d_h, d_w, 1], padding='SAME')\n",
    "\n",
    "        _biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0))\n",
    "        _conv = tf.reshape(tf.nn.bias_add(_conv, _biases), _conv.get_shape())\n",
    "\n",
    "        return _conv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def deconv2d(_input, output_shape, k_h=5, k_w=5, d_h=2, d_w=2, stddev=0.02,  name=\"deconv2d\"):\n",
    "    with tf.variable_scope(name):\n",
    "        # filter : [height, width, output_channels, in_channels]\n",
    "        w = tf.get_variable('w', [k_h, k_h, output_shape[-1], _input.get_shape()[-1]],\n",
    "                            initializer=tf.random_normal_initializer(stddev=stddev))\n",
    "        _deconv = tf.nn.conv2d_transpose(_input, w, output_shape=output_shape, strides=[1, d_h, d_w, 1])\n",
    "        _biases = tf.get_variable('biases', [output_shape[-1]], initializer=tf.constant_initializer(0.0))\n",
    "        _deconv = tf.reshape(tf.nn.bias_add(_deconv, _biases), _deconv.get_shape())\n",
    "    return _deconv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_layer(input_, output_size, scope=None, stddev=0.02, bias_start=0.0, with_w=False):\n",
    "    shape = input_.get_shape().as_list()\n",
    "\n",
    "    with tf.variable_scope(scope or \"Linear\"):\n",
    "        matrix = tf.get_variable(\"Matrix\", [shape[1], output_size], tf.float32,\n",
    "                                 tf.random_normal_initializer(stddev=stddev))\n",
    "        bias = tf.get_variable(\"bias\", [output_size],\n",
    "            initializer=tf.constant_initializer(bias_start))\n",
    "        if with_w:\n",
    "            return tf.matmul(input_, matrix) + bias, matrix, bias\n",
    "        else:\n",
    "            return tf.matmul(input_, matrix) + bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generator(t_z, t_text_embedding):\n",
    "    s = image_size\n",
    "    s2, s4, s8, s16 = int(s / 2), int(s / 4), int(s / 8), int(s / 16)\n",
    "\n",
    "    reduced_text_embedding = tf.nn.leaky_relu(linear_layer(t_text_embedding, t_dim, scope= 'g_embedding'))\n",
    "    z_concat = tf.concat([t_z, reduced_text_embedding], 1)\n",
    "    z_ = linear_layer(z_concat, gf_dim * 8 * s16 * s16, scope = 'g_h0_lin')\n",
    "    h0 = tf.reshape(z_, [-1, s16, s16, gf_dim * 8])\n",
    "    h0 = tf.nn.relu(tf.layers.batch_normalization(h0, momentum=0.9,epsilon = 1e-5,  training=True))\n",
    "\n",
    "    h1 = deconv2d(h0, [batch_size, s8, s8, gf_dim * 4], name='g_h1')\n",
    "    h1 = tf.nn.relu(tf.layers.batch_normalization(h1, momentum=0.9,epsilon = 1e-5,  training=True))\n",
    "\n",
    "    h2 = deconv2d(h1, [batch_size, s4, s4, gf_dim * 2], name='g_h2')\n",
    "    h2 = tf.nn.relu(tf.layers.batch_normalization(h2, momentum=0.9,epsilon = 1e-5,  training=True))\n",
    "\n",
    "    h3 = deconv2d(h2, [batch_size, s2, s2, gf_dim * 1], name='g_h3')\n",
    "    h3 = tf.nn.relu(tf.layers.batch_normalization(h3, momentum=0.9,epsilon = 1e-5,  training=True))\n",
    "\n",
    "    h4 = deconv2d(h3, [batch_size, s, s, 3], name='g_h4')\n",
    "\n",
    "    return (tf.tanh(h4) / 2. + 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def discriminator(image, t_text_embedding, reuse=False):\n",
    "    with tf.variable_scope(tf.get_variable_scope(), reuse=reuse):\n",
    "        h0 = tf.nn.leaky_relu(conv2d(image, df_dim, name='d_h0_conv'))  # 32\n",
    "        h1 = tf.nn.leaky_relu(tf.layers.batch_normalization(conv2d(h0, df_dim * 2, name='d_h1_conv'), momentum=0.9,epsilon = 1e-5,  training=True))  # 16\n",
    "        h2 = tf.nn.leaky_relu(tf.layers.batch_normalization(conv2d(h1, df_dim * 4, name='d_h2_conv'), momentum=0.9,epsilon = 1e-5,  training=True))  # 8\n",
    "        h3 = tf.nn.leaky_relu(tf.layers.batch_normalization(conv2d(h2, df_dim * 8, name='d_h3_conv'), momentum=0.9,epsilon = 1e-5,  training=True))  # 4\n",
    "\n",
    "        # ADD TEXT EMBEDDING TO THE NETWORK\n",
    "        reduced_text_embeddings = tf.nn.leaky_relu(linear_layer(t_text_embedding, t_dim, scope = 'd_embedding'))\n",
    "        reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 1)\n",
    "        reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 2)\n",
    "        tiled_embeddings = tf.tile(reduced_text_embeddings, [1, 4, 4, 1], name='tiled_embeddings')\n",
    "\n",
    "        h3_concat = tf.concat([h3, tiled_embeddings], 3, name='h3_concat')\n",
    "        h3_new = tf.nn.leaky_relu(\n",
    "            tf.layers.batch_normalization(conv2d(h3_concat, df_dim * 8, 1, 1, 1, 1, name='d_h3_conv_new'), momentum=0.9,epsilon = 1e-5,  training=True))  # 4\n",
    "\n",
    "        h4 = linear_layer(tf.reshape(h3_new, [batch_size, -1]), 1, scope = 'd_h3_lin')\n",
    "\n",
    "    return tf.nn.sigmoid(h4), h4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model():\n",
    "    t_real_image = tf.placeholder('float32', [batch_size, image_size, image_size, 3], name='real_image')\n",
    "    t_wrong_image = tf.placeholder('float32', [batch_size, image_size, image_size, 3], name='wrong_image')\n",
    "    t_real_caption = tf.placeholder('float32', [batch_size, caption_vector_length], name='real_caption_input')\n",
    "    t_z = tf.placeholder('float32', [batch_size, z_dim])\n",
    "\n",
    "    fake_image = generator(t_z, t_real_caption)\n",
    "\n",
    "    disc_real_image, disc_real_image_logits = discriminator(t_real_image, t_real_caption)\n",
    "    disc_wrong_image, disc_wrong_image_logits = discriminator(t_wrong_image, t_real_caption, reuse=True)\n",
    "    disc_fake_image, disc_fake_image_logits = discriminator(fake_image, t_real_caption, reuse=True)\n",
    "\n",
    "    g_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=disc_fake_image_logits,\n",
    "                                                                    labels=tf.ones_like(disc_fake_image)))\n",
    "\n",
    "    d_loss1 = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=disc_real_image_logits,\n",
    "                                                                     labels=tf.ones_like(disc_real_image)))\n",
    "    d_loss2 = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=disc_wrong_image_logits,\n",
    "                                                                     labels=tf.zeros_like(disc_wrong_image)))\n",
    "    d_loss3 = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=disc_fake_image_logits,\n",
    "                                                                     labels=tf.zeros_like(disc_fake_image)))\n",
    "\n",
    "    d_loss = d_loss1 + d_loss2 + d_loss3\n",
    "\n",
    "    t_vars = tf.trainable_variables()\n",
    "    d_vars = [var for var in t_vars if 'd_' in var.name]\n",
    "    g_vars = [var for var in t_vars if 'g_' in var.name]\n",
    "\n",
    "    input_tensors = {\n",
    "        't_real_image': t_real_image,\n",
    "        't_wrong_image': t_wrong_image,\n",
    "        't_real_caption': t_real_caption,\n",
    "        't_z': t_z\n",
    "    }\n",
    "\n",
    "    variables = {\n",
    "        'd_vars': d_vars,\n",
    "        'g_vars': g_vars\n",
    "    }\n",
    "\n",
    "    loss = {\n",
    "        'g_loss': g_loss,\n",
    "        'd_loss': d_loss\n",
    "    }\n",
    "\n",
    "    outputs = {\n",
    "        'generator': fake_image\n",
    "    }\n",
    "\n",
    "    checks = {\n",
    "        'd_loss1': d_loss1,\n",
    "        'd_loss2': d_loss2,\n",
    "        'd_loss3': d_loss3,\n",
    "        'disc_real_image_logits': disc_real_image_logits,\n",
    "        'disc_wrong_image_logits': disc_wrong_image,\n",
    "        'disc_fake_image_logits': disc_fake_image_logits\n",
    "    }\n",
    "\n",
    "    return input_tensors, variables, loss, outputs, checks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-6-742fc1e60b54>:9: batch_normalization (from tensorflow.python.layers.normalization) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.BatchNormalization instead.  In particular, `tf.control_dependencies(tf.GraphKeys.UPDATE_OPS)` should not be used (consult the `tf.keras.layers.batch_normalization` documentation).\n",
      "WARNING:tensorflow:From /Users/s0d02zy/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/nn_impl.py:180: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    }
   ],
   "source": [
    "input_tensors, variables, loss, outputs, checks = build_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_training_data(data_dir):\n",
    "    h = h5py.File(os.path.join(data_dir, 'flower_tv.hdf5'))\n",
    "    flower_captions = {}\n",
    "    for ds in h.items():\n",
    "        flower_captions[ds[0]] = np.array(ds[1])\n",
    "    image_list = [key for key in flower_captions]\n",
    "    image_list.sort()\n",
    "\n",
    "    img_75 = int(len(image_list)*0.75)\n",
    "    training_image_list = image_list[0:img_75]\n",
    "    random.shuffle(training_image_list)\n",
    "\n",
    "    return {\n",
    "        'image_list' : training_image_list,\n",
    "        'captions' : flower_captions,\n",
    "        'data_length' : len(training_image_list)\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pre_process_image(image_file, image_size):\n",
    "    _img = opencv.imread(image_file)\n",
    "    _img_scaled = opencv.resize(_img, (image_size, image_size))\n",
    "    if random.random() > 0.5:\n",
    "        _img_scaled = np.fliplr(_img_scaled)\n",
    "    return _img_scaled.astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_training_batch(batch_no, batch_size, image_size, z_dim, \n",
    "    caption_vector_length, split, data_dir, loaded_data = None):\n",
    "    real_images = np.zeros((batch_size, 64, 64, 3))\n",
    "    wrong_images = np.zeros((batch_size, 64, 64, 3))\n",
    "    captions = np.zeros((batch_size, caption_vector_length))\n",
    "\n",
    "    cnt = 0\n",
    "    image_files = []\n",
    "    for i in range(batch_no * batch_size, batch_no * batch_size + batch_size):\n",
    "        idx = i % len(loaded_data['image_list'])\n",
    "        image_file =  os.path.join(data_dir, 'flowers/jpg/'+loaded_data['image_list'][idx])\n",
    "        image_array = pre_process_image(image_file, image_size)\n",
    "        real_images[cnt,:,:,:] = image_array\n",
    "\n",
    "        # Improve this selection of wrong image\n",
    "        wrong_image_id = random.randint(0,len(loaded_data['image_list'])-1)\n",
    "        wrong_image_file =  os.path.join(data_dir, 'flowers/jpg/'+loaded_data['image_list'][wrong_image_id])\n",
    "        wrong_image_array = pre_process_image(wrong_image_file, image_size)\n",
    "        wrong_images[cnt, :,:,:] = wrong_image_array\n",
    "\n",
    "        random_caption = random.randint(0,4)\n",
    "        captions[cnt,:] = loaded_data['captions'][ loaded_data['image_list'][idx] ][ random_caption ][0:caption_vector_length]\n",
    "        image_files.append( image_file )\n",
    "        cnt += 1\n",
    "\n",
    "    z_noise = np.random.uniform(-1, 1, [batch_size, z_dim])\n",
    "    return real_images, wrong_images, captions, z_noise, image_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/s0d02zy/anaconda3/lib/python3.7/site-packages/tensorflow/python/util/tf_should_use.py:193: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n"
     ]
    }
   ],
   "source": [
    "d_optim = tf.train.AdamOptimizer(learning_rate, beta1 = beta1).minimize(loss['d_loss'], var_list=variables['d_vars'])\n",
    "g_optim = tf.train.AdamOptimizer(learning_rate, beta1 = beta1).minimize(loss['g_loss'], var_list=variables['g_vars'])\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "tf.initialize_all_variables().run()\n",
    "\n",
    "saver = tf.train.Saver()\n",
    "if resume_model:\n",
    "    saver.restore(sess, resume_model)\n",
    "\n",
    "loaded_data = load_training_data(data_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "********************************\n",
      "Epoch: 0  , Batch No: 0\n",
      "********************************\n",
      "First Discriminator Loss: \t 2.0663457\n",
      "Second Discriminator Loss: \t 0.35173687\n",
      "Third Discriminator Loss: \t 0.4354528\n",
      "Total Discriminator Loss: \t 2.8535352\n",
      "Generator Loss:  1.0315427\n",
      "\n",
      "********************************\n",
      "Epoch: 0  , Batch No: 1\n",
      "********************************\n",
      "First Discriminator Loss: \t 0.2915724\n",
      "Second Discriminator Loss: \t 4.2517543\n",
      "Third Discriminator Loss: \t 1.4265585\n",
      "Total Discriminator Loss: \t 5.9698853\n",
      "Generator Loss:  10.653404\n",
      "\n",
      "********************************\n",
      "Epoch: 0  , Batch No: 2\n",
      "********************************\n",
      "First Discriminator Loss: \t 5.8490763\n",
      "Second Discriminator Loss: \t 0.047663122\n",
      "Third Discriminator Loss: \t 0.00025780228\n",
      "Total Discriminator Loss: \t 5.8969975\n",
      "Generator Loss:  3.591416\n",
      "\n",
      "********************************\n",
      "Epoch: 0  , Batch No: 3\n",
      "********************************\n",
      "First Discriminator Loss: \t 3.4688506\n",
      "Second Discriminator Loss: \t 0.24122734\n",
      "Third Discriminator Loss: \t 0.25830555\n",
      "Total Discriminator Loss: \t 3.9683836\n",
      "Generator Loss:  0.058487296\n",
      "\n",
      "********************************\n",
      "Epoch: 0  , Batch No: 4\n",
      "********************************\n",
      "First Discriminator Loss: \t 0.5303961\n",
      "Second Discriminator Loss: \t 2.246564\n",
      "Third Discriminator Loss: \t 3.7802916\n",
      "Total Discriminator Loss: \t 6.5572515\n",
      "Generator Loss:  10.102921\n",
      "\n",
      "********************************\n",
      "Epoch: 0  , Batch No: 5\n",
      "********************************\n",
      "First Discriminator Loss: \t 3.3467946\n",
      "Second Discriminator Loss: \t 0.3653529\n",
      "Third Discriminator Loss: \t 0.000986618\n",
      "Total Discriminator Loss: \t 3.713134\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-94b1e133f7a5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     37\u001b[0m                 \u001b[0minput_tensors\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m't_wrong_image'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0mwrong_images\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m                 \u001b[0minput_tensors\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m't_real_caption'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0mcaption_vectors\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 39\u001b[0;31m                 \u001b[0minput_tensors\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m't_z'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0mz_noise\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     40\u001b[0m             })\n\u001b[1;32m     41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    948\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 950\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    951\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    952\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1171\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1172\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1173\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1174\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1175\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1348\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1349\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1350\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1351\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1354\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1355\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1356\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1357\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1358\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1339\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1340\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1341\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1343\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1427\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[1;32m   1428\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1429\u001b[0;31m         run_metadata)\n\u001b[0m\u001b[1;32m   1430\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1431\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for i in range(epochs):\n",
    "    batch_no = 0\n",
    "    while batch_no*batch_size < loaded_data['data_length']:\n",
    "        real_images, wrong_images, caption_vectors, z_noise, image_files = get_training_batch(batch_no, batch_size, \n",
    "            image_size, z_dim, caption_vector_length, 'train', data_dir, loaded_data)\n",
    "\n",
    "        # DISCR UPDATE\n",
    "        check_ts = [ checks['d_loss1'] , checks['d_loss2'], checks['d_loss3']]\n",
    "        _, d_loss, gen, d1, d2, d3 = sess.run([d_optim, loss['d_loss'], outputs['generator']] + check_ts,\n",
    "            feed_dict = {\n",
    "                input_tensors['t_real_image'] : real_images,\n",
    "                input_tensors['t_wrong_image'] : wrong_images,\n",
    "                input_tensors['t_real_caption'] : caption_vectors,\n",
    "                input_tensors['t_z'] : z_noise,\n",
    "            })\n",
    "        print(\"********************************\")\n",
    "        print(\"Epoch:\",i,\" , Batch No:\",batch_no)\n",
    "        print(\"********************************\")\n",
    "        print(\"First Discriminator Loss: \\t\", d1)\n",
    "        print(\"Second Discriminator Loss: \\t\", d2)\n",
    "        print(\"Third Discriminator Loss: \\t\", d3)\n",
    "        print(\"Total Discriminator Loss: \\t\", d_loss)\n",
    "\n",
    "        # GEN UPDATE\n",
    "        _, g_loss, gen = sess.run([g_optim, loss['g_loss'], outputs['generator']],\n",
    "            feed_dict = {\n",
    "                input_tensors['t_real_image'] : real_images,\n",
    "                input_tensors['t_wrong_image'] : wrong_images,\n",
    "                input_tensors['t_real_caption'] : caption_vectors,\n",
    "                input_tensors['t_z'] : z_noise,\n",
    "            })\n",
    "\n",
    "        # GEN UPDATE TWICE, to make sure d_loss does not go to 0\n",
    "        _, g_loss, gen = sess.run([g_optim, loss['g_loss'], outputs['generator']],\n",
    "            feed_dict = {\n",
    "                input_tensors['t_real_image'] : real_images,\n",
    "                input_tensors['t_wrong_image'] : wrong_images,\n",
    "                input_tensors['t_real_caption'] : caption_vectors,\n",
    "                input_tensors['t_z'] : z_noise,\n",
    "            })\n",
    "\n",
    "        print(\"Generator Loss: \", g_loss)\n",
    "        print(\"\")\n",
    "        batch_no += 1\n",
    "        if (batch_no % save_every) == 0:\n",
    "            print(\"Saving Model...\")\n",
    "            save_path = saver.save(sess, \"Data/Models/latest_model_{}_temp.ckpt\".format(data_set))\n",
    "    if i%5 == 0:\n",
    "        save_path = saver.save(sess, \"Data/Models/model_after_{}_epoch_{}.ckpt\".format(data_set, i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}