remove strong supervision

Author: Alex Barron

babi_input.py

@@ -141,7 +141,6 @@ def process_input(data_raw, floatX, word2vec, vocab, ivocab, embed_size, split_s
     inputs = []
     answers = []
     input_masks = []
-    relevant_labels = []
     for x in data_raw:
         if split_sentences:
             inp = x["C"].lower().split(' . ') 
@@ -197,9 +196,7 @@ def process_input(data_raw, floatX, word2vec, vocab, ivocab, embed_size, split_s
             else:
                 raise Exception("invalid input_mask_mode")
 
-        relevant_labels.append(x["S"])
-    
-    return inputs, questions, answers, input_masks, relevant_labels 
+    return inputs, questions, answers, input_masks
 
 def get_lens(inputs, split_sentences=False):
     lens = np.zeros((len(inputs)), dtype=int)
@@ -280,7 +277,7 @@ def load_babi(config, split_sentences=False):
     else:
         word_embedding = np.random.uniform(-config.embedding_init, config.embedding_init, (len(ivocab), config.embed_size))
 
-    inputs, questions, answers, input_masks, rel_labels = train_data if config.train_mode else test_data
+    inputs, questions, answers, input_masks = train_data if config.train_mode else test_data
 
     if split_sentences:
         input_lens, sen_lens, max_sen_len = get_sentence_lens(inputs)
@@ -307,17 +304,12 @@ def load_babi(config, split_sentences=False):
 
     answers = np.stack(answers)
 
-    rel_labels = np.array(rel_labels)
-
     if config.train_mode:
-        train = questions[:config.num_train], inputs[:config.num_train], q_lens[:config.num_train], input_lens[:config.num_train], input_masks[:config.num_train], answers[:config.num_train], rel_labels[:config.num_train] 
+        train = questions[:config.num_train], inputs[:config.num_train], q_lens[:config.num_train], input_lens[:config.num_train], input_masks[:config.num_train], answers[:config.num_train]
 
-        valid = questions[config.num_train:], inputs[config.num_train:], q_lens[config.num_train:], input_lens[config.num_train:], input_masks[config.num_train:], answers[config.num_train:], rel_labels[config.num_train:] 
-        return train, valid, word_embedding, max_q_len, max_input_len, max_mask_len, rel_labels.shape[1], len(vocab)
+        valid = questions[config.num_train:], inputs[config.num_train:], q_lens[config.num_train:], input_lens[config.num_train:], input_masks[config.num_train:], answers[config.num_train:]
+        return train, valid, word_embedding, max_q_len, max_input_len, max_mask_len, len(vocab)
 
     else:
-        test = questions, inputs, q_lens, input_lens, input_masks, answers, rel_labels
-        return test, word_embedding, max_q_len, max_input_len, max_mask_len, rel_labels.shape[1], len(vocab)
-
-
-    
+        test = questions, inputs, q_lens, input_lens, input_masks, answers
+        return test, word_embedding, max_q_len, max_input_len, max_mask_len, len(vocab)

dmn_plus.py

@@ -35,10 +35,6 @@ class Config(object):
     word2vec_init = False
     embedding_init = np.sqrt(3)
 
-    # set to zero with strong supervision to only train gates
-    strong_supervision = False
-    beta = 1
-
     # NOTE not currently used hence non-sensical anneal_threshold
     anneal_threshold = 1000
     anneal_by = 1.5
@@ -84,9 +80,9 @@ class DMN_PLUS(object):
     def load_data(self, debug=False):
         """Loads train/valid/test data and sentence encoding"""
         if self.config.train_mode:
-            self.train, self.valid, self.word_embedding, self.max_q_len, self.max_sentences, self.max_sen_len, self.num_supporting_facts, self.vocab_size = babi_input.load_babi(self.config, split_sentences=True)
+            self.train, self.valid, self.word_embedding, self.max_q_len, self.max_sentences, self.max_sen_len, self.vocab_size = babi_input.load_babi(self.config, split_sentences=True)
         else:
-            self.test, self.word_embedding, self.max_q_len, self.max_sentences, self.max_sen_len, self.num_supporting_facts, self.vocab_size = babi_input.load_babi(self.config, split_sentences=True)
+            self.test, self.word_embedding, self.max_q_len, self.max_sentences, self.max_sen_len, self.vocab_size = babi_input.load_babi(self.config, split_sentences=True)
         self.encoding = _position_encoding(self.max_sen_len, self.config.embed_size)
 
     def add_placeholders(self):
@@ -99,9 +95,6 @@ def add_placeholders(self):
 
         self.answer_placeholder = tf.placeholder(tf.int64, shape=(self.config.batch_size,))
 
-        # fact corresponding to answer. Useful for strong supervision
-        self.rel_label_placeholder = tf.placeholder(tf.int32, shape=(self.config.batch_size, self.num_supporting_facts))
-
         self.dropout_placeholder = tf.placeholder(tf.float32)
 
     def get_predictions(self, output):
@@ -111,14 +104,7 @@ def get_predictions(self, output):
 
     def add_loss_op(self, output):
         """Calculate loss"""
-        # optional strong supervision of attention with supporting facts
-        gate_loss = 0
-        if self.config.strong_supervision:
-            for i, att in enumerate(self.attentions):
-                labels = tf.gather(tf.transpose(self.rel_label_placeholder), 0)
-                gate_loss += tf.reduce_sum(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=att, labels=labels))
-
-        loss = self.config.beta*tf.reduce_sum(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=output, labels=self.answer_placeholder)) + gate_loss
+        loss = tf.reduce_sum(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=output, labels=self.answer_placeholder))
 
         # add l2 regularization for all variables except biases
         for v in tf.trainable_variables():
@@ -298,8 +284,8 @@ def run_epoch(self, session, data, num_epoch=0, train_writer=None, train_op=None
 
         # shuffle data
         p = np.random.permutation(len(data[0]))
-        qp, ip, ql, il, im, a, r = data
-        qp, ip, ql, il, im, a, r = qp[p], ip[p], ql[p], il[p], im[p], a[p], r[p] 
+        qp, ip, ql, il, im, a = data
+        qp, ip, ql, il, im, a = qp[p], ip[p], ql[p], il[p], im[p], a[p]
 
         for step in range(total_steps):
             index = range(step*config.batch_size,(step+1)*config.batch_size)
@@ -308,7 +294,6 @@ def run_epoch(self, session, data, num_epoch=0, train_writer=None, train_op=None
                   self.question_len_placeholder: ql[index],
                   self.input_len_placeholder: il[index],
                   self.answer_placeholder: a[index],
-                  self.rel_label_placeholder: r[index],
                   self.dropout_placeholder: dp}
             loss, pred, summary, _ = session.run(
               [self.calculate_loss, self.pred, self.merged, train_op], feed_dict=feed)