flake8

Author: hpasapp

metrics/cluster_metrics.py

@@ -1,12 +1,9 @@
 """
 cluster metrics: precision, recall, )f1)
 """
-import math
-import random
 from collections import Counter
 
 import scipy.optimize
-import numpy as np
 import torch
 
 

model/__init__.py

@@ -1 +0,0 @@
-

model/decoder.py

@@ -1,9 +1,10 @@
 import torch as t
 import torch.nn as nn
-import torch.nn.functional as F
+
 
 class Decoder(nn.Module):
-    def __init__(self, latent_z_size, word_emb_size, word_vocab_size, decoder_rnn_size, decoder_num_layers, dropout=0.5):
+    def __init__(self, latent_z_size, word_emb_size, word_vocab_size, decoder_rnn_size,
+                 decoder_num_layers, dropout=0.5):
         super(Decoder, self).__init__()
         self.latent_z_size = latent_z_size
         self.word_vocab_size = word_vocab_size
@@ -27,7 +28,8 @@ def forward(self, decoder_input, latent_z):
         """
 
         [batch_size, seq_len, _] = decoder_input.size()
-        # decoder rnn is conditioned on context via additional bias = W_cond * z to every input token
+        # decoder rnn is conditioned on context via additional bias = W_cond * z to every input
+        # token
         latent_z = t.cat([latent_z] * seq_len, 1).view(batch_size, seq_len, self.latent_z_size)
         decoder_input = t.cat([decoder_input, latent_z], 2)
         rnn_out, _ = self.rnn(decoder_input)

model/encoder.py

@@ -1,6 +1,6 @@
 import torch
 import torch.nn as nn
-import torch.nn.functional as F
+
 
 class Encoder(nn.Module):
     def __init__(self, word_emb_size, encoder_rnn_size, encoder_num_layers, dropout=0.5):
@@ -28,7 +28,8 @@ def forward(self, encoder_input, lengths):
                 encoder_input, lengths, True)
         # Unfold rnn with zero initial state and get its final state from the last layer
         rnn_out, (_, final_state) = self.rnn(packed_words, None)
-        final_state = final_state.view(self.encoder_num_layers, 2, batch_size, self.encoder_rnn_size)[-1]
+        final_state = final_state.view(
+            self.encoder_num_layers, 2, batch_size, self.encoder_rnn_size)[-1]
         h_1, h_2 = final_state[0], final_state[1]
         final_state = torch.cat([h_1, h_2], 1)
         _, unperm_idx = perm_idx.sort(0)

model/multiview_encoders.py

@@ -8,9 +8,12 @@
 
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 
+
 class MultiviewEncoders(nn.Module):
 
-    def __init__(self, vocab_size, num_layers, embedding_size, lstm_hidden_size, word_dropout, dropout, start_idx=2, end_idx=3, pad_idx=0):
+    def __init__(
+            self, vocab_size, num_layers, embedding_size, lstm_hidden_size, word_dropout, dropout,
+            start_idx=2, end_idx=3, pad_idx=0):
         super().__init__()
         self.pad_idx = pad_idx
         self.start_idx = start_idx   # for RNN autoencoder training
@@ -24,6 +27,7 @@ def __init__(self, vocab_size, num_layers, embedding_size, lstm_hidden_size, wor
         self.crit = nn.CrossEntropyLoss()
 
         self.embedder = nn.Embedding(vocab_size, embedding_size)
+
         def create_rnn(embedding_size, bidirectional=True):
             return nn.LSTM(
                 embedding_size,
@@ -52,7 +56,9 @@ def get_encoder(self, encoder):
         }[encoder]
 
     @classmethod
-    def construct_from_embeddings(cls, embeddings, num_layers, embedding_size, lstm_hidden_size, word_dropout, dropout, vocab_size, start_idx=2, end_idx=3, pad_idx=0):
+    def construct_from_embeddings(
+            cls, embeddings, num_layers, embedding_size, lstm_hidden_size, word_dropout, dropout,
+            vocab_size, start_idx=2, end_idx=3, pad_idx=0):
         model = cls(
             num_layers=num_layers,
             embedding_size=embedding_size,
@@ -82,7 +88,8 @@ def decode(self, decoder_input, latent_z):
         embeddings = self.embedder(padded)
         embeddings = self.word_dropout(embeddings)
         [batch_size, seq_len, _] = embeddings.size()
-        # decoder rnn is conditioned on context via additional bias = W_cond * z to every input token
+        # decoder rnn is conditioned on context via additional bias = W_cond * z
+        # to every input token
         latent_z = t.cat([latent_z] * seq_len, 1).view(batch_size, seq_len, -1)
         embeddings = t.cat([embeddings, latent_z], 2)
         rnn = self.ae_decoder
@@ -130,7 +137,8 @@ def hierarchical_forward(self, input):
         _, (_, final_word_state) = self.view2_word_rnn(packed, None)
         _, unperm_idx = perm_idx.sort(0)
         final_word_state = final_word_state[:, unperm_idx]
-        final_word_state = final_word_state.view(self.num_layers, 2, batch_size*max_sent_len, self.lstm_hidden_size)[-1] \
+        final_word_state = final_word_state.view(
+            self.num_layers, 2, batch_size*max_sent_len, self.lstm_hidden_size)[-1] \
             .transpose(0, 1).contiguous() \
             .view(batch_size, max_sent_len, 2 * self.lstm_hidden_size)
 
@@ -140,7 +148,8 @@ def hierarchical_forward(self, input):
         _, (_, final_sent_state) = self.view2_sent_rnn(sent_packed, None)
         _, sent_unperm_idx = sent_perm_idx.sort(0)
         final_sent_state = final_sent_state[:, sent_unperm_idx]
-        final_sent_state = final_sent_state.view(self.num_layers, 2, batch_size, self.lstm_hidden_size)[-1] \
+        final_sent_state = final_sent_state.view(
+            self.num_layers, 2, batch_size, self.lstm_hidden_size)[-1] \
             .transpose(0, 1).contiguous() \
             .view(batch_size, 2 * self.lstm_hidden_size)
         return final_sent_state
@@ -169,13 +178,15 @@ def reconst_loss(self, gnd_utts, reconst):
         padded, lengths = pad_sentences(gnd_utts, pad_idx=self.pad_idx, rpad=self.end_idx)
         batch_size = len(lengths)
         crit = nn.CrossEntropyLoss()
-        loss += crit(reconst.view(batch_size * seq_len, vocab_size), padded.view(batch_size * seq_len))
+        loss += crit(
+            reconst.view(batch_size * seq_len, vocab_size), padded.view(batch_size * seq_len))
         _, argmax = reconst.max(dim=-1)
         correct = (argmax == padded)
         acc = correct.float().mean().item()
         return loss, acc
 
-def create_model_from_embeddings(glove_path, id_to_token, token_to_id):
+
+def from_embeddings(glove_path, id_to_token, token_to_id):
     vocab_size = len(token_to_id)
 
     # Load pre-trained GloVe vectors
@@ -184,7 +195,8 @@ def create_model_from_embeddings(glove_path, id_to_token, token_to_id):
     print('loading glove')
     for line in open(glove_path):
         parts = line.strip().split()
-        if len(parts) % 100 != 1: continue
+        if len(parts) % 100 != 1:
+            continue
         word = parts[0]
         if word not in token_to_id:
             continue
@@ -213,7 +225,8 @@ def create_model_from_embeddings(glove_path, id_to_token, token_to_id):
         vocab_size=vocab_size
     )
     model.to(device)
-    return  id_to_token, token_to_id, vocab_size, word_emb_size, model
+    return id_to_token, token_to_id, vocab_size, word_emb_size, model
+
 
 def load_model(model_path):
     with open(model_path, 'rb') as f:

model/utils.py

@@ -1,9 +1,12 @@
-import os, time, pprint
 import torch
 
+
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 
-def pad_sentences(sents, lpad=None, rpad=None, reverse=False, pad_idx=0, max_sent_len=100):
+
+def pad_sentences(
+        sents, lpad=None, rpad=None, reverse=False, pad_idx=0,
+        max_sent_len=100):
     sentences = []
     max_len = 0
     for i in range(len(sents)):
@@ -24,12 +27,15 @@ def pad_sentences(sents, lpad=None, rpad=None, reverse=False, pad_idx=0, max_sen
     for i in range(len(sentences)):
         lengths.append(len(sentences[i]))
         sentences[i] = sentences[i] + [pad_idx]*(max_len - len(sentences[i]))
-    return torch.LongTensor(sentences).to(device), torch.LongTensor(lengths).to(device)
+    return (torch.LongTensor(sentences).to(device),
+            torch.LongTensor(lengths).to(device))
+
 
 def pad_paragraphs(paras, pad_idx=0):
     sentences, lengths = [], []
     max_len = 0
-    for para in paras: max_len = max(max_len, len(para))
+    for para in paras:
+        max_len = max(max_len, len(para))
     for para in paras:
         for sent in para:
             sentences.append(sent[:])
@@ -39,12 +45,11 @@ def pad_paragraphs(paras, pad_idx=0):
     ret_sents, sent_lens = pad_sentences(sentences, pad_idx=pad_idx)
     return ret_sents, sent_lens, torch.LongTensor(lengths).to(device), max_len
 
+
 def euclidean_metric(a, b):
     n = a.shape[0]
     m = b.shape[0]
     a = a.unsqueeze(1).expand(n, m, -1)
     b = b.unsqueeze(0).expand(n, m, -1)
     logits = -((a - b)**2).sum(dim=2)
     return logits
-
-

pretrain.py

@@ -1,9 +1,9 @@
 import copy
-import torch
 import numpy as np
 
 from train import BATCH_SIZE, AE_BATCH_SIZE
 
+
 def pretrain_qt(dataset, perm_idx, expressions, train=True):
     """
     for each pair of utterances:
@@ -48,9 +48,11 @@ def pretrain_qt(dataset, perm_idx, expressions, train=True):
             optimizer.step()
     return total_loss, total_acc / len(qt_ex)
 
+
 def after_pretrain_qt(model):
     model.view2_word_rnn = copy.deepcopy(model.view1_word_rnn)
 
+
 def pretrain_ae(dataset, perm_idx, expressions, train=True):
     """
     uses v1 encoder to encode all utterances in both view1 and view2
@@ -84,6 +86,7 @@ def pretrain_ae(dataset, perm_idx, expressions, train=True):
     total_acc = total_acc / len(utterances)
     return total_loss, total_acc
 
+
 def after_pretrain_ae(model):
     # we'll use the view1 encoder for both view 1 and view 2
     model.view2_word_rnn = copy.deepcopy(model.view1_word_rnn)

proc_data.py

@@ -9,17 +9,18 @@
 START = "__START__"
 END = "__END__"
 
-class Dataset(Dataset):
 
-    def __init__(self, fname, view1_col='view1_col', view2_col='view2_col', label_col='cluster_id', tokenized=True, max_sent=10, train_ratio=.9):
+class Dataset(Dataset):
+    def __init__(self, fname, view1_col='view1_col', view2_col='view2_col', label_col='cluster_id',
+                 tokenized=True, max_sent=10, train_ratio=.9):
         """
         Args:
             fname: str, training data file
             view1_col: str, the column corresponding to view 1 input
             view2_col: str, the column corresponding to view 2 input
             label_col: str, the column corresponding to label
         """
-        
+
         def tokens_to_idices(tokens):
             token_idices = []
             for token in tokens:
@@ -28,7 +29,7 @@ def tokens_to_idices(tokens):
                     id_to_token.append(token)
                 token_idices.append(token_to_id[token])
             return token_idices
-                
+
         id_to_token = [PAD, UNK, START, END]
         token_to_id = {PAD: 0, UNK: 1, START: 2, END: 3}
         id_to_label = [UNK]
@@ -43,14 +44,15 @@ def tokens_to_idices(tokens):
             for row in reader:
                 view1_text, view2_text = row[view1_col], row[view2_col]
                 label = row[label_col]
-                if 'UNK' == label: label = UNK
+                if 'UNK' == label:
+                    label = UNK
                 if '<cust_' not in view1_text:
                     view2_sents = sent_tokenize(view2_text.lower())
                 else:
                     view2_sents = view2_text.split("> <")
                     for i in range(len(view2_sents) - 1):
-                         view2_sents[i] = view2_sents[i] + '>'
-                         view2_sents[i+1] = '<' + view2_sents[i]
+                        view2_sents[i] = view2_sents[i] + '>'
+                        view2_sents[i+1] = '<' + view2_sents[i]
                 v1_utts.append(view1_text)
                 if not tokenized:
                     v1_tokens = word_tokenize(view1_text.lower())