Fix #167 remove part’s tokens

Rostlab · Jan 28, 2017 · 28f3d3c · 28f3d3c
1 parent 8d2c504
commit 28f3d3c
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 6 deletions.
diff --git a/nalaf/preprocessing/parsers.py b/nalaf/preprocessing/parsers.py
@@ -99,8 +99,6 @@ def parse(self, dataset):
                         'is_root': False,
                     }
 
-                    part.tokens.append(nalaf_token)
-
                 for spacy_token in spacy_doc:
                     self._dependency_path(spacy_token, sent_index, part)
 
@@ -194,7 +192,6 @@ def parse(self, dataset):
                                         'dependency_to': [],
                                         'is_root': False,
                                         }
-                        part.tokens.append(tok)
 
                     for token in tokens:
                         tok = part.sentences[index][token.index-1]

diff --git a/nalaf/structures/data.py b/nalaf/structures/data.py
@@ -1049,11 +1049,9 @@ def __init__(self, text, is_abstract=True):
         self.is_abstract = is_abstract
         """whether the part is the abstract of the paper"""
 
+        # TODO this may be too relna-specific
         self.sentence_parse_trees = []
         """the parse trees for each sentence stored as a string."""
-        # TODO this may be too relna-specific
-
-        self.tokens = []
 
 
     def get_sentence_string_array(self):