From 28f3d3cc3585fdea46bf2f0d0a2229e9d97e027f Mon Sep 17 00:00:00 2001 From: Juan Miguel Cejuela Date: Sat, 28 Jan 2017 18:46:05 +0100 Subject: [PATCH] =?UTF-8?q?Fix=20#167=20remove=20part=E2=80=99s=20tokens?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nalaf/preprocessing/parsers.py | 3 --- nalaf/structures/data.py | 4 +--- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/nalaf/preprocessing/parsers.py b/nalaf/preprocessing/parsers.py index 0d6cc9a7..c7c428a1 100644 --- a/nalaf/preprocessing/parsers.py +++ b/nalaf/preprocessing/parsers.py @@ -99,8 +99,6 @@ def parse(self, dataset): 'is_root': False, } - part.tokens.append(nalaf_token) - for spacy_token in spacy_doc: self._dependency_path(spacy_token, sent_index, part) @@ -194,7 +192,6 @@ def parse(self, dataset): 'dependency_to': [], 'is_root': False, } - part.tokens.append(tok) for token in tokens: tok = part.sentences[index][token.index-1] diff --git a/nalaf/structures/data.py b/nalaf/structures/data.py index 585036fe..b63f73fb 100644 --- a/nalaf/structures/data.py +++ b/nalaf/structures/data.py @@ -1049,11 +1049,9 @@ def __init__(self, text, is_abstract=True): self.is_abstract = is_abstract """whether the part is the abstract of the paper""" + # TODO this may be too relna-specific self.sentence_parse_trees = [] """the parse trees for each sentence stored as a string.""" - # TODO this may be too relna-specific - - self.tokens = [] def get_sentence_string_array(self):