Merge pull request #6 from wellcometrust/feature/ivyleavedtoadflax/add_parsing

Prepare package for training parsing models

Author: ivyleavedtoadflax

.gitignore

@@ -9,4 +9,5 @@ deep_reference_parser/models/
 *.whl
 embeddings/
 models/
-.tox/
+.tox/
+*__pycache__/

Makefile

@@ -78,9 +78,12 @@ $(artefacts):
 models: $(artefacts)
 
 
-datasets = data/2019.12.0_train.tsv \
-           data/2019.12.0_test.tsv \
-           data/2019.12.0_valid.tsv
+datasets = data/splitting/2019.12.0_splitting_train.tsv \
+           data/splitting/2019.12.0_splitting_test.tsv \
+           data/splitting/2019.12.0_splitting_valid.tsv \
+		   data/splitting/2020.2.0_parsing_train.tsv \
+           data/splitting/2020.2.0_parsing_test.tsv \
+           data/splitting/2020.2.0_parsing_valid.tsv
 
 
 rodrigues_datasets = data/rodrigues/clean_train.txt \
@@ -90,7 +93,7 @@ rodrigues_datasets = data/rodrigues/clean_train.txt \
 RODRIGUES_DATA_URL =  https://github.com/dhlab-epfl/LinkedBooksDeepReferenceParsing/raw/master/dataset/
 
 $(datasets): 
-	@ mkdir -p data
+	@ mkdir -p $(@D)
 	curl -s $(S3_BUCKET_HTTP)/$@ --output $@
 
 $(rodrigues_datasets): 

deep_reference_parser/__version__.py

@@ -5,4 +5,4 @@
 __author__ = "Wellcome Trust DataLabs Team"
 __author_email__ = "[email protected]"
 __license__ = "MIT"
-__model_version__ = "2019.12.0"
+__model_version__ = "2019.12.0_splitting"

…p_reference_parser/configs/2019.12.0.ini …e_parser/configs/2019.12.0_splitting.inideep_reference_parser/configs/2019.12.0.ini renamed to deep_reference_parser/configs/2019.12.0_splitting.ini deep_reference_parser/configs/2019.12.0.ini renamed to deep_reference_parser/configs/2019.12.0_splitting.ini

@@ -1,5 +1,5 @@
 [DEFAULT]
-version = 2019.12.0
+version = 2019.12.0_splitting
 
 [data]
 test_proportion = 0.25
@@ -8,13 +8,13 @@ data_path = data/
 respect_line_endings = 0
 respect_doc_endings = 1
 line_limit = 250
-policy_train = data/2019.12.0_train.tsv
-policy_test = data/2019.12.0_test.tsv
-policy_valid = data/2019.12.0_valid.tsv
+policy_train = data/splitting/2019.12.0_splitting_train.tsv
+policy_test = data/splitting/2019.12.0_splitting_test.tsv
+policy_valid = data/splitting/2019.12.0_splitting_valid.tsv
 s3_slug = https://datalabs-public.s3.eu-west-2.amazonaws.com/deep_reference_parser/
 
 [build]
-output_path = models/2019.12.0/
+output_path = models/splitting/2019.12.0_splitting/
 output = crf
 word_embeddings = embeddings/2020.1.1-wellcome-embeddings-300.txt
 pretrained_embedding = 0

deep_reference_parser/prodigy/__init__.py

@@ -1,2 +1,7 @@
+from .numbered_reference_annotator import (NumberedReferenceAnnotator,
+                                           annotate_numbered_references)
+from .prodigy_to_tsv import TokenLabelPairs, prodigy_to_tsv
+from .reach_to_prodigy import ReachToProdigy, reach_to_prodigy
+from .reference_to_token_annotations import (TokenTagger,
+                                             reference_to_token_annotations)
 from .spacy_doc_to_prodigy import SpacyDocToProdigy
-from .reference_to_token_annotations import TokenTagger