Rare and genetic subset update pipeline

src/ontology/Makefile

@@ -85,7 +85,7 @@ all_imports: $(IMPORT_FILES)
 COMPONENTS = mondo-tags
 COMPONENT_FILES = $(foreach n, $(COMPONENTS), components/$(n).owl)
 
-SUBSETS = mondo-rare
+SUBSETS = mondo-rare mondo-clingen
 SUBSET_ROOTS = $(patsubst %, subsets/%, $(SUBSETS))
 SUBSET_FILES = $(foreach n,$(SUBSET_ROOTS), $(n).owl $(n).obo $(n).json $(n)_nodes.tsv $(n)_edges.tsv)
 
@@ -400,6 +400,15 @@ subsets/mondo-rare.owl: $(ONT)-base.owl $(TMPDIR)/hgnc_import.owl  tmp/rare-seed
 	$(ROBOT) annotate --input $@ --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o [email protected] && mv [email protected] $@
 .PRECIOUS: $(SUBSETDIR)/mondo-rare.owl
 
+tmp/clingen-seed.txt: $(SRC)
+	$(ROBOT) merge -i $< query --query ../sparql/signature/clingen-subset.sparql $@
+.PRECIOUS: tmp/rare-seed-entities.txt
+
+subsets/mondo-clingen.owl: $(ONT)-base.owl tmp/clingen-seed.txt | $(SUBSETDIR)
+	$(ROBOT) merge -i $< query --update ../sparql/update/clingen-labels.ru extract --method subset -T tmp/clingen-seed.txt --copy-ontology-annotations true --output $@ &&\
+	$(ROBOT) annotate --input $@ --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o [email protected] && mv [email protected] $@
+.PRECIOUS: $(SUBSETDIR)/mondo-rare.owl
+
 subsets/%.json: subsets/%.owl
 	$(ROBOT) convert -i $< -o $@
 

src/ontology/metadata/mondo.sssom.config.yml

@@ -70,7 +70,7 @@ curie_map:
   # ICD10: "http://purl.obolibrary.org/obo/mondo/mappings/unknown_prefix/ICD10/"
   ICD10EXP: "http://purl.obolibrary.org/obo/mondo/mappings/unknown_prefix/ICD10EXP/"
   DERMO: "http://purl.obolibrary.org/obo/mondo/mappings/unknown_prefix/DERMO/"
-  GARD: "http://purl.obolibrary.org/obo/mondo/mappings/unknown_prefix/GARD/"
+  GARD: "https://rarediseases.info.nih.gov/diseases/"
   # SNOMEDCT_US: "http://purl.obolibrary.org/obo/mondo/mappings/unknown_prefix/SNOMEDCT_US/"
   # MSH: "http://purl.obolibrary.org/obo/mondo/mappings/unknown_prefix/MSH/"
   GC_ID: "http://purl.obolibrary.org/obo/mondo/mappings/unknown_prefix/GC_ID/"

src/ontology/mondo-edit.obo

@@ -45,6 +45,7 @@ synonymtypedef: DUBIOUS "dubious synonym"
 synonymtypedef: EXCLUDE "Synonym to be removed from public release but maintained in edit version as record of external usage"
 synonymtypedef: MISSPELLING "A synonym that is recorded for consistency with another source but is a misspelling"
 synonymtypedef: NON_HUMAN "A synonym that is used for non-human animal variants of a disease"
+synonymtypedef: NORD_PREFERRED "NORD preferred"
 synonymtypedef: OMO:0003005 "UK spelling synonym"
 import: http://purl.obolibrary.org/obo/mondo/components/mondo-tags.owl
 import: http://purl.obolibrary.org/obo/mondo/imports/axioms.owl

src/ontology/mondo.Makefile

@@ -183,6 +183,9 @@ reports/mondo_analysis.md: $(QC_REPORTS)
 	# This is a hack to get rid of <style> tags that are rendered very ugly by github.
 	perl -0777 -i.original -pe 's#<style[^<]*<\/style>##igs' $@
 
+reports/all_synonym_data.tsv: #mondo.owl
+	$(ROBOT) query -i mondo.owl --query ../sparql/reports/all_synonym_data.sparql $@
+
 reports/mondo_analysis.pdf: $(QC_REPORTS)
 	jupyter nbconvert --execute --to pdf --TemplateExporter.exclude_input=True reports/mondo_analysis.ipynb
 
@@ -212,6 +215,251 @@ clean:
 		debug.owl roundtrip.obo test_nomerge sparql_test_* disjoint_sibs.obo \
 		reasoned-plus-equivalents.owl reasoned.owl tmp/*
 
+#############################################
+##### Mondo Subsets Pipeline ################
+#############################################
+
+tmp/%.template.owl: subsets/%.template.tsv $(SRC)
+	$(ROBOT) template --template $< convert -f ttl -o $@
+
+####################################
+##### Inferred #####################
+####################################
+
+# The inferred subset depends on the other ones, so we need to first remove the old subsets
+# Then add the gard, nord and orphanet subsets back in
+tmp/inferred-rare-subset.owl: $(SRC)
+	$(ROBOT) merge -i $(SRC) \
+		reason \
+		query --format ttl --query ../sparql/construct/construct-inferred-rare-subset.sparql $@
+
+update-inferred-subset:
+	$(MAKE) tmp/inferred-rare-subset.owl
+	grep -vE '^(subset: inferred_rare)' $(SRC) > tmp/mondo-edit.tmp || true
+	mv tmp/mondo-edit.tmp mondo-edit.obo
+	$(ROBOT) merge -i $(SRC) -i tmp/inferred-rare-subset.owl --collapse-import-closure false convert -f obo --check false -o $(SRC).obo
+	mv $(SRC).obo $(SRC) && make NORM && mv NORM $(SRC)
+
+####################################
+##### Orphanet #####################
+####################################
+
+tmp/ordo-subsets.robot.owl:
+	wget "https://raw.githubusercontent.com/monarch-initiative/mondo-ingest/main/src/ontology/external/ordo-subsets.robot.owl" -O $@
+
+.PHONY: update-ordo-subsets
+update-ordo-subsets:
+	$(MAKE) tmp/ordo-subsets.robot.owl -B
+	grep -vE '^(subset: ordo_group_of_disorders)' $(SRC) | grep -vE '^(subset: ordo_disorder)' | grep -vE '^(subset: ordo_subtype_of_a_disorder)' > tmp/mondo-edit.tmp || true
+	mv tmp/mondo-edit.tmp mondo-edit.obo
+	$(ROBOT) merge -i $(SRC) -i tmp/ordo-subsets.robot.owl --collapse-import-closure false convert -f obo --check false -o tmp/mondo-edit.tmp
+	mv tmp/mondo-edit.tmp mondo-edit.obo
+	make NORM && mv NORM $(SRC)
+
+tmp/orphanet-rare-subset.owl: $(SRC)
+	$(ROBOT) merge -i $(SRC) reason \
+		query --format ttl --query ../sparql/construct/construct-orphanet-rare-subset.sparql $@
+
+.PHONY: update-orphanet-subset
+update-orphanet-subset:
+	# We always need to update the ordo subsets (disease vs grouping vs subtype)
+	# Which is a prerequisite for the rare determination in ORDO
+	$(MAKE) tmp/orphanet-rare-subset.owl
+	grep -vE '^(subset: orphanet_rare)' $(SRC) > tmp/mondo-edit.tmp || true
+	mv tmp/mondo-edit.tmp mondo-edit.obo
+	$(ROBOT) merge -i $(SRC) -i tmp/orphanet-rare-subset.owl --collapse-import-closure false convert -f obo --check false -o $(SRC).obo
+	mv $(SRC).obo $(SRC) && make NORM && mv NORM $(SRC)
+
+
+####################################
+##### GARD #########################
+####################################
+
+subsets/gard-subset.template.tsv:
+	wget "https://docs.google.com/spreadsheets/d/e/2PACX-1vTGh0fO79utQPPmjSykHatfQMc6P_bnkqPfiqHJDdmqWJk7H1pSm8OkrJclJT9SiA/pub?gid=1163320745&single=true&output=tsv" -O $@
+
+# The complex part here is that we need to dynamically update the MONDO source code, i.e. 
+# MONDO:equivalentTo and MONDO:obsoleteEquivalentTo.
+tmp/gard-rare-subset.owl: $(SRC) subsets/gard-subset.template.tsv
+	$(ROBOT) template --template subsets/gard-subset.template.tsv convert -f ttl -o [email protected]
+	$(ROBOT) remove -i $< --select imports merge -i $@ query -f ttl --query ../sparql/construct/construct-equivalent-obsolete-gard.sparql [email protected]
+	$(ROBOT) merge -i [email protected] -i [email protected] convert -f ofn -o $@
+
+.PHONY: update-gard
+update-gard:
+	$(MAKE) tmp/gard-rare-subset.owl
+	grep -vE '^(xref: GARD:|subset: gard_rare)' $(SRC) > tmp/mondo-edit.tmp || true
+	mv tmp/mondo-edit.tmp mondo-edit.obo
+	$(ROBOT) merge -i $(SRC) -i tmp/gard-rare-subset.owl --collapse-import-closure false convert -f obo --check false -o $(SRC).obo
+	mv $(SRC).obo $(SRC) && make NORM && mv NORM $(SRC)
+
+####################################
+##### NANDO #########################
+####################################
+
+tmp/nando.template.owl: tmp/nando.template.tsv
+	wget https://raw.githubusercontent.com/monarch-initiative/mondo-ingest/main/src/ontology/external/nando-mappings.robot.owl -O $@
+
+.PHONY: update-nando
+update-nando:
+	make tmp/nando.template.owl -B
+	grep -vE '^(xref: NANDO:|subset: nando)' $(SRC) > tmp/mondo-edit.tmp || true
+	mv tmp/mondo-edit.tmp mondo-edit.obo
+	$(ROBOT) merge -i $(SRC) -i tmp/nando.template.owl --collapse-import-closure false convert -f obo --check false -o $(SRC).obo
+	mv $(SRC).obo $(SRC) && make NORM && mv NORM $(SRC)
+
+
+####################################
+##### NORD #########################
+####################################
+
+tmp/nord.template.owl:
+	wget "https://raw.githubusercontent.com/monarch-initiative/mondo-ingest/master/src/ontology/external/nord.robot.owl" -O $@
+
+.PHONY: update-nord
+update-nord:
+	make tmp/nord.template.owl -B
+	grep -vE '^(xref: NORD:|subset: nord_rare)' $(SRC) > tmp/mondo-edit.tmp || true
+	mv tmp/mondo-edit.tmp mondo-edit.obo
+	$(ROBOT) merge -i $(SRC) -i tmp/nord.template.owl --collapse-import-closure false convert -f obo --check false -o $(SRC).obo
+	mv $(SRC).obo $(SRC) && make NORM && mv NORM $(SRC)
+
+
+subsets/mondo-rare.kgx.tsv:
+	kgx transform --input-format obojson \
+                  --output subsets/mondo-rare.kgx \
+                  --output-format tsv \
+                  --knowledge-sources aggregator_knowledge_source "mondo,mondo disease ontology" \
+                  subsets/mondo-rare.json
+
+####################################
+##### CLINGEN ######################
+####################################
+
+subsets/clingen.template.tsv:
+	wget "https://docs.google.com/spreadsheets/d/e/2PACX-1vRiYDV1n1nDuJOgnlFx6DsYGyIGlbgI1HeDzI740OgmOKYy2RCCyBqLHiBh-IMadYXjVglsxDPypArh/pub?gid=637121472&single=true&output=tsv" -O $@
+
+.PHONY: update-clingen
+update-clingen:
+	$(MAKE) tmp/clingen.template.owl
+	grep -vE '^(xref: CGGV:|xref: CGGCIEX:|subset: clingen)' mondo-edit.obo > tmp/mondo-edit.tmp
+	#sed -i 's/EXACT CLINGEN_LABEL/EXACT/g' tmp/mondo-edit.tmp || true
+	mv tmp/mondo-edit.tmp mondo-edit.obo
+	$(ROBOT) merge -i $(SRC) -i tmp/clingen.template.owl --collapse-import-closure false convert -f obo --check false -o $(SRC).obo
+	mv $(SRC).obo $(SRC) && make NORM && mv NORM $(SRC)
+
+####################################
+##### RARE #########################
+####################################
+
+tmp/rare-subset.owl: $(SRC)
+	$(ROBOT) merge -i $(SRC) \
+		query --format ttl --query ../sparql/construct/construct-rare-subset.sparql $@
+
+.PHONY: update-rare-subset
+update-rare-subset:
+	$(MAKE) tmp/rare-subset.owl
+	grep -vE '^(subset: rare)$$' $(SRC) > tmp/mondo-edit.tmp || true
+	mv tmp/mondo-edit.tmp mondo-edit.obo
+	$(ROBOT) merge -i $(SRC) -i tmp/rare-subset.owl --collapse-import-closure false convert -f obo --check false -o $(SRC).obo
+	mv $(SRC).obo $(SRC) && make NORM && mv NORM $(SRC)
+
+####################################
+##### EFO ##########################
+####################################
+
+
+subsets/mondo-efo.template.tsv:
+	wget "https://raw.githubusercontent.com/EBISPOT/efo/master/src/ontology/reports/mondo-efo.robot.tsv" -O $@
+
+subsets/mondo-otar-subset.template.tsv:
+	wget "https://raw.githubusercontent.com/EBISPOT/efo/master/src/ontology/reports/mondo-otar-subset.robot.tsv" -O $@
+
+.PHONY: update-efo-subset
+update-efo-subset:
+	$(MAKE) tmp/mondo-otar-subset.template.owl tmp/mondo-efo.template.owl tmp/efo-proxy-merges.template.owl
+	grep -vE '^(xref: EFO:|subset: otar)' $(SRC) > tmp/mondo-edit.tmp || true
+	mv tmp/mondo-edit.tmp mondo-edit.obo
+	$(ROBOT) merge -i $(SRC) -i tmp/mondo-otar-subset.template.owl -i tmp/mondo-efo.template.owl -i tmp/efo-proxy-merges.template.owl --collapse-import-closure false \
+		query --use-graphs false --update ../sparql/update/update-equivalent-obsolete.ru \
+		convert -f obo --check false -o $(SRC).obo
+	mv $(SRC).obo $(SRC) && make NORM && mv NORM $(SRC)
+
+
+####################################
+##### MedGen #######################
+####################################
+
+
+tmp/mondo-medgen.template.tsv:
+	wget "https://github.com/monarch-initiative/medgen/releases/download/2024-06-09/medgen-xrefs.robot.template.tsv" -O $@
+
+tmp/mondo-medgen.template.owl: tmp/mondo-medgen.template.tsv
+	$(ROBOT) template --prefix "orcid: https://orcid.org/" --template $< convert -f ofn -o $@
+
+.PHONY: update-medgen
+update-medgen:
+	$(MAKE) tmp/mondo-medgen.template.owl
+	grep -vE '^(xref: UMLS:|xref: MEDGEN:|subset: medgen)' $(SRC) > tmp/mondo-edit.tmp || true
+	mv tmp/mondo-edit.tmp mondo-edit.obo
+	$(ROBOT) merge -i $(SRC) -i tmp/mondo-medgen.template.owl --collapse-import-closure false \
+		query --use-graphs false --update ../sparql/update/update-equivalent-obsolete.ru \
+		convert -f obo --check false -o $(SRC).obo
+	mv $(SRC).obo $(SRC) && make NORM && mv NORM $(SRC) 
+
+
+##########################################
+###### Update all external content #######
+##########################################
+
+update-rare-disease-subset:
+	$(MAKE) subset-metrics -B && cp tmp/subset-metrics.tsv tmp/subset-metrics-before.tsv
+	$(MAKE) update-orphanet-subset -B
+	$(MAKE) update-gard -B
+	$(MAKE) update-nord -B
+	$(MAKE) update-inferred-subset -B
+	$(MAKE) update-rare-subset -B
+	$(MAKE) subset-metrics -B && cp tmp/subset-metrics.tsv tmp/subset-metrics-after.tsv
+	@echo "Subset metrics before..."
+	cat tmp/subset-metrics-before.tsv
+	@echo "Subset metrics after..."
+	cat tmp/subset-metrics-after.tsv
+
+clingen-otar-ordo-content:
+	$(MAKE) subset-metrics -B && cp tmp/subset-metrics.tsv tmp/subset-metrics-before.tsv
+	$(MAKE) update-efo-subset -B
+	$(MAKE) update-clingen -B
+	$(MAKE) update-ordo-subsets -B
+	$(MAKE) update-nando -B
+	$(MAKE)  subset-metrics -B && cp tmp/subset-metrics.tsv tmp/subset-metrics-after.tsv
+	@echo "Subset metrics before..."
+	cat tmp/subset-metrics-before.tsv
+	@echo "Subset metrics after..."
+	cat tmp/subset-metrics-after.tsv
+
+update-external-content:
+	$(MAKE) subset-metrics -B && cp tmp/subset-metrics.tsv tmp/subset-metrics-before.tsv
+	$(MAKE) update-efo-subset -B
+	#$(MAKE) update-clingen -B
+	#$(MAKE) update-nando -B
+	#$(MAKE) update-nord -B
+	#$(MAKE) update-gard -B
+	$(MAKE) update-orphanet-subset -B
+	$(MAKE) update-inferred-subset -B
+	$(MAKE) update-rare-subset -B
+	$(MAKE)  subset-metrics -B && cp tmp/subset-metrics.tsv tmp/subset-metrics-after.tsv
+	@echo "Subset metrics before..."
+	cat tmp/subset-metrics-before.tsv
+	@echo "Subset metrics after..."
+	cat tmp/subset-metrics-after.tsv
+
+.PHONY: subset-metrics
+subset-metrics:
+	$(ROBOT) query -f tsv -i $(SRC) --query $(SPARQLDIR)/reports/count-subsets.sparql tmp/[email protected]
+
+##########################################
+##### RARE REPORT ########################
+##########################################
 
 reports/new-rare-diseases.txt: $(ONT)-base.owl
 	$(ROBOT) query -i $(ONT)-base.owl --query ../sparql/signature/rare-subset.sparql $@
@@ -1105,6 +1353,15 @@ mondo_obo:
 tmp/mondo-ingest.owl:
 	curl https://github.com/monarch-initiative/mondo-ingest/releases/latest/download/mondo-ingest.owl -L --output $@
 
+tmp/mondo_paper.owl: mondo.owl
+	cp $< $@
+
+tmp/mondo_paper.db: tmp/mondo_paper.owl
+	@rm -f .template.db
+	@rm -f .template.db.tmp
+	RUST_BACKTRACE=full semsql make $@ -P config/prefixes.csv
+	@rm -f .template.db
+	@rm -f .template.db.tmp
 
 tmp/mondo-ingest.db: tmp/mondo-ingest.owl
 	@rm -f .template.db
@@ -1134,6 +1391,11 @@ mondo-harrisons-view.owl: mondo.owl tmp/harrisons_seed.txt
 	$(ROBOT) remove -i $< -T tmp/harrisons_seed.txt --select complement --select classes --select "MONDO:*" \
 	annotate -V $(ONTBASE)/releases/`date +%Y-%m-%d`/$@ annotate --ontology-iri $(ONTBASE)/$@ -o $@
 
+qdiff:
+	wget "http://purl.obolibrary.org/obo/mondo.obo" -O tmp/m.obo
+	wget "http://purl.obolibrary.org/obo/mondo/mondo-base.obo" -O tmp/mb.obo
+	robot diff --left tmp/m.obo --right tmp/mb.obo -o tmp/qdiff.txt
+	robot diff --left tmp/m.obo --right tmp/mb.obo -f markdown -o tmp/qdiff.md
 
 ######################################
 ### Mondo managing major use ids #####
@@ -1227,13 +1489,6 @@ americanize: $(SRC) tmp/british_english_dictionary.csv
 	python ../scripts/clean-british-english.py $^
 
 
-.PHONY: update-gard-mappings
-update-gard-mappings:
-	grep -v '^xref: GARD:' mondo-edit.obo > TT || true
-	mv TT mondo-edit.obo
-	# make NORM
-	# mv NORM $(SRC)
-
 
 #######################################
 ### New Pattern merge pipeline ########
@@ -1282,7 +1537,7 @@ config/exclusion_reasons.tsv:
 
 all: config/exclusion_reasons.tsv
 
-##################################
+#################################
 ##### Scheduled GH Actions #######
 ##################################