From 876581c77c51a92235cc1ffce957e25bc9ee46d9 Mon Sep 17 00:00:00 2001 From: Oriol Lopez-Doriga Date: Wed, 5 Jun 2024 09:55:40 +0200 Subject: [PATCH 1/4] adding useful filtering terms --- deploy/data/filtering_terms.json | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 deploy/data/filtering_terms.json diff --git a/deploy/data/filtering_terms.json b/deploy/data/filtering_terms.json new file mode 100644 index 00000000..23a750a5 --- /dev/null +++ b/deploy/data/filtering_terms.json @@ -0,0 +1,23 @@ +[ + { + "type": "alphanumeric", + "id": "libraryStrategy", + "scopes": [ + "run" + ] + }, + { + "type": "alphanumeric", + "id": "molecularAttributes.geneIds", + "scopes": [ + "genomicVariation" + ] + }, + { + "type": "alphanumeric", + "id": "diseases.ageOfOnset.iso8601duration", + "scopes": [ + "individual" + ] + } +] \ No newline at end of file From e95d3972d6e9e788294074c3b9999e62a9d4eaa1 Mon Sep 17 00:00:00 2001 From: Oriol Lopez-Doriga Date: Thu, 6 Jun 2024 09:12:47 +0200 Subject: [PATCH 2/4] changing conf file --- beacon/conf.py | 6 +++--- deploy/conf.py | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/beacon/conf.py b/beacon/conf.py index ecdbca7e..21a2dd9e 100644 --- a/beacon/conf.py +++ b/beacon/conf.py @@ -9,10 +9,10 @@ # # Beacon general info # -beacon_id = 'org.ega-archive.gdi-spanish-beacon' # ID of the Beacon -beacon_name = 'GDI Spanish Beacon' # Name of the Beacon service +beacon_id = 'org.ega-archive.beacon-ri-demo' # ID of the Beacon +beacon_name = 'Beacon Reference Implementation demo' # Name of the Beacon service api_version = api_version['api_version'] # Version of the Beacon implementation -uri = 'https://beacon-spain.ega-archive.org/api/' +uri = 'https://beacon-ri-demo.ega-archive.org/api/' # # Beacon granularity diff --git a/deploy/conf.py b/deploy/conf.py index 0179b30f..572fa4d4 100644 --- a/deploy/conf.py +++ b/deploy/conf.py @@ -9,10 +9,11 @@ # # Beacon general info # -beacon_id = 'org.ega-archive.gdi-spanish-beacon' # ID of the Beacon -beacon_name = 'GDI Spanish Beacon' # Name of the Beacon service +# +beacon_id = 'org.ega-archive.beacon-ri-demo' # ID of the Beacon +beacon_name = 'Beacon Reference Implementation demo' # Name of the Beacon service api_version = api_version['api_version'] # Version of the Beacon implementation -uri = 'https://beacon-spain.ega-archive.org/api/' +uri = 'https://beacon-ri-demo.ega-archive.org/api/' # # Beacon granularity From b311fc54a6f78ba9442915005d4c245e0555dfa3 Mon Sep 17 00:00:00 2001 From: Oriol Lopez-Doriga Date: Thu, 6 Jun 2024 09:16:53 +0200 Subject: [PATCH 3/4] thread pool implemented --- beacon/__main__.py | 1 + beacon/request/handlers.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/beacon/__main__.py b/beacon/__main__.py index 166a1894..ce79ac14 100644 --- a/beacon/__main__.py +++ b/beacon/__main__.py @@ -20,6 +20,7 @@ import asyncio import socket + from beacon import conf, load_logger from beacon.request import ontologies from beacon.response import middlewares diff --git a/beacon/request/handlers.py b/beacon/request/handlers.py index a9658d8f..ba6e9718 100644 --- a/beacon/request/handlers.py +++ b/beacon/request/handlers.py @@ -8,7 +8,7 @@ import yaml import jwt import requests - +from concurrent.futures import ThreadPoolExecutor from beacon.request import ontologies from beacon.request.model import Granularity, RequestParams from beacon.response.build_response import ( @@ -160,10 +160,11 @@ async def wrapper(request: Request): datasets_docs={} datasets_count={} #LOG.debug(response_datasets) - new_count=0 + loop = asyncio.get_running_loop() for dataset in response_datasets: + with ThreadPoolExecutor() as pool: + entity_schema, count, dataset_count, records = await loop.run_in_executor(pool, db_fn, entry_id, qparams, dataset) #LOG.debug(dataset) - entity_schema, count, dataset_count, records = db_fn(entry_id, qparams, dataset) if dataset_count != -1: new_count+=dataset_count From 5f0245433632bdaedd1601b393145a8839e9f79b Mon Sep 17 00:00:00 2001 From: Oriol Lopez-Doriga Date: Thu, 6 Jun 2024 09:33:19 +0200 Subject: [PATCH 4/4] adding new_count and Makefile --- beacon/request/handlers.py | 1 + deploy/Makefile | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/beacon/request/handlers.py b/beacon/request/handlers.py index ba6e9718..16b244cc 100644 --- a/beacon/request/handlers.py +++ b/beacon/request/handlers.py @@ -160,6 +160,7 @@ async def wrapper(request: Request): datasets_docs={} datasets_count={} #LOG.debug(response_datasets) + new_count=0 loop = asyncio.get_running_loop() for dataset in response_datasets: with ThreadPoolExecutor() as pool: diff --git a/deploy/Makefile b/deploy/Makefile index 6120a59b..88eb2bf9 100644 --- a/deploy/Makefile +++ b/deploy/Makefile @@ -23,5 +23,21 @@ build: docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/biosamples.json --collection biosamples docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/runs.json --collection runs docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/genomicVariations.json --collection genomicVariations + docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/analyses.json mongoega:tmp/analyses.json + docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/biosamples.json mongoega:tmp/biosamples.json + docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/cohorts.json mongoega:tmp/cohorts.json + docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/datasets.json mongoega:tmp/datasets.json + docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/genomicVariations.json mongoega:tmp/genomicVariations.json + docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/individuals.json mongoega:tmp/individuals.json + docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/runs.json mongoega:tmp/runs.json + docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/filtering_terms.json mongoega:tmp/filtering_terms.json + docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27020/beacon?authSource=admin" --file /tmp/datasets.json --collection datasets + docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27020/beacon?authSource=admin" --file /tmp/individuals.json --collection individuals + docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27020/beacon?authSource=admin" --file /tmp/cohorts.json --collection cohorts + docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27020/beacon?authSource=admin" --file /tmp/analyses.json --collection analyses + docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27020/beacon?authSource=admin" --file /tmp/biosamples.json --collection biosamples + docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27020/beacon?authSource=admin" --file /tmp/runs.json --collection runs + docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27020/beacon?authSource=admin" --file /tmp/genomicVariations.json --collection genomicVariations + docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27020/beacon?authSource=admin" --file /tmp/filtering_terms.json --collection filtering_terms docker exec beacon python beacon/reindex.py docker exec beacon python beacon/db/extract_filtering_terms.py \ No newline at end of file