Merge pull request #333 from EGA-archive/develop

Adding thread pool executor to avoid blocking event loop

beacon/__main__.py

@@ -20,6 +20,7 @@
 import asyncio
 import socket
 
+
 from beacon import conf, load_logger
 from beacon.request import ontologies
 from beacon.response import middlewares

beacon/conf.py

@@ -9,10 +9,10 @@
 #
 # Beacon general info
 #
-beacon_id = 'org.ega-archive.gdi-spanish-beacon'  # ID of the Beacon
-beacon_name = 'GDI Spanish Beacon'  # Name of the Beacon service
+beacon_id = 'org.ega-archive.beacon-ri-demo'  # ID of the Beacon
+beacon_name = 'Beacon Reference Implementation demo'  # Name of the Beacon service
 api_version = api_version['api_version'] # Version of the Beacon implementation
-uri = 'https://beacon-spain.ega-archive.org/api/'
+uri = 'https://beacon-ri-demo.ega-archive.org/api/'
 
 #
 # Beacon granularity

beacon/request/handlers.py

@@ -8,7 +8,7 @@
 import yaml
 import jwt
 import requests
-
+from concurrent.futures import ThreadPoolExecutor
 from beacon.request import ontologies
 from beacon.request.model import Granularity, RequestParams
 from beacon.response.build_response import (
@@ -161,9 +161,11 @@ async def wrapper(request: Request):
             datasets_count={}
             #LOG.debug(response_datasets)
             new_count=0
+            loop = asyncio.get_running_loop()
             for dataset in response_datasets:
+                with ThreadPoolExecutor() as pool:
+                    entity_schema, count, dataset_count, records = await loop.run_in_executor(pool, db_fn, entry_id, qparams, dataset)
                 #LOG.debug(dataset)
-                entity_schema, count, dataset_count, records = db_fn(entry_id, qparams, dataset)
 
                 if dataset_count != -1:
                     new_count+=dataset_count

deploy/Makefile

@@ -23,5 +23,21 @@ build:
 	docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:[email protected]:27017/beacon?authSource=admin" --file /tmp/biosamples.json --collection biosamples
 	docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:[email protected]:27017/beacon?authSource=admin" --file /tmp/runs.json --collection runs
 	docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:[email protected]:27017/beacon?authSource=admin" --file /tmp/genomicVariations.json --collection genomicVariations
+	docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/analyses.json mongoega:tmp/analyses.json
+	docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/biosamples.json mongoega:tmp/biosamples.json
+	docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/cohorts.json mongoega:tmp/cohorts.json
+	docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/datasets.json mongoega:tmp/datasets.json
+	docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/genomicVariations.json mongoega:tmp/genomicVariations.json
+	docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/individuals.json mongoega:tmp/individuals.json
+	docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/runs.json mongoega:tmp/runs.json
+	docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/filtering_terms.json mongoega:tmp/filtering_terms.json
+	docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:[email protected]:27020/beacon?authSource=admin" --file /tmp/datasets.json --collection datasets
+	docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:[email protected]:27020/beacon?authSource=admin" --file /tmp/individuals.json --collection individuals
+	docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:[email protected]:27020/beacon?authSource=admin" --file /tmp/cohorts.json --collection cohorts
+	docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:[email protected]:27020/beacon?authSource=admin" --file /tmp/analyses.json --collection analyses
+	docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:[email protected]:27020/beacon?authSource=admin" --file /tmp/biosamples.json --collection biosamples
+	docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:[email protected]:27020/beacon?authSource=admin" --file /tmp/runs.json --collection runs
+	docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:[email protected]:27020/beacon?authSource=admin" --file /tmp/genomicVariations.json --collection genomicVariations
+	docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:[email protected]:27020/beacon?authSource=admin" --file /tmp/filtering_terms.json --collection filtering_terms
 	docker exec beacon python beacon/reindex.py
 	docker exec beacon python beacon/db/extract_filtering_terms.py

deploy/conf.py

@@ -9,10 +9,11 @@
 #
 # Beacon general info
 #
-beacon_id = 'org.ega-archive.gdi-spanish-beacon'  # ID of the Beacon
-beacon_name = 'GDI Spanish Beacon'  # Name of the Beacon service
+#
+beacon_id = 'org.ega-archive.beacon-ri-demo'  # ID of the Beacon
+beacon_name = 'Beacon Reference Implementation demo'  # Name of the Beacon service
 api_version = api_version['api_version'] # Version of the Beacon implementation
-uri = 'https://beacon-spain.ega-archive.org/api/'
+uri = 'https://beacon-ri-demo.ega-archive.org/api/'
 
 #
 # Beacon granularity

deploy/data/filtering_terms.json

@@ -0,0 +1,23 @@
+[
+    {
+        "type": "alphanumeric",
+        "id": "libraryStrategy",
+        "scopes": [
+            "run"
+        ]
+    },
+    {
+        "type": "alphanumeric",
+        "id": "molecularAttributes.geneIds",
+        "scopes": [
+            "genomicVariation"
+        ]
+    },
+    {
+        "type": "alphanumeric",
+        "id": "diseases.ageOfOnset.iso8601duration",
+        "scopes": [
+            "individual"
+        ]
+    }
+]