Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding thread pool executor to avoid blocking event loop #333

Merged
merged 5 commits into from
Jun 6, 2024
Merged

Adding thread pool executor to avoid blocking event loop #333

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
876581c
adding useful filtering terms
costero-e Jun 5, 2024
e95d397
changing conf file
costero-e Jun 6, 2024
b311fc5
thread pool implemented
costero-e Jun 6, 2024
5f02454
adding new_count and Makefile
costero-e Jun 6, 2024
baf0075
Merge branch 'master' into develop
costero-e Jun 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions beacon/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import asyncio
import socket


from beacon import conf, load_logger
from beacon.request import ontologies
from beacon.response import middlewares
Expand Down
6 changes: 3 additions & 3 deletions beacon/conf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,10 @@
#
# Beacon general info
#
beacon_id = 'org.ega-archive.gdi-spanish-beacon' # ID of the Beacon
beacon_name = 'GDI Spanish Beacon' # Name of the Beacon service
beacon_id = 'org.ega-archive.beacon-ri-demo' # ID of the Beacon
beacon_name = 'Beacon Reference Implementation demo' # Name of the Beacon service
api_version = api_version['api_version'] # Version of the Beacon implementation
uri = 'https://beacon-spain.ega-archive.org/api/'
uri = 'https://beacon-ri-demo.ega-archive.org/api/'

#
# Beacon granularity
Expand Down
6 changes: 4 additions & 2 deletions beacon/request/handlers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import yaml
import jwt
import requests

from concurrent.futures import ThreadPoolExecutor
from beacon.request import ontologies
from beacon.request.model import Granularity, RequestParams
from beacon.response.build_response import (
Expand Down Expand Up @@ -161,9 +161,11 @@ async def wrapper(request: Request):
datasets_count={}
#LOG.debug(response_datasets)
new_count=0
loop = asyncio.get_running_loop()
for dataset in response_datasets:
with ThreadPoolExecutor() as pool:
entity_schema, count, dataset_count, records = await loop.run_in_executor(pool, db_fn, entry_id, qparams, dataset)
#LOG.debug(dataset)
entity_schema, count, dataset_count, records = db_fn(entry_id, qparams, dataset)

if dataset_count != -1:
new_count+=dataset_count
Expand Down
16 changes: 16 additions & 0 deletions deploy/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,5 +23,21 @@ build:
docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:[email protected]:27017/beacon?authSource=admin" --file /tmp/biosamples.json --collection biosamples
docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:[email protected]:27017/beacon?authSource=admin" --file /tmp/runs.json --collection runs
docker exec rimongo mongoimport --jsonArray --uri "mongodb://root:[email protected]:27017/beacon?authSource=admin" --file /tmp/genomicVariations.json --collection genomicVariations
docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/analyses.json mongoega:tmp/analyses.json
docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/biosamples.json mongoega:tmp/biosamples.json
docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/cohorts.json mongoega:tmp/cohorts.json
docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/datasets.json mongoega:tmp/datasets.json
docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/genomicVariations.json mongoega:tmp/genomicVariations.json
docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/individuals.json mongoega:tmp/individuals.json
docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/runs.json mongoega:tmp/runs.json
docker cp /data/vault/bio-scratch/arnau/beacon/beacon2-ri-tools-v2_test_anot/tcga-coad/filtering_terms.json mongoega:tmp/filtering_terms.json
docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:[email protected]:27020/beacon?authSource=admin" --file /tmp/datasets.json --collection datasets
docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:[email protected]:27020/beacon?authSource=admin" --file /tmp/individuals.json --collection individuals
docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:[email protected]:27020/beacon?authSource=admin" --file /tmp/cohorts.json --collection cohorts
docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:[email protected]:27020/beacon?authSource=admin" --file /tmp/analyses.json --collection analyses
docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:[email protected]:27020/beacon?authSource=admin" --file /tmp/biosamples.json --collection biosamples
docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:[email protected]:27020/beacon?authSource=admin" --file /tmp/runs.json --collection runs
docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:[email protected]:27020/beacon?authSource=admin" --file /tmp/genomicVariations.json --collection genomicVariations
docker exec mongoega mongoimport --jsonArray --uri "mongodb://root:[email protected]:27020/beacon?authSource=admin" --file /tmp/filtering_terms.json --collection filtering_terms
docker exec beacon python beacon/reindex.py
docker exec beacon python beacon/db/extract_filtering_terms.py
7 changes: 4 additions & 3 deletions deploy/conf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,11 @@
#
# Beacon general info
#
beacon_id = 'org.ega-archive.gdi-spanish-beacon' # ID of the Beacon
beacon_name = 'GDI Spanish Beacon' # Name of the Beacon service
#
beacon_id = 'org.ega-archive.beacon-ri-demo' # ID of the Beacon
beacon_name = 'Beacon Reference Implementation demo' # Name of the Beacon service
api_version = api_version['api_version'] # Version of the Beacon implementation
uri = 'https://beacon-spain.ega-archive.org/api/'
uri = 'https://beacon-ri-demo.ega-archive.org/api/'

#
# Beacon granularity
Expand Down
23 changes: 23 additions & 0 deletions deploy/data/filtering_terms.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
[
{
"type": "alphanumeric",
"id": "libraryStrategy",
"scopes": [
"run"
]
},
{
"type": "alphanumeric",
"id": "molecularAttributes.geneIds",
"scopes": [
"genomicVariation"
]
},
{
"type": "alphanumeric",
"id": "diseases.ageOfOnset.iso8601duration",
"scopes": [
"individual"
]
}
]
Loading