From 3d8980ff6f53ff52e68dda31bf5572675eb38c80 Mon Sep 17 00:00:00 2001 From: costero-e Date: Tue, 7 Jan 2025 17:48:51 +0100 Subject: [PATCH 1/2] created compose for deployment in independent servers --- README.md | 38 ++++++++++------- docker-compose.remote.yml | 85 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 15 deletions(-) create mode 100644 docker-compose.remote.yml diff --git a/README.md b/README.md index 28c99c8..bdc37e0 100644 --- a/README.md +++ b/README.md @@ -44,28 +44,36 @@ If you are using a build with all the services in the same cluster, you can use: docker compose up -d --build ``` +#### Up the containers (with services in independent servers) + +If you wish to have each service (or some of them) in different servers, you will need to use the remote version of the docker compose file, and deploy the remote services you need by selecting them individually in the build. Example: + +```bash +docker-compose -f docker-compose.remote.yml up -d --build beaconprod +``` + +After that, you will need to configure the IPs in the different conf files to make them connect. Remember to bind the IP in mongo to 0.0.0.0 in case you are making an independent deployment of the beacon and the mongodb. + #### Load the data -To load the database we execute the following commands: +To load the database (mongo) just copy your files in the data folder. Then, locate yourself in the mongo folder: ```bash -docker cp /path/to/analyses.json mongoprod:tmp/analyses.json -docker cp /path/to/biosamples.json mongoprod:tmp/biosamples.json -docker cp /path/to/cohorts.json mongoprod:tmp/cohorts.json -docker cp /path/to/datasets.json mongoprod:tmp/datasets.json -docker cp /path/to/genomicVariations.json mongoprod:tmp/genomicVariations.json -docker cp /path/to/individuals.json mongoprod:tmp/individuals.json -docker cp /path/to/runs.json mongoprod:tmp/runs.json +cd beacon/connections/mongo ``` +And execute the next commands (only the ones you need): + ```bash -docker exec mongoprod mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/datasets.json --collection datasets -docker exec mongoprod mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/analyses.json --collection analyses -docker exec mongoprod mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/biosamples.json --collection biosamples -docker exec mongoprod mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/cohorts.json --collection cohorts -docker exec mongoprod mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/genomicVariations.json --collection genomicVariations -docker exec mongoprod mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/individuals.json --collection individuals -docker exec mongoprod mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /tmp/runs.json --collection runs + docker exec mongoprod mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /data/datasets.json --collection datasets + docker exec mongoprod mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /data/individuals.json --collection individuals + docker exec mongoprod mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /data/cohorts.json --collection cohorts + docker exec mongoprod mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /data/analyses.json --collection analyses + docker exec mongoprod mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /data/biosamples.json --collection biosamples + docker exec mongoprod mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /data/genomicVariations.json --collection genomicVariations + docker exec mongoprod mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /data/runs.json --collection runs + docker exec mongoprod mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /data/targets.json --collection targets + docker exec mongoprod mongoimport --jsonArray --uri "mongodb://root:example@127.0.0.1:27017/beacon?authSource=admin" --file /data/caseLevelData.json --collection caseLevelData ``` This loads the JSON files inside of the `data` folder into the MongoDB database container. Each time you import data you will have to create indexes for the queries to run smoothly. Please, check the next point about how to Create the indexes. diff --git a/docker-compose.remote.yml b/docker-compose.remote.yml new file mode 100644 index 0000000..cdbf29f --- /dev/null +++ b/docker-compose.remote.yml @@ -0,0 +1,85 @@ +version: '3.1' + +services: + beaconprod: + build: + context: . + container_name: beaconprod + ports: + - "5050:5050" + entrypoint: ['python','-m','beacon'] + volumes: + - ./beacon/logs/logs.log:/beacon/logs/logs.log + - ./beacon/conf/conf.py:/beacon/conf/conf.py + - ./beacon/permissions:/beacon/permissions + + idp: + #image: quay.io/keycloak/keycloak:12.0.0 + image: sleighzy/keycloak + hostname: idp + container_name: idp + environment: + # - KEYCLOAK_FRONTEND_URL=http://idp:8080/auth + # - KEYCLOAK_LOGLEVEL=DEBUG + # - ROOT_LOGLEVEL=DEBUG + - KEYCLOAK_USER=admin + - KEYCLOAK_PASSWORD=secret + - DB_USER=admin + - DB_PASSWORD=secret + - DB_VENDOR=postgres + - DB_ADDR=127.0.0.1 + - DB_PORT=5432 + - DB_DATABASE=keycloak + #- DB_SCHEMA=public + - KEYCLOAK_IMPORT=/tmp/beacon-realm.json -Dkeycloak.profile.feature.upload_scripts=enabled -Dkeycloak.profile.feature.token_exchange=enabled -Dkeycloak.profile.feature.admin_fine_grained_authz=enabled + - KEYCLOAK_HOSTNAME=localhost + #- KC_HOSTNAME_URL=https://beacon-network-test2.ega-archive.org + #- KC_HOSTNAME_ADMIN_URL=https://beacon-network-test2.ega-archive.org + #- KEYCLOAK_FRONTEND_URL=https://beacon-network-test2.ega-archive.org/auth/ + #- PROXY_ADDRESS_FORWARDING=true + volumes: + - ./beacon/auth/realms/beacon-realm.json:/tmp/beacon-realm.json + ports: + - "8080:8080" + - "9991:8443" + + idp-db: + image: postgres + hostname: idp-db + container_name: idp-db + ports: + - 5432:5432 + environment: + - POSTGRES_DB=keycloak + - POSTGRES_USER=admin + - POSTGRES_PASSWORD=secret + + db: + image: mongo:5 + command: --wiredTigerCacheSizeGB 4 + hostname: mongo + container_name: mongoprod + ports: + - 27017:27017 + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: example + MONGO_INITDB_DATABASE: beacon + volumes: + - ./beacon/connections/mongo/mongo-init/:/docker-entrypoint-initdb.d/:ro + - ./beacon/connections/mongo/data/:/data + - ./beacon/connections/mongo/data/db:/data/db + - ./beacon/connections/mongo/data/configdb:/data/configdb + - ./beacon/connections/mongo/data/caseLevelData:/data/caseLevelData + #command: --verbose + + beacon-ri-tools: + image: ghcr.io/ega-archive/beacon2-ri-tools-v2:latest + hostname: beacon-ri-tools + container_name: ri-tools + tty: true + volumes: + - ./ri-tools/output_docs:/usr/src/app/output_docs + - ./ri-tools/conf:/usr/src/app/conf + - ./ri-tools/files/vcf/files_to_read:/usr/src/app/files/vcf/files_to_read + - ./ri-tools/csv:/usr/src/app/csv From 8f1fd7ac2ddae41d4a692f3dbfe04269c73ceeef Mon Sep 17 00:00:00 2001 From: API version update Date: Tue, 7 Jan 2025 16:49:28 +0000 Subject: [PATCH 2/2] api version automatically generated --- beacon/conf/api_version.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon/conf/api_version.yml b/beacon/conf/api_version.yml index b86fd70..f1a43e1 100644 --- a/beacon/conf/api_version.yml +++ b/beacon/conf/api_version.yml @@ -1 +1 @@ -api_version: v2.0-9f42772 +api_version: v2.0-eeabc63