diff --git a/.github/workflows/CI-full.yml b/.github/workflows/CI-full.yml index f6f4338a6e..af845565cc 100644 --- a/.github/workflows/CI-full.yml +++ b/.github/workflows/CI-full.yml @@ -12,7 +12,7 @@ jobs: build: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Get the VCell version from tags id: version run: echo ::set-output name=tag::$(echo ${GITHUB_REF:10}) @@ -23,7 +23,7 @@ jobs: echo "VCELL_DEPLOY_REMOTE_DIR=/share/apps/vcell3/deployed_github" >> $GITHUB_ENV echo "VCELL_MANAGER_NODE=vcellapi-beta.cam.uchc.edu" >> $GITHUB_ENV - name: setup ssh-agent - uses: webfactory/ssh-agent@v0.5.3 + uses: webfactory/ssh-agent@v0.8.0 with: ssh-private-key: ${{ secrets.VC_KEY }} - name: get installer secrets diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml index 6447658bec..17a20f6bac 100644 --- a/.github/workflows/NightlyBMDB_CLI.yml +++ b/.github/workflows/NightlyBMDB_CLI.yml @@ -163,7 +163,7 @@ jobs: fi - name: Upload failures - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: set_${{ matrix.sets }}.report path: ${{ github.workspace }}/errors.report @@ -241,7 +241,7 @@ jobs: fi - name: Upload failures - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: set_${{ matrix.sets }}.report path: ${{ github.workspace }}/errors.report diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 6065a10663..b743da06a9 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -160,7 +160,7 @@ jobs: if: ${{ github.event_name == 'release' }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python uses: actions/setup-python@v4 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 33c23d49dc..5dc63e514a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -38,7 +38,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: setup java 17 with maven cache uses: actions/setup-java@v3 diff --git a/.github/workflows/create_bs_singularity.yml b/.github/workflows/create_bs_singularity.yml index 5ddf63729a..27cad0ff85 100644 --- a/.github/workflows/create_bs_singularity.yml +++ b/.github/workflows/create_bs_singularity.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.event.inputs.commit_sha }} diff --git a/.github/workflows/debug.yml b/.github/workflows/debug.yml index 545560f1f5..0da989b72f 100644 --- a/.github/workflows/debug.yml +++ b/.github/workflows/debug.yml @@ -11,7 +11,7 @@ jobs: build: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: insert version run: | set -ux diff --git a/.github/workflows/objective-c-xcode-debug.yml b/.github/workflows/objective-c-xcode-debug.yml index 2e0c64a247..15f4a653fe 100644 --- a/.github/workflows/objective-c-xcode-debug.yml +++ b/.github/workflows/objective-c-xcode-debug.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set Default Scheme run: | scheme_list=$(xcodebuild -list -json | tr -d "\n") diff --git a/.github/workflows/site_deploy.yml b/.github/workflows/site_deploy.yml index 879c207485..62d1518bad 100644 --- a/.github/workflows/site_deploy.yml +++ b/.github/workflows/site_deploy.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: checkout tag - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: ${{ github.event.inputs.vcell_version }}.${{ github.event.inputs.vcell_build }} - name: setup global environment variables @@ -50,7 +50,7 @@ jobs: echo "VCELL_INSTALLER_REMOTE_DIR="/share/apps/vcell3/apache_webroot/htdocs/webstart/Test"" >> $GITHUB_ENV;\ else echo "Unknown site name ${{ github.event.inputs.vcell_site }}"; exit 1; fi; fi; fi - name: setup ssh-agent - uses: webfactory/ssh-agent@v0.5.3 + uses: webfactory/ssh-agent@v0.8.0 with: ssh-private-key: ${{ secrets.VC_KEY }} - name: get installer secrets @@ -77,7 +77,7 @@ jobs: scp docker/swarm/$VCELL_CONFIG_FILE_NAME ${{ secrets.CD_FULL_USER }}@${VCELL_MANAGER_NODE}:${VCELL_DEPLOY_REMOTE_DIR} scp docker/swarm/variables ${{ secrets.CD_FULL_USER }}@${VCELL_MANAGER_NODE}:${VCELL_DEPLOY_REMOTE_DIR}/variables_$VCELL_TAG - name: upload variables - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: variables path: docker/swarm/variables @@ -91,7 +91,7 @@ jobs: ./generate_installers.sh ./${VCELL_CONFIG_FILE_NAME} - name: upload generated installers if: ${{ github.event.inputs.server_only != 'true' }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: installers path: docker/swarm/generated_installers @@ -142,7 +142,7 @@ jobs: fi - name: update generated installers if: ${{ github.event.inputs.server_only != 'true' }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: installers path: installers @@ -157,7 +157,7 @@ jobs: needs: notarize steps: - name: checkout tag - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: ${{ github.event.inputs.vcell_version }}.${{ github.event.inputs.vcell_build }} - name: retrieve installers and variables @@ -172,7 +172,7 @@ jobs: mkdir ../docker/swarm/generated_installers mv * ../docker/swarm/generated_installers - name: setup ssh-agent - uses: webfactory/ssh-agent@v0.5.3 + uses: webfactory/ssh-agent@v0.8.0 with: ssh-private-key: ${{ secrets.VC_KEY }} - name: retrieve config file diff --git a/docker/build/Dockerfile-submit-dev b/docker/build/Dockerfile-submit-dev index 307d81e08b..ad602c6206 100644 --- a/docker/build/Dockerfile-submit-dev +++ b/docker/build/Dockerfile-submit-dev @@ -73,6 +73,7 @@ ENV softwareVersion=SOFTWARE-VERSION-NOT-SET \ slurm_tmpdir="slurm-tmpdir-not-set" \ slurm_local_singularity_dir="slurm_local_singularity_dir-not-set" \ slurm_central_singularity_dir="slurm_central_singularity_dir-not-set" \ + slurm_singularity_module_name="slurm_singularity_module_name-not-set" \ jmsblob_minsize=100000 \ vcell_ssh_cmd_cmdtimeout="cmdSrvcSshCmdTimeoutMS-not-set" \ vcell_ssh_cmd_restoretimeout="cmdSrvcSshCmdRestoreTimeoutFactor-not-set" \ @@ -124,6 +125,7 @@ ENTRYPOINT java \ -Dvcell.slurm.tmpdir="${slurm_tmpdir}" \ -Dvcell.slurm.local.singularity.dir="${slurm_local_singularity_dir}" \ -Dvcell.slurm.central.singularity.dir="${slurm_central_singularity_dir}" \ + -Dvcell.slurm.singularity.module.name="${slurm_singularity_module_name}" \ -Dvcell.batch.singularity.image="${batch_singularity_imagefile}" \ -Dvcell.opt.singularity.image="${opt_singularity_imagefile}" \ -Dvcell.batch.docker.name="${docker_name}" \ diff --git a/docker/swarm/docker-compose.yml b/docker/swarm/docker-compose.yml index 07ea8a9273..e8529c72bf 100644 --- a/docker/swarm/docker-compose.yml +++ b/docker/swarm/docker-compose.yml @@ -174,6 +174,7 @@ services: - slurm_tmpdir=${VCELL_SLURM_TMPDIR} - slurm_local_singularity_dir=${VCELL_SLURM_LOCAL_SINGULARITY_DIR} - slurm_central_singularity_dir=${VCELL_SLURM_CENTRAL_SINGULARITY_DIR} + - slurm_singularity_module_name=${VCELL_SLURM_SINGULARITY_MODULE_NAME} - simdatadir_archive_internal=${VCELL_SIMDATADIR_ARCHIVE_INTERNAL} - simdatadir_archive_external=${VCELL_SIMDATADIR_ARCHIVE_EXTERNAL} diff --git a/docker/swarm/localconfig_realslurm_oracle.sh b/docker/swarm/localconfig_realslurm_oracle.sh index 4f1b6cebdc..69d6f2432c 100755 --- a/docker/swarm/localconfig_realslurm_oracle.sh +++ b/docker/swarm/localconfig_realslurm_oracle.sh @@ -97,6 +97,7 @@ VCELL_SLURM_QOS_PU=$vcell_slurm_qos_pu VCELL_SLURM_TMPDIR=/scratch/vcell VCELL_SLURM_LOCAL_SINGULARITY_DIR=/state/partition1/singularityImages VCELL_SLURM_CENTRAL_SINGULARITY_DIR=/share/apps/vcell3/singularityImages +VCELL_SLURM_SINGULARITY_MODULE_NAME=singularity/vcell-3.10.0 # # VCELL_API_PORT_EXTERNAL uses 443 for Beta and Rel (but on different machines/swarm clusters) @@ -112,10 +113,10 @@ VCELL_S3PROXY_PORT_EXTERNAL=$((8086 + _site_port_offset)) VCELL_HTC_NODELIST= VCELL_BATCH_DOCKER_IMAGE="${VCELL_REPO_NAMESPACE}/vcell-batch:${VCELL_TAG}" VCELL_BATCH_SINGULARITY_FILENAME="${VCELL_BATCH_DOCKER_IMAGE//[\/:]/_}.img" -VCELL_BATCH_SINGULARITY_IMAGE_EXTERNAL="/state/partition1/singularityImages/${VCELL_BATCH_SINGULARITY_FILENAME}" +VCELL_BATCH_SINGULARITY_IMAGE_EXTERNAL="${VCELL_SLURM_LOCAL_SINGULARITY_DIR}/${VCELL_BATCH_SINGULARITY_FILENAME}" VCELL_OPT_DOCKER_IMAGE="${VCELL_REPO_NAMESPACE}/vcell-opt:${VCELL_TAG}" VCELL_OPT_SINGULARITY_FILENAME="${VCELL_OPT_DOCKER_IMAGE//[\/:]/_}.img" -VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL="/state/partition1/singularityImages/${VCELL_OPT_SINGULARITY_FILENAME}" +VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL="${VCELL_SLURM_LOCAL_SINGULARITY_DIR}/${VCELL_OPT_SINGULARITY_FILENAME}" VCELL_SMTP_HOSTNAME=vdsmtp.cam.uchc.edu VCELL_SMTP_PORT=25 VCELL_SMTP_EMAILADDRESS=VCell_Support@uchc.edu @@ -156,6 +157,7 @@ VCELL_DEBUG_PORT_VCELL_SUBMIT=$((VCELL_DEBUG_PORT_BASE+5)) VCELL_DEBUG_PORT_VCELL_WEB=$((VCELL_DEBUG_PORT_BASE+6)) VCELL_DEPLOY_SECRETS_DIR=${HOME}/vcellkeys_oracle VCELL_EXPORT_BASEURL=$VCELL_EXPORT_BASEURL +VCELL_S3_EXPORT_BASEURL=VCELL_S3_EXPORT_BASEURL_not_set VCELL_EXPORTDIR_HOST=$VCELL_EXPORTDIR_HOST VCELL_HTC_NODELIST=$VCELL_HTC_NODELIST VCELL_HTCLOGS_EXTERNAL=/share/apps/vcell3/dev_htclogs @@ -181,17 +183,18 @@ VCELL_SIMDATADIR_ARCHIVE_INTERNAL=/share/apps/vcell12/users VCELL_SIMDATADIR_ARCHIVE_HOST=/Volumes/vcell-1/users VCELL_SIMDATADIR_EXTERNAL=/share/apps/vcell3/users VCELL_SIMDATADIR_HOST=/Volumes/vcell/users -VCELL_SIMDATADIR_SECONDARY_EXTERNAL=/share/apps/vcell3/users +VCELL_SIMDATADIR_SECONDARY_EXTERNAL=/share/apps/vcell7/users VCELL_SIMDATADIR_SECONDARY_HOST=/Volumes/vcell/users VCELL_SIMDATADIR_PARALLEL_EXTERNAL=/share/apps/vcell3parallel VCELL_SITE_CAMEL=${_site_camel} VCELL_SITE=$VCELL_SITE -VCELL_SLURM_CENTRAL_SINGULARITY_DIR=${VCELL_SLURM_CENTRAL_SINGULARITY_DIR} VCELL_SLURM_CMD_SACCT=$VCELL_SLURM_CMD_SACCT VCELL_SLURM_CMD_SBATCH=$VCELL_SLURM_CMD_SBATCH VCELL_SLURM_CMD_SCANCEL=$VCELL_SLURM_CMD_SCANCEL VCELL_SLURM_CMD_SQUEUE=$VCELL_SLURM_CMD_SQUEUE VCELL_SLURM_LOCAL_SINGULARITY_DIR=${VCELL_SLURM_LOCAL_SINGULARITY_DIR} +VCELL_SLURM_CENTRAL_SINGULARITY_DIR=${VCELL_SLURM_CENTRAL_SINGULARITY_DIR} +VCELL_SLURM_SINGULARITY_MODULE_NAME=${VCELL_SLURM_SINGULARITY_MODULE_NAME} VCELL_SLURM_PARTITION_PU=$VCELL_SLURM_PARTITION_PU VCELL_SLURM_PARTITION=$VCELL_SLURM_PARTITION VCELL_SLURM_QOS_PU=$VCELL_SLURM_QOS_PU diff --git a/docker/swarm/localconfig_realslurm_oracle_zeke.sh b/docker/swarm/localconfig_realslurm_oracle_zeke.sh index 8ee1b6a824..a6e9aca142 100755 --- a/docker/swarm/localconfig_realslurm_oracle_zeke.sh +++ b/docker/swarm/localconfig_realslurm_oracle_zeke.sh @@ -99,6 +99,7 @@ VCELL_SLURM_QOS_PU=$vcell_slurm_qos_pu VCELL_SLURM_TMPDIR=/scratch/vcell VCELL_SLURM_LOCAL_SINGULARITY_DIR=/state/partition1/singularityImages VCELL_SLURM_CENTRAL_SINGULARITY_DIR=/share/apps/vcell3/singularityImages +VCELL_SLURM_SINGULARITY_MODULE_NAME=singularity/vcell-3.10.0 # # VCELL_API_PORT_EXTERNAL uses 443 for Beta and Rel (but on different machines/swarm clusters) @@ -114,10 +115,10 @@ VCELL_S3PROXY_PORT_EXTERNAL=$((8086 + _site_port_offset)) VCELL_HTC_NODELIST= VCELL_BATCH_DOCKER_IMAGE="${VCELL_REPO_NAMESPACE}/vcell-batch:${VCELL_TAG}" VCELL_BATCH_SINGULARITY_FILENAME="${VCELL_BATCH_DOCKER_IMAGE//[\/:]/_}.img" -VCELL_BATCH_SINGULARITY_IMAGE_EXTERNAL="/state/partition1/singularityImages/${VCELL_BATCH_SINGULARITY_FILENAME}" +VCELL_BATCH_SINGULARITY_IMAGE_EXTERNAL="${VCELL_SLURM_LOCAL_SINGULARITY_DIR}/${VCELL_BATCH_SINGULARITY_FILENAME}" VCELL_OPT_DOCKER_IMAGE="${VCELL_REPO_NAMESPACE}/vcell-opt:${VCELL_TAG}" VCELL_OPT_SINGULARITY_FILENAME="${VCELL_OPT_DOCKER_IMAGE//[\/:]/_}.img" -VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL="/state/partition1/singularityImages/${VCELL_OPT_SINGULARITY_FILENAME}" +VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL="${VCELL_SLURM_LOCAL_SINGULARITY_DIR}/${VCELL_OPT_SINGULARITY_FILENAME}" VCELL_SMTP_HOSTNAME=vdsmtp.cam.uchc.edu VCELL_SMTP_PORT=25 VCELL_SMTP_EMAILADDRESS=VCell_Support@uchc.edu @@ -164,6 +165,7 @@ VCELL_DEBUG_PORT_VCELL_WEB=$((VCELL_DEBUG_PORT_BASE+6)) VCELL_DEPLOY_SECRETS_DIR=/media/zeke/DiskDrive/Home/Work/CCAM/VCellDummyFiles VCELL_EXPORT_BASEURL=$VCELL_EXPORT_BASEURL +VCELL_S3_EXPORT_BASEURL=VCELL_S3_EXPORT_BASEURL_not_set VCELL_EXPORTDIR_HOST=$VCELL_EXPORTDIR_HOST VCELL_HTC_NODELIST=$VCELL_HTC_NODELIST VCELL_HTCLOGS_EXTERNAL=/share/apps/vcell3/dev_htclogs @@ -203,12 +205,13 @@ VCELL_SIMDATADIR_PARALLEL_EXTERNAL=/share/apps/vcell3parallel VCELL_SITE_CAMEL=${_site_camel} VCELL_SITE=$VCELL_SITE -VCELL_SLURM_CENTRAL_SINGULARITY_DIR=${VCELL_SLURM_CENTRAL_SINGULARITY_DIR} VCELL_SLURM_CMD_SACCT=$VCELL_SLURM_CMD_SACCT VCELL_SLURM_CMD_SBATCH=$VCELL_SLURM_CMD_SBATCH VCELL_SLURM_CMD_SCANCEL=$VCELL_SLURM_CMD_SCANCEL VCELL_SLURM_CMD_SQUEUE=$VCELL_SLURM_CMD_SQUEUE VCELL_SLURM_LOCAL_SINGULARITY_DIR=${VCELL_SLURM_LOCAL_SINGULARITY_DIR} +VCELL_SLURM_CENTRAL_SINGULARITY_DIR=${VCELL_SLURM_CENTRAL_SINGULARITY_DIR} +VCELL_SLURM_SINGULARITY_MODULE_NAME=${VCELL_SLURM_SINGULARITY_MODULE_NAME} VCELL_SLURM_PARTITION_PU=$VCELL_SLURM_PARTITION_PU VCELL_SLURM_PARTITION=$VCELL_SLURM_PARTITION VCELL_SLURM_QOS_PU=$VCELL_SLURM_QOS_PU diff --git a/docker/swarm/localconfig_realslurm_postgres.sh b/docker/swarm/localconfig_realslurm_postgres.sh index 384915a186..ce1a3a6043 100755 --- a/docker/swarm/localconfig_realslurm_postgres.sh +++ b/docker/swarm/localconfig_realslurm_postgres.sh @@ -97,6 +97,7 @@ VCELL_SLURM_QOS_PU=$vcell_slurm_qos_pu VCELL_SLURM_TMPDIR=/scratch/vcell VCELL_SLURM_LOCAL_SINGULARITY_DIR=/state/partition1/singularityImages VCELL_SLURM_CENTRAL_SINGULARITY_DIR=/share/apps/vcell3/singularityImages +VCELL_SLURM_SINGULARITY_MODULE_NAME=singularity/vcell-3.10.0 # # VCELL_API_PORT_EXTERNAL uses 443 for Beta and Rel (but on different machines/swarm clusters) @@ -112,10 +113,10 @@ VCELL_S3PROXY_PORT_EXTERNAL=$((8086 + _site_port_offset)) VCELL_HTC_NODELIST= VCELL_BATCH_DOCKER_IMAGE="${VCELL_REPO_NAMESPACE}/vcell-batch:${VCELL_TAG}" VCELL_BATCH_SINGULARITY_FILENAME="${VCELL_BATCH_DOCKER_IMAGE//[\/:]/_}.img" -VCELL_BATCH_SINGULARITY_IMAGE_EXTERNAL="/state/partition1/singularityImages/${VCELL_BATCH_SINGULARITY_FILENAME}" +VCELL_BATCH_SINGULARITY_IMAGE_EXTERNAL="${VCELL_SLURM_LOCAL_SINGULARITY_DIR}/${VCELL_BATCH_SINGULARITY_FILENAME}" VCELL_OPT_DOCKER_IMAGE="${VCELL_REPO_NAMESPACE}/vcell-opt:${VCELL_TAG}" VCELL_OPT_SINGULARITY_FILENAME="${VCELL_OPT_DOCKER_IMAGE//[\/:]/_}.img" -VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL="/state/partition1/singularityImages/${VCELL_OPT_SINGULARITY_FILENAME}" +VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL="${VCELL_SLURM_LOCAL_SINGULARITY_DIR}/${VCELL_OPT_SINGULARITY_FILENAME}" VCELL_SMTP_HOSTNAME=vdsmtp.cam.uchc.edu VCELL_SMTP_PORT=25 VCELL_SMTP_EMAILADDRESS=VCell_Support@uchc.edu @@ -156,6 +157,7 @@ VCELL_DEBUG_PORT_VCELL_SUBMIT=$((VCELL_DEBUG_PORT_BASE+5)) VCELL_DEBUG_PORT_VCELL_WEB=$((VCELL_DEBUG_PORT_BASE+6)) VCELL_DEPLOY_SECRETS_DIR=${HOME}/vcellkeys_postgres VCELL_EXPORT_BASEURL=$VCELL_EXPORT_BASEURL +VCELL_S3_EXPORT_BASEURL=VCELL_S3_EXPORT_BASEURL_not_set VCELL_EXPORTDIR_HOST=$VCELL_EXPORTDIR_HOST VCELL_HTC_NODELIST=$VCELL_HTC_NODELIST VCELL_HTCLOGS_EXTERNAL=/share/apps/vcell3/dev_htclogs @@ -181,17 +183,18 @@ VCELL_SIMDATADIR_ARCHIVE_INTERNAL=/share/apps/vcell12/users VCELL_SIMDATADIR_ARCHIVE_HOST=/Volumes/vcell-1/users VCELL_SIMDATADIR_EXTERNAL=/share/apps/vcell3/users VCELL_SIMDATADIR_HOST=/Volumes/vcell/users -VCELL_SIMDATADIR_SECONDARY_EXTERNAL=/share/apps/vcell3/users +VCELL_SIMDATADIR_SECONDARY_EXTERNAL=/share/apps/vcell7/users VCELL_SIMDATADIR_SECONDARY_HOST=/Volumes/vcell/users VCELL_SIMDATADIR_PARALLEL_EXTERNAL=/share/apps/vcell3parallel VCELL_SITE_CAMEL=${_site_camel} VCELL_SITE=$VCELL_SITE -VCELL_SLURM_CENTRAL_SINGULARITY_DIR=${VCELL_SLURM_CENTRAL_SINGULARITY_DIR} VCELL_SLURM_CMD_SACCT=$VCELL_SLURM_CMD_SACCT VCELL_SLURM_CMD_SBATCH=$VCELL_SLURM_CMD_SBATCH VCELL_SLURM_CMD_SCANCEL=$VCELL_SLURM_CMD_SCANCEL VCELL_SLURM_CMD_SQUEUE=$VCELL_SLURM_CMD_SQUEUE VCELL_SLURM_LOCAL_SINGULARITY_DIR=${VCELL_SLURM_LOCAL_SINGULARITY_DIR} +VCELL_SLURM_CENTRAL_SINGULARITY_DIR=${VCELL_SLURM_CENTRAL_SINGULARITY_DIR} +VCELL_SLURM_SINGULARITY_MODULE_NAME=${VCELL_SLURM_SINGULARITY_MODULE_NAME} VCELL_SLURM_PARTITION_PU=$VCELL_SLURM_PARTITION_PU VCELL_SLURM_PARTITION=$VCELL_SLURM_PARTITION VCELL_SLURM_QOS_PU=$VCELL_SLURM_QOS_PU diff --git a/docker/swarm/serverconfig-uch.sh b/docker/swarm/serverconfig-uch.sh index 1912d9c254..3f43602168 100755 --- a/docker/swarm/serverconfig-uch.sh +++ b/docker/swarm/serverconfig-uch.sh @@ -46,7 +46,7 @@ case $VCELL_SITE in ;; BETA) _site_port_offset=1 - VCELL_API_HOST_EXTERNAL=vcellapi.cam.uchc.edu + VCELL_API_HOST_EXTERNAL=http://vcellapi.cam.uchc.edu VCELL_API_PORT_EXTERNAL=443 VCELL_DEBUG_PORT_BASE=5010 _applicationId="1471-8022-1038-5552" @@ -62,7 +62,7 @@ case $VCELL_SITE in TEST) _site_port_offset=3 # VCELL_API_PORT_EXTERNAL=8081 - VCELL_API_HOST_EXTERNAL=vcellapi-test.cam.uchc.edu + VCELL_API_HOST_EXTERNAL=http://vcellapi-test.cam.uchc.edu VCELL_API_PORT_EXTERNAL=443 VCELL_DEBUG_PORT_BASE=5030 _applicationId="1471-8022-1038-5555" @@ -119,8 +119,9 @@ VCELL_SLURM_PARTITION_PU=$vcell_slurm_partition_pu VCELL_SLURM_RESERVATION_PU=$vcell_slurm_reservation_pu VCELL_SLURM_QOS_PU=$vcell_slurm_qos_pu VCELL_SLURM_TMPDIR=/scratch/vcell -VCELL_SLURM_LOCAL_SINGULARITY_DIR=/state/partition1/singularityImages +VCELL_SLURM_LOCAL_SINGULARITY_DIR=/local/singularityImages VCELL_SLURM_CENTRAL_SINGULARITY_DIR=/share/apps/vcell3/singularityImages +VCELL_SLURM_SINGULARITY_MODULE_NAME=singularity/vcell-3.10.0 # # VCELL_API_PORT_EXTERNAL uses 443 for Beta and Rel (but on different machines/swarm clusters) @@ -136,10 +137,10 @@ VCELL_S3PROXY_PORT_EXTERNAL=$((8086 + _site_port_offset)) VCELL_HTC_NODELIST= VCELL_BATCH_DOCKER_IMAGE="${VCELL_REPO_NAMESPACE}/vcell-batch:${VCELL_TAG}" VCELL_BATCH_SINGULARITY_FILENAME="${VCELL_BATCH_DOCKER_IMAGE//[\/:]/_}.img" -VCELL_BATCH_SINGULARITY_IMAGE_EXTERNAL="/state/partition1/singularityImages/${VCELL_BATCH_SINGULARITY_FILENAME}" +VCELL_BATCH_SINGULARITY_IMAGE_EXTERNAL="${VCELL_SLURM_LOCAL_SINGULARITY_DIR}/${VCELL_BATCH_SINGULARITY_FILENAME}" VCELL_OPT_DOCKER_IMAGE="${VCELL_REPO_NAMESPACE}/vcell-opt:${VCELL_TAG}" VCELL_OPT_SINGULARITY_FILENAME="${VCELL_OPT_DOCKER_IMAGE//[\/:]/_}.img" -VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL="/state/partition1/singularityImages/${VCELL_OPT_SINGULARITY_FILENAME}" +VCELL_OPT_SINGULARITY_IMAGE_EXTERNAL="${VCELL_SLURM_LOCAL_SINGULARITY_DIR}/${VCELL_OPT_SINGULARITY_FILENAME}" VCELL_SMTP_HOSTNAME=vdsmtp.cam.uchc.edu VCELL_SMTP_PORT=25 VCELL_SMTP_EMAILADDRESS=VCell_Support@uchc.edu @@ -211,12 +212,13 @@ VCELL_SIMDATADIR_SECONDARY_HOST=/share/apps/vcell7/users VCELL_SIMDATADIR_PARALLEL_EXTERNAL=/share/apps/vcell3parallel VCELL_SITE_CAMEL=${_site_camel} VCELL_SITE=$VCELL_SITE -VCELL_SLURM_CENTRAL_SINGULARITY_DIR=${VCELL_SLURM_CENTRAL_SINGULARITY_DIR} VCELL_SLURM_CMD_SACCT=$VCELL_SLURM_CMD_SACCT VCELL_SLURM_CMD_SBATCH=$VCELL_SLURM_CMD_SBATCH VCELL_SLURM_CMD_SCANCEL=$VCELL_SLURM_CMD_SCANCEL VCELL_SLURM_CMD_SQUEUE=$VCELL_SLURM_CMD_SQUEUE VCELL_SLURM_LOCAL_SINGULARITY_DIR=${VCELL_SLURM_LOCAL_SINGULARITY_DIR} +VCELL_SLURM_CENTRAL_SINGULARITY_DIR=${VCELL_SLURM_CENTRAL_SINGULARITY_DIR} +VCELL_SLURM_SINGULARITY_MODULE_NAME=${VCELL_SLURM_SINGULARITY_MODULE_NAME} VCELL_SLURM_PARTITION_PU=$VCELL_SLURM_PARTITION_PU VCELL_SLURM_PARTITION=$VCELL_SLURM_PARTITION VCELL_SLURM_QOS_PU=$VCELL_SLURM_QOS_PU diff --git a/vcell-core/src/main/java/cbit/vcell/resource/PropertyLoader.java b/vcell-core/src/main/java/cbit/vcell/resource/PropertyLoader.java index ade8a5709b..70dcd3c68c 100644 --- a/vcell-core/src/main/java/cbit/vcell/resource/PropertyLoader.java +++ b/vcell-core/src/main/java/cbit/vcell/resource/PropertyLoader.java @@ -96,6 +96,7 @@ public static void setConfigProvider(VCellConfigProvider configProvider) { public static final String slurm_tmpdir = record("vcell.slurm.tmpdir",ValueType.GEN); public static final String slurm_local_singularity_dir = record("vcell.slurm.local.singularity.dir",ValueType.GEN); public static final String slurm_central_singularity_dir= record("vcell.slurm.central.singularity.dir",ValueType.GEN); + public static final String slurm_singularity_module_name= record("vcell.slurm.singularity.module.name",ValueType.GEN); public static final String sgeModulePath = record("vcell.htc.sge.module",ValueType.GEN); public static final String pbsModulePath = record("vcell.htc.pbs.module",ValueType.GEN); public static final String MPI_HOME_INTERNAL = record("vcell.htc.mpi.home",ValueType.DIR); diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/batch/sim/HtcSimulationWorker.java b/vcell-server/src/main/java/cbit/vcell/message/server/batch/sim/HtcSimulationWorker.java index a5d4973142..1229185e8e 100644 --- a/vcell-server/src/main/java/cbit/vcell/message/server/batch/sim/HtcSimulationWorker.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/batch/sim/HtcSimulationWorker.java @@ -671,6 +671,7 @@ public static void main(String[] args) throws IOException { PropertyLoader.vcellbatch_docker_name, PropertyLoader.slurm_local_singularity_dir, PropertyLoader.slurm_central_singularity_dir, + PropertyLoader.slurm_singularity_module_name, PropertyLoader.slurm_reservation, PropertyLoader.slurm_qos, PropertyLoader.slurm_partition_pu, diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/htc/slurm/SlurmProxy.java b/vcell-server/src/main/java/cbit/vcell/message/server/htc/slurm/SlurmProxy.java index 90804289e6..2e5cc336f1 100644 --- a/vcell-server/src/main/java/cbit/vcell/message/server/htc/slurm/SlurmProxy.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/htc/slurm/SlurmProxy.java @@ -455,6 +455,7 @@ SbatchSolverComponents generateScript(String jobName, ExecutableCommand.Containe String slurm_tmpdir = PropertyLoader.getRequiredProperty(PropertyLoader.slurm_tmpdir); String slurm_central_singularity_dir = PropertyLoader.getRequiredProperty(PropertyLoader.slurm_central_singularity_dir); String slurm_local_singularity_dir = PropertyLoader.getRequiredProperty(PropertyLoader.slurm_local_singularity_dir); + String slurm_singularity_module_name = PropertyLoader.getRequiredProperty(PropertyLoader.slurm_singularity_module_name); String simDataDirArchiveExternal = PropertyLoader.getRequiredProperty(PropertyLoader.simDataDirArchiveExternal); String simDataDirArchiveInternal = PropertyLoader.getRequiredProperty(PropertyLoader.simDataDirArchiveInternal); File slurm_singularity_central_filepath = new File(slurm_central_singularity_dir,new File(slurm_singularity_local_image_filepath).getName()); @@ -481,7 +482,7 @@ SbatchSolverComponents generateScript(String jobName, ExecutableCommand.Containe slurmInitSingularity(singularityLSB, primaryDataDirExternal, Optional.of(secondaryDataDirExternal), htclogdir_external, softwareVersion, slurm_singularity_local_image_filepath, slurm_tmpdir, slurm_central_singularity_dir, slurm_local_singularity_dir, simDataDirArchiveExternal, simDataDirArchiveInternal, slurm_singularity_central_filepath, - environmentVars); + slurm_singularity_module_name, environmentVars); LineStringBuilder sendFailMsgLSB = new LineStringBuilder(); sendFailMsgScript(simTask, sendFailMsgLSB, jmshost_sim_external, jmsport_sim_external, jmsuser, jmspswd); @@ -676,7 +677,7 @@ private void slurmInitSingularity(LineStringBuilder lsb, String primaryDataDirEx String slurm_singularity_local_image_filepath, String slurm_tmpdir, String slurm_central_singularity_dir, String slurm_local_singularity_dir, String simDataDirArchiveExternal, String simDataDirArchiveInternal, - File slurm_singularity_central_filepath, String[] environmentVars) { + File slurm_singularity_central_filepath, String singularity_module_name, String[] environmentVars) { lsb.write("#BEGIN---------SlurmProxy.generateScript():slurmInitSingularity----------"); lsb.write("set -x"); lsb.newline(); @@ -690,7 +691,7 @@ private void slurmInitSingularity(LineStringBuilder lsb, String primaryDataDirEx lsb.write("echo `hostname`\n"); lsb.write("export MODULEPATH=/isg/shared/modulefiles:/tgcapps/modulefiles\n"); lsb.write("source /usr/share/Modules/init/bash\n"); - lsb.write("module load singularity\n"); + lsb.write("module load "+singularity_module_name+"\n"); lsb.write("echo \"job running on host `hostname -f`\""); lsb.newline(); @@ -761,7 +762,7 @@ private void slurmInitSingularity(LineStringBuilder lsb, String primaryDataDirEx "--bind "+slurm_tmpdir+":/solvertmp " + "$localSingularityImage "+singularityEnvironmentVars+" \""); lsb.write("else"); - lsb.write(" echo \"Required singularity command not found (module load singularity/2.4.2) \""); + lsb.write(" echo \"Required singularity command not found (maybe 'module load "+singularity_module_name+"' command didn't work) \""); lsb.write(" exit 1"); // StringBuffer dockerEnvironmentVars = new StringBuffer(); // for (String envVar : environmentVars) { @@ -1086,6 +1087,7 @@ public HtcJobID submitOptimizationJob(String jobName, File sub_file_internal, Fi String slurm_tmpdir = PropertyLoader.getRequiredProperty(PropertyLoader.slurm_tmpdir); String slurm_central_singularity_dir = PropertyLoader.getRequiredProperty(PropertyLoader.slurm_central_singularity_dir); String slurm_local_singularity_dir = PropertyLoader.getRequiredProperty(PropertyLoader.slurm_local_singularity_dir); + String slurm_singularity_module_name = PropertyLoader.getRequiredProperty(PropertyLoader.slurm_singularity_module_name); String simDataDirArchiveExternal = PropertyLoader.getRequiredProperty(PropertyLoader.simDataDirArchiveExternal); String simDataDirArchiveInternal = PropertyLoader.getRequiredProperty(PropertyLoader.simDataDirArchiveInternal); File slurm_singularity_central_filepath = new File(slurm_central_singularity_dir,new File(slurm_singularity_local_image_filepath).getName()); @@ -1106,7 +1108,7 @@ public HtcJobID submitOptimizationJob(String jobName, File sub_file_internal, Fi slurmInitSingularity(lsb, optDataDirExternal.getAbsolutePath(), Optional.empty(), htclogdir_external, softwareVersion, slurm_singularity_local_image_filepath, slurm_tmpdir, slurm_central_singularity_dir, slurm_local_singularity_dir, simDataDirArchiveExternal, simDataDirArchiveInternal, - slurm_singularity_central_filepath, environmentVars); + slurm_singularity_central_filepath, slurm_singularity_module_name, environmentVars); lsb.write(" cmd_prefix=\"$container_prefix\""); lsb.write("echo \"cmd_prefix is '${cmd_prefix}'\"");