changes needed to test use of cromwell 87 and the GCP Batch backend

wustl-oncology · Oct 15, 2024 · 11fda4f · 11fda4f
1 parent d47e34c
commit 11fda4f
Show file tree

Hide file tree

Showing 5 changed files with 22 additions and 7 deletions.
diff --git a/manual-workflows/base_cromwell.conf b/manual-workflows/base_cromwell.conf
@@ -24,11 +24,10 @@ engine.filesystems {
 
 backend.default = "default"
 backend.providers.default {
-  actor-factory = "cromwell.backend.google.pipelines.v2beta.PipelinesApiLifecycleActorFactory"
+  actor-factory = "cromwell.backend.google.batch.GcpBatchBackendLifecycleActorFactory"
   config {
     genomics {
       auth = "application-default"
-      endpoint-url = "https://lifesciences.googleapis.com/"
       location = "us-central1"
     }
     filesystems {
@@ -37,7 +36,6 @@ backend.providers.default {
         caching.duplication-strategy = "reference"
       }
     }
-    include "papi_v2_reference_image_manifest.conf"
   }
 }
 

diff --git a/manual-workflows/start.sh b/manual-workflows/start.sh
@@ -18,7 +18,7 @@ arguments:
 --workflow-options   Local path to workflow_options.json. DEFAULT \$SRC_DIR/workflow_options.json
 --machine-type       GCP machine type for the instance. DEFAULT e2-standard-2
 --zone               DEFAULT us-central1-c. For options, visit: https://cloud.google.com/compute/docs/regions-zones 
---analysis-release   DEFAULT 1.1.4. For options, visit: https://github.com/wustl-oncology/analysis-wdls/releases
+--analysis-release   DEFAULT 1.2.2-gcpbranch. For options, visit: https://github.com/wustl-oncology/analysis-wdls/releases
 
 Additional arguments are passed directly to gsutil compute instances
 create command. For more information on those arguments, check that commands
@@ -100,7 +100,7 @@ while test $# -gt 0; do
             ;;
         --analysis-release*)
             if [ ! "$2" ]; then
-                ANALYSIS_RELEASE="1.1.4"
+                ANALYSIS_RELEASE="1.2.2-gcpbatch"
             else
                 ANALYSIS_RELEASE=$2
                 shift
@@ -120,7 +120,7 @@ MACHINE_TYPE=${MACHINE_TYPE:-"e2-standard-2"}
 [ -z $CROMWELL_CONF    ] && CROMWELL_CONF="$SRC_DIR/cromwell.conf"
 [ -z $WORKFLOW_OPTIONS ] && WORKFLOW_OPTIONS="$SRC_DIR/workflow_options.json"
 [ -z $ZONE             ] && ZONE="us-central1-c"
-[ -z $ANALYSIS_RELEASE ] && ANALYSIS_RELEASE="1.1.4"
+[ -z $ANALYSIS_RELEASE ] && ANALYSIS_RELEASE="1.2.2-gcpbatch"
 
 if [[ ! -f $CROMWELL_CONF ]]; then
     cat <<EOF
@@ -152,7 +152,7 @@ gcloud compute instances create $INSTANCE_NAME \
        --machine-type=$MACHINE_TYPE \
        --service-account=$SERVER_ACCOUNT --scopes=cloud-platform \
        --network=$NETWORK --subnet=$SUBNET \
-       --metadata=cromwell-version=71,analysis-release="$ANALYSIS_RELEASE" \
+       --metadata=cromwell-version=87,analysis-release="$ANALYSIS_RELEASE" \
        --metadata-from-file=startup-script=$SRC_DIR/server_startup.py,cromwell-conf=$CROMWELL_CONF,helpers-sh=$SRC_DIR/helpers.sh,cromwell-service=$SRC_DIR/cromwell.service,workflow-options=$WORKFLOW_OPTIONS,persist-artifacts=$SRC_DIR/../scripts/persist_artifacts.py \
        $@
 

diff --git a/scripts/cloudize-workflow.py b/scripts/cloudize-workflow.py
diff --git a/scripts/create_resources.sh b/scripts/create_resources.sh
@@ -24,6 +24,12 @@ gcloud iam service-accounts create $SERVER_NAME \
 gcloud projects add-iam-policy-binding $PROJECT \
        --member="serviceAccount:$SERVER_ACCOUNT" \
        --role='roles/lifesciences.workflowsRunner' > /dev/null
+gcloud projects add-iam-policy-binding $PROJECT \
+       --member="serviceAccount:$SERVER_ACCOUNT" \
+       --role='roles/batch.jobsEditor' > /dev/null
+gcloud projects add-iam-policy-binding $PROJECT \
+       --member="serviceAccount:$SERVER_ACCOUNT" \
+       --role='roles/batch.agentReporter' > /dev/null
 gcloud projects add-iam-policy-binding $PROJECT \
        --member="serviceAccount:$SERVER_ACCOUNT" \
        --role='roles/compute.instanceAdmin' > /dev/null
@@ -40,6 +46,15 @@ gcloud iam service-accounts add-iam-policy-binding $COMPUTE_ACCOUNT \
        --project=$PROJECT \
        --role='roles/iam.serviceAccountUser' > /dev/null
 
+gcloud projects add-iam-policy-binding $PROJECT \
+       --member="serviceAccount:$COMPUTE_ACCOUNT" \
+       --role='roles/batch.jobsEditor' > /dev/null
+gcloud projects add-iam-policy-binding $PROJECT \
+       --member="serviceAccount:$COMPUTE_ACCOUNT" \
+       --role='roles/batch.agentReporter' > /dev/null
+gcloud projects add-iam-policy-binding $PROJECT \
+       --member="serviceAccount:$COMPUTE_ACCOUNT" \
+       --role='roles/compute.instanceAdmin' > /dev/null
 
 # Network
 gcloud compute networks create $NETWORK \

diff --git a/scripts/enable_api.sh b/scripts/enable_api.sh
@@ -7,3 +7,5 @@ gcloud services enable storage.googleapis.com
 gcloud services enable lifesciences.googleapis.com
 gcloud services enable iam.googleapis.com
 gcloud services enable compute.googleapis.com
+gcloud services enable batch.googleapis.com
+