From a8c64c33ddc1648b3a3790c9bb3db3be2a305c91 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Fri, 10 May 2024 22:21:03 -0400 Subject: [PATCH] Allow describing wdl with zipped imports. --- CHANGELOG.md | 8 +++ .../src/main/resources/swagger/cromiam.yaml | 5 ++ .../src/main/scala/centaur/test/Test.scala | 3 +- .../centaur/test/workflow/Workflow.scala | 3 +- .../scala/cromwell/api/CromwellClient.scala | 8 ++- .../api/model/WorkflowDescribeRequest.scala | 5 +- docs/api/RESTAPI.md | 3 +- .../src/main/resources/swagger/cromwell.yaml | 5 ++ .../routes/WomtoolRouteSupport.scala | 3 +- .../routes/CromwellApiServiceSpec.scala | 4 +- .../routes/WomtoolRouteSupportSpec.scala | 49 +++++++++++++++++-- .../cromwell/services/womtool/Describer.scala | 23 +++++++-- .../wdl_1_0/relative_imports/description.json | 33 +++++++++++++ .../wdl_1_0/relative_imports/workflow.wdl | 11 +++++ .../structs/my_struct.wdl | 5 ++ .../workflowDependencies/sub_wfs/foo.wdl | 11 +++++ .../sub_wfs/tasks/add5.wdl | 18 +++++++ .../services/womtool/DescriberSpec.scala | 29 +++++++---- 18 files changed, 202 insertions(+), 24 deletions(-) create mode 100644 services/src/test/resources/describe/wdl_1_0/relative_imports/description.json create mode 100644 services/src/test/resources/describe/wdl_1_0/relative_imports/workflow.wdl create mode 100644 services/src/test/resources/describe/wdl_1_0/relative_imports/workflowDependencies/structs/my_struct.wdl create mode 100644 services/src/test/resources/describe/wdl_1_0/relative_imports/workflowDependencies/sub_wfs/foo.wdl create mode 100644 services/src/test/resources/describe/wdl_1_0/relative_imports/workflowDependencies/sub_wfs/tasks/add5.wdl diff --git a/CHANGELOG.md b/CHANGELOG.md index 50af73e8260..fa9ec56b930 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ Cromwell now allows opting into configured soft links on shared file systems such as HPC environments. More details can be found [here](https://cromwell.readthedocs.io/en/stable/backends/HPC/#optional-docker-soft-links). +### `/describe` endpoint support for `workflowDependencies` + +Previously it was not possible to describe a `workflowSource` that required `workflowDependencies` as the `/describe` +endpoint did not allow specifying the additional zip file. + +Now the endpoint will allow the user to upload the `workflowDependencies` zip file, will validate the top level +workflow plus the dependencies, then return the appropriate response including the defined workflow inputs and outputs. + ## 87 Release Notes ### `upgrade` command removed from Womtool diff --git a/CromIAM/src/main/resources/swagger/cromiam.yaml b/CromIAM/src/main/resources/swagger/cromiam.yaml index c6bf57fa609..8db42bbb567 100644 --- a/CromIAM/src/main/resources/swagger/cromiam.yaml +++ b/CromIAM/src/main/resources/swagger/cromiam.yaml @@ -838,6 +838,11 @@ paths: required: false type: file in: formData + - name: workflowDependencies + description: ZIP file containing workflow source files that are used to resolve local imports. This zip bundle will be unpacked in a sandbox accessible to this workflow. + required: false + type: file + in: formData - $ref: '#/parameters/workflowTypeParam' - $ref: '#/parameters/workflowTypeVersionParam' responses: diff --git a/centaur/src/main/scala/centaur/test/Test.scala b/centaur/src/main/scala/centaur/test/Test.scala index 66a4655a107..f685d4280b9 100644 --- a/centaur/src/main/scala/centaur/test/Test.scala +++ b/centaur/src/main/scala/centaur/test/Test.scala @@ -244,8 +244,7 @@ object Operations extends StrictLogging { } override def run: IO[Unit] = - // We can't describe workflows based on zipped imports, so don't try: - if (workflow.skipDescribeEndpointValidation || workflow.data.zippedImports.nonEmpty) { + if (workflow.skipDescribeEndpointValidation) { IO.pure(()) } else { checkDescriptionInner(0) diff --git a/centaur/src/main/scala/centaur/test/workflow/Workflow.scala b/centaur/src/main/scala/centaur/test/workflow/Workflow.scala index 212064acfe1..2bef0db2fac 100644 --- a/centaur/src/main/scala/centaur/test/workflow/Workflow.scala +++ b/centaur/src/main/scala/centaur/test/workflow/Workflow.scala @@ -45,7 +45,8 @@ final case class Workflow private (testName: String, workflowUrl = data.workflowUrl, workflowType = data.workflowType, workflowTypeVersion = data.workflowTypeVersion, - inputsJson = data.inputs.map(_.unsafeRunSync()) + inputsJson = data.inputs.map(_.unsafeRunSync()), + zippedImports = data.zippedImports ) def secondRun: Workflow = diff --git a/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala b/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala index 18df2ff5244..453c0008d63 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala @@ -266,7 +266,13 @@ object CromwellClient { Multipart.FormData.BodyPart(name, HttpEntity(MediaTypes.`application/json`, ByteString(source))) } - val multipartFormData = Multipart.FormData(sourceBodyParts.toSeq: _*) + val zipBodyParts = Map( + "workflowDependencies" -> describeRequest.zippedImports + ) collect { case (name, Some(file)) => + Multipart.FormData.BodyPart.fromPath(name, MediaTypes.`application/zip`, file.path) + } + + val multipartFormData = Multipart.FormData((sourceBodyParts ++ zipBodyParts).toSeq: _*) multipartFormData.toEntity() } diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowDescribeRequest.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowDescribeRequest.scala index 1a46e2117f9..58452cf115f 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowDescribeRequest.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowDescribeRequest.scala @@ -1,8 +1,11 @@ package cromwell.api.model +import better.files.File + final case class WorkflowDescribeRequest(workflowSource: Option[String], workflowUrl: Option[String], workflowType: Option[String], workflowTypeVersion: Option[String], - inputsJson: Option[String] + inputsJson: Option[String], + zippedImports: Option[File] ) diff --git a/docs/api/RESTAPI.md b/docs/api/RESTAPI.md index 64ffe19a3fd..3311b779f2e 100644 --- a/docs/api/RESTAPI.md +++ b/docs/api/RESTAPI.md @@ -1,5 +1,5 @@