Merge branch 'develop'

.circleci/config.yml

@@ -4,16 +4,6 @@ orbs:
   slack: circleci/[email protected]
   build-tools: circleci/[email protected]
 
-commands:
-  install_adoptopenjdk:
-    description: "Installing adoptopenjdk 8 and setting it as default Java"
-    steps:
-      - run: wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | sudo apt-key add -
-      - run: sudo add-apt-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/
-      - run: sudo apt update
-      - run: sudo apt install adoptopenjdk-8-hotspot
-      - run: sudo update-java-alternatives --set adoptopenjdk-8-hotspot-amd64
-
 jobs:
   test:
     parameters:
@@ -41,7 +31,6 @@ jobs:
             if [[ -z "${CI_PULL_REQUEST}" ]] && [[ "${BUILD_TYPE}" != "sbt" ]] ; then
               circleci-agent step halt
             fi
-      - install_adoptopenjdk
       - checkout
       - run:
           name: Custom step - configure GIT identity

.gitignore

@@ -2,6 +2,7 @@
 *~
 .DS_Store
 .artifactory
+.bsp
 .idea/*
 **/.idea/*
 .ensime_cache/*

.sdkmanrc

@@ -0,0 +1,3 @@
+# Enable auto-env through the sdkman_auto_env config
+# Add key=value pairs of SDKs to use below
+java=11.0.10.hs-adpt

.travis.yml

@@ -1,14 +1,10 @@
-sudo: required
-dist: xenial
+os: linux
+dist: focal
 services:
   - docker
-language: scala
-scala:
-  - 2.12.9
+language: minimal
 git:
   depth: false
-jdk:
-  - openjdk8
 cache:
   directories:
     - $HOME/.ivy2/cache
@@ -21,10 +17,7 @@ before_cache:
   - find $HOME/.coursier/cache -name "ivydata-*.properties" -print -delete
   - find $HOME/.sbt -name "*.lock" -print -delete
 env:
-  global:
-    - PYTHON3_HOME=/opt/python/3.7.1
-    - PATH=$PYTHON3_HOME/bin\:$PATH
-  matrix:
+  jobs:
     # Setting this variable twice will cause the 'script' section to run twice with the respective env var invoked
     - >-
       BUILD_TYPE=centaurAws
@@ -101,6 +94,16 @@ env:
       BUILD_TYPE=dockerScripts
     - >-
       BUILD_TYPE=sbt
+      BUILD_SBT_INCLUDE=engine
+    - >-
+      BUILD_TYPE=sbt
+      BUILD_SBT_INCLUDE=server
+    - >-
+      BUILD_TYPE=sbt
+      BUILD_SBT_INCLUDE=services
+    - >-
+      BUILD_TYPE=sbt
+      BUILD_SBT_EXCLUDE='engine|server|services'
     - >-
       BUILD_TYPE=dbms
     - >-

CHANGELOG.md

@@ -1,5 +1,22 @@
 # Cromwell Change Log
 
+## 60 Release Notes
+
+### Java 11
+
+As of this version, a distribution of Java 11 is required to run Cromwell. Cromwell is developed, tested, and
+containerized using [AdoptOpenJDK 11 HotSpot](https://adoptopenjdk.net/).
+
+### Hybrid metadata storage ("carboniting") removed
+
+Carboniting functionality has been removed from Cromwell. 
+There will be no effect for customers who store metadata permanently in the relational database (most common),
+and there will also be no effect for customers who use the in-memory database.
+
+Breaking change only for customers who explicitly enabled `carbonite-metadata-service` in their configuration to split
+metadata storage between a relational database and Google Cloud Storage. If you had previously enabled carboniting and 
+deletion, any workflows marked as `ArchivedAndPurged` in your database will no longer be accessible via the Cromwell metadata API.
+
 ## 59 Release Notes
 
 ### Bug Fixes

build.sbt

@@ -188,17 +188,6 @@ lazy val services = project
   .dependsOn(ftpFileSystem % "test->test")
   .dependsOn(common % "test->test")
 
-lazy val hybridCarboniteMetadataService = project
-  .withLibrarySettings("hybrid-carbonite-metadata-service", hybridCarboniteMetadataServiceDependencies)
-  .dependsOn(services)
-  .dependsOn(engine)
-  .dependsOn(core % "test->test")
-  .dependsOn(services % "test->test")
-  .dependsOn(ftpFileSystem % "test->test")
-  .dependsOn(httpFileSystem % "test->test")
-  .dependsOn(common % "test->test")
-
-
 lazy val backendRoot = Path("supportedBackends")
 
 lazy val backend = project
@@ -427,7 +416,6 @@ lazy val server = project
   .dependsOn(wdlDraft3LanguageFactory)
   .dependsOn(wdlBiscayneLanguageFactory)
   .dependsOn(cwlV1_0LanguageFactory)
-  .dependsOn(hybridCarboniteMetadataService)
   .dependsOn(engine % "test->test")
   .dependsOn(common % "test->test")
 
@@ -489,5 +477,4 @@ lazy val root = (project in file("."))
   .aggregate(wes2cromwell)
   .aggregate(wom)
   .aggregate(womtool)
-  .aggregate(hybridCarboniteMetadataService)
   .withAggregateSettings()

centaur/src/main/resources/reference.conf

@@ -1,7 +1,5 @@
 # Definitions of new config values. Overloads should go into application.conf.
 centaur {
-
-  expectCarbonite: false
   workflow-progress-timeout: 5 minutes
 
   cromwell {

centaur/src/main/resources/standardTestCases/control_chars.test

@@ -9,4 +9,5 @@ metadata {
   workflowName: control_chars
   status: Succeeded
   "calls.control_chars.control.executionStatus": Done
+  "calls.control_chars.control.outputs.salutation": "“Control characters should work with metadata” — Cromwell"
 }

centaur/src/main/resources/standardTestCases/control_chars/control_chars.wdl

@@ -2,7 +2,7 @@ version 1.0
 
 task control {
   command {
-    echo "“Control characters should work with Carbonited metadata” — Cromwell"
+    echo "“Control characters should work with metadata” — Cromwell"
   }
   output {
     String salutation = read_string(stdout())

centaur/src/main/scala/centaur/CentaurConfig.scala

@@ -50,7 +50,6 @@ object CentaurConfig {
   lazy val conf: Config = ConfigFactory.load().getConfig("centaur")
 
   lazy val runMode: CentaurRunMode = CentaurRunMode(conf)
-  lazy val expectCarbonite: Boolean = conf.getBoolean("expectCarbonite")
 
   lazy val cromwellUrl: URL = runMode.cromwellUrl
   lazy val workflowProgressTimeout: FiniteDuration = conf.getDuration("workflow-progress-timeout").toScala

centaur/src/main/scala/centaur/test/formulas/TestFormulas.scala

@@ -24,7 +24,6 @@ import spray.json.JsString
 object TestFormulas extends StrictLogging {
   logger.info(
     s"""Running with CentaurConfig:
-       |  - expectCarbonite: '${CentaurConfig.expectCarbonite}'
        |  - workflowProgressTimeout: '${CentaurConfig.workflowProgressTimeout}'
        |  - sendReceiveTimeout: '${CentaurConfig.sendReceiveTimeout}'
        |  - maxWorkflowLength: '${CentaurConfig.maxWorkflowLength}'

centaur/test_cromwell.sh

@@ -111,7 +111,7 @@ if [[ -n ${CROMWELL_BRANCH} ]]; then
     git checkout "${CROMWELL_BRANCH}"
     git pull
     echo "Building Cromwell"
-    sbt --warn assembly >> "${ASSEMBLY_LOG}" 2>&1
+    sbt -Dsbt.supershell=false --warn assembly >> "${ASSEMBLY_LOG}" 2>&1
     cd ..
     # This is the "branch" logic but sets the CROMWELL_JAR to be used in either the "branch" or "jar" use cases.
     # Note that this may not be necessary in the docker-compose use case.
@@ -124,11 +124,11 @@ cd "${RUN_DIR}"
 TEST_STATUS="failed"
 
 if [[ "${CENTAUR_SBT_COVERAGE}" == "true" ]]; then
-    sbt --warn coverage centaur/it:compile
-    CP=$(sbt --warn coverage "export centaur/it:dependencyClasspath" -error)
+    sbt -Dsbt.supershell=false --warn coverage centaur/it:compile
+    CP=$(sbt -no-colors --error coverage "export centaur/it:dependencyClasspath")
 else
-    sbt --warn centaur/it:compile
-    CP=$(sbt --warn "export centaur/it:dependencyClasspath" -error)
+    sbt -Dsbt.supershell=false --warn centaur/it:compile
+    CP=$(sbt -no-colors --error "export centaur/it:dependencyClasspath")
 fi
 
 # Add the it-classes folder to the classpath to ensure logback configuration files are picked up.

codegen_java/build.sbt

@@ -6,10 +6,10 @@ lazy val root = (project in file(".")).
     Seq(organization := "org.broadinstitute.cromwell",
     name := "cromwell-client",
     version := createVersion("0.1"),
-    scalaVersion := "2.12.8",
+    scalaVersion := "2.12.12",
     scalacOptions ++= Seq("-feature"),
-    javacOptions in compile ++= Seq("-Xlint:deprecation"),
-    publishArtifact in (Compile, packageDoc) := false,
+    compile / javacOptions ++= Seq("-Xlint:deprecation"),
+    Compile / packageDoc / publishArtifact := false,
     resolvers += Resolver.mavenLocal,
     updateOptions := updateOptions.value.withGigahorse(false),
     libraryDependencies ++= Seq(
@@ -25,4 +25,3 @@ lazy val root = (project in file(".")).
       "com.novocode" % "junit-interface" % "0.10" % "test"
     )) ++ publishSettings:_*
   )
-

codegen_java/project/Publishing.scala

@@ -24,8 +24,8 @@ import Artifactory._
   //priority over the local package cache. see here: https://github.com/sbt/sbt/issues/2687#issuecomment-236586241
     Seq(
       publishTo := Option(artifactoryResolver(false)),
-      publishArtifact in Compile := true,
-      publishArtifact in Test := true,
+      Compile / publishArtifact := true,
+      Test / publishArtifact := true,
       credentials += artifactoryCredentials
     )
 

codegen_java/project/build.properties

@@ -1 +1 @@
-sbt.version=1.2.6
+sbt.version=1.4.9

core/src/main/scala/cromwell/core/WorkflowMetadataKeys.scala

@@ -35,6 +35,4 @@ object WorkflowMetadataKeys {
   val SummaryNameDecreasing = "WORKFLOW_METADATA_SUMMARY_ENTRY_DECREASING"
 
   val Labels = "labels"
-
-  val MetadataSource = "metadataSource"
 }

core/src/main/scala/cromwell/core/path/PathBuilder.scala

@@ -46,7 +46,7 @@ trait Path extends PathObjectMethods with NioPathMethods with BetterFileMethods
     *
     * @return The internal nioPath.
     */
-  protected def nioPath: NioPath
+  def nioPath: NioPath
 
   /**
     * Wraps the nioPath in our container type.