Merge branch 'develop'

.travis.yml

@@ -6,5 +6,5 @@ jdk:
 before_script:
   - mysql -e "create database IF NOT EXISTS cromwell_test;"
 
-script: sbt -Dbackend.shared-filesystem.localization.0=copy clean coverage nodocker:test
+script: sbt -Dbackend.shared-filesystem.localization.0=copy clean coverage nodocker:test assembly
 after_success: sbt coveralls

build.sbt

@@ -3,22 +3,32 @@ import sbtrelease.ReleasePlugin._
 
 name := "cromwell"
 
-version := "0.12"
+version := "0.13"
 
 organization := "org.broadinstitute"
 
 scalaVersion := "2.11.7"
 
+val lenthallV = "0.13"
+
 val sprayV = "1.3.2"
 
 val DowngradedSprayV = "1.3.1"
 
 val akkaV = "2.3.12"
 
+val slickV = "3.1.0"
+
 val googleClientApiV = "1.20.0"
 
+resolvers ++= Seq(
+  "Broad Artifactory Releases" at "https://artifactory.broadinstitute.org/artifactory/libs-release/",
+  "Broad Artifactory Snapshots" at "https://artifactory.broadinstitute.org/artifactory/libs-snapshot/")
+
 libraryDependencies ++= Seq(
-  "com.gettyimages" %% "spray-swagger" % "0.5.1",
+  "org.broadinstitute" %% "lenthall" % lenthallV,
+  "com.typesafe.scala-logging" %% "scala-logging" % "3.1.0",
+  "org.joda" % "joda-convert" % "1.8.1",
   "org.webjars" % "swagger-ui" % "2.1.1",
   "io.spray" %% "spray-can" % sprayV,
   "io.spray" %% "spray-routing" % sprayV,
@@ -33,14 +43,14 @@ libraryDependencies ++= Seq(
   "ch.qos.logback" % "logback-classic" % "1.1.3",
   "ch.qos.logback" % "logback-access" % "1.1.3",
   "org.codehaus.janino" % "janino" % "2.7.8",
-  "com.typesafe.slick" %% "slick" % "3.0.0",
-  "com.zaxxer" % "HikariCP" % "2.3.3",
+  "com.typesafe.slick" %% "slick" % slickV,
+  "com.typesafe.slick" %% "slick-hikaricp" % slickV,
   "org.hsqldb" % "hsqldb" % "2.3.2",
   "com.google.gcloud" % "gcloud-java" % "latest.integration",
   "com.google.api-client" % "google-api-client-java6" % googleClientApiV,
   "com.google.api-client" % "google-api-client-jackson2" % googleClientApiV,
   "com.google.oauth-client" % "google-oauth-client" % googleClientApiV,
-  "mysql" % "mysql-connector-java" % "5.1.35",
+  "mysql" % "mysql-connector-java" % "5.1.36",
   "org.scalaz" % "scalaz-core_2.11" % "7.1.3",
   //---------- Test libraries -------------------//
   "io.spray" %% "spray-testkit" % sprayV % Test,
@@ -81,14 +91,20 @@ val customMergeStrategy: String => MergeStrategy = {
         MergeStrategy.filterDistinctLines
       case _ => MergeStrategy.deduplicate
     }
-  case "asm-license.txt" | "overview.html" =>
+  case "asm-license.txt" | "overview.html" | "cobertura.properties" =>
     MergeStrategy.discard
   case _ => MergeStrategy.deduplicate
 }
 
 assemblyMergeStrategy in assembly := customMergeStrategy
 
-scalacOptions ++= Seq("-deprecation", "-unchecked", "-feature")
+// The reason why -Xmax-classfile-name is set is because this will fail
+// to build on Docker otherwise.  The reason why it's 200 is because it
+// fails if the value is too close to 256 (even 254 fails).  For more info:
+//
+// https://github.com/sbt/sbt-assembly/issues/69
+// https://github.com/scala/pickling/issues/10
+scalacOptions ++= Seq("-deprecation", "-unchecked", "-feature", "-Xmax-classfile-name", "200")
 
 lazy val DockerTest = config("docker") extend Test
 
@@ -107,3 +123,6 @@ testOptions in DockerTest += Tests.Argument("-n", "DockerTest")
 testOptions in NoDockerTest += Tests.Argument("-l", "DockerTest")
 
 test in assembly := {}
+
+parallelExecution := false
+

src/main/resources/application.conf

@@ -8,6 +8,11 @@ akka {
   loggers = ["akka.event.slf4j.Slf4jLogger"]
 }
 
+swagger {
+  docsPath = "swagger/cromwell.yaml"
+  uiVersion = "2.1.1"
+}
+
 spray.can {
   server {
     request-timeout = 40s
@@ -94,30 +99,30 @@ database {
 
   main {
     hsqldb {
-      url = "jdbc:hsqldb:mem:${slick.uniqueSchema};shutdown=false;hsqldb.tx=mvcc"
-      driver = "org.hsqldb.jdbcDriver"
-      slick.driver = "slick.driver.HsqldbDriver"
+      db.url = "jdbc:hsqldb:mem:${slick.uniqueSchema};shutdown=false;hsqldb.tx=mvcc"
+      db.driver = "org.hsqldb.jdbcDriver"
+      driver = "slick.driver.HsqldbDriver$"
       slick.createSchema = true
     }
   }
 
   test {
     hsqldb {
-      url = "jdbc:hsqldb:mem:testdb;shutdown=false;hsqldb.tx=mvcc"
-      driver = "org.hsqldb.jdbcDriver"
-      slick.driver = "slick.driver.HsqldbDriver"
+      db.url = "jdbc:hsqldb:mem:testdb;shutdown=false;hsqldb.tx=mvcc"
+      db.driver = "org.hsqldb.jdbcDriver"
+      driver = "slick.driver.HsqldbDriver$"
       liquibase = {
         changelog = "src/main/migrations/changelog.xml"
         connection = "liquibase.database.jvm.HsqlConnection"
       }
     }
 
     mysql {
-      url = "jdbc:mysql://localhost/cromwell_test"
-      user = "travis"
-      password = ""
-      driver = "com.mysql.jdbc.Driver"
-      slick.driver = "slick.driver.MySQLDriver"
+      db.url = "jdbc:mysql://localhost/cromwell_test"
+      db.user = "travis"
+      db.password = ""
+      db.driver = "com.mysql.jdbc.Driver"
+      driver = "slick.driver.MySQLDriver$"
       liquibase = {
         changelog = "src/main/migrations/changelog.xml"
       }

src/main/scala/cromwell/binding/AstTools.scala

@@ -52,6 +52,28 @@ object AstTools {
     }
 
     def wdlValue(wdlType: WdlType, wdlSyntaxErrorFormatter: WdlSyntaxErrorFormatter): WdlValue = {
+
+      def astToMap(ast: Ast) = {
+        val mapType = wdlType.asInstanceOf[WdlMapType]
+        val elements = ast.getAttribute("map").asInstanceOf[AstList].asScala.toVector.map({ kvnode =>
+          val k = kvnode.asInstanceOf[Ast].getAttribute("key").wdlValue(mapType.keyType, wdlSyntaxErrorFormatter)
+          val v = kvnode.asInstanceOf[Ast].getAttribute("value").wdlValue(mapType.valueType, wdlSyntaxErrorFormatter)
+          k -> v
+        }).toMap
+
+        WdlMap(mapType, elements)
+      }
+
+      def astToObject(ast: Ast) = {
+        val elements = ast.getAttribute("map").asInstanceOf[AstList].asScala.toVector.map({ kvnode =>
+          val k = kvnode.asInstanceOf[Ast].getAttribute("key").sourceString
+          val v = kvnode.asInstanceOf[Ast].getAttribute("value").wdlValue(WdlStringType, wdlSyntaxErrorFormatter)
+          k -> v
+        }).toMap
+
+        WdlObject(elements)
+      }
+
       astNode match {
         case t: Terminal if t.getTerminalStr == "string" && wdlType == WdlStringType => WdlString(t.getSourceString)
         case t: Terminal if t.getTerminalStr == "string" && wdlType == WdlFileType => WdlFile(t.getSourceString)
@@ -61,21 +83,15 @@ object AstTools {
           case "true" => WdlBoolean.True
           case "false" => WdlBoolean.False
         }
-        // TODO: The below cases, ArrayLiteral and MapLiteral are brittle.  They recursively call this wdlValue().
+        // TODO: The below cases, ArrayLiteral and MapLiteral, ObjectLiteral are brittle. They recursively call this wdlValue().
         // However, those recursive calls might contain full-on expressions instead of just other literals.  This
         // whole thing ought to be part of the regular expression evaluator, though I imagine that's non-trivial.
         case a: Ast if a.getName == "ArrayLiteral" && wdlType.isInstanceOf[WdlArrayType] =>
           val arrType = wdlType.asInstanceOf[WdlArrayType]
           val elements = a.getAttribute("values").astListAsVector map {node => node.wdlValue(arrType.memberType, wdlSyntaxErrorFormatter)}
           WdlArray(arrType, elements)
-        case a: Ast if a.getName == "MapLiteral" && wdlType.isInstanceOf[WdlMapType] =>
-          val mapType = wdlType.asInstanceOf[WdlMapType]
-          val elements = a.getAttribute("map").asInstanceOf[AstList].asScala.toVector.map({ kvnode =>
-            val k = kvnode.asInstanceOf[Ast].getAttribute("key").wdlValue(mapType.keyType, wdlSyntaxErrorFormatter)
-            val v = kvnode.asInstanceOf[Ast].getAttribute("value").wdlValue(mapType.valueType, wdlSyntaxErrorFormatter)
-            k -> v
-          }).toMap
-          WdlMap(mapType, elements)
+        case a: Ast if a.getName == "MapLiteral" && wdlType.isInstanceOf[WdlMapType] => astToMap(a)
+        case a: Ast if a.getName == "ObjectLiteral" && wdlType == WdlObjectType => astToObject(a)
         case _ => throw new SyntaxError(s"Could not convert AST to a $wdlType (${Option(astNode).getOrElse("No AST").toString})")
       }
     }

src/main/scala/cromwell/binding/WdlExpression.scala

@@ -26,6 +26,7 @@ object WdlExpression {
     def isMemberAccess: Boolean = ast.getName == "MemberAccess"
     def isArrayLiteral: Boolean = ast.getName == "ArrayLiteral"
     def isMapLiteral: Boolean = ast.getName == "MapLiteral"
+    def isObjectLiteral: Boolean = ast.getName == "ObjectLiteral"
     def isArrayOrMapLookup: Boolean = ast.getName == "ArrayOrMapLookup"
     def params = ast.getAttribute("params").asInstanceOf[AstList].asScala.toVector
     def name = ast.getAttribute("name").asInstanceOf[Terminal].getSourceString
@@ -70,7 +71,8 @@ object WdlExpression {
     "read_float",
     "read_boolean",
     "read_lines",
-    "read_map"
+    "read_map",
+    "read_object"
   )
 
   def evaluate(ast: AstNode, lookup: ScopedLookupFunction, functions: WdlFunctions[WdlValue]): Try[WdlValue] =

src/main/scala/cromwell/binding/WdlNamespace.scala

@@ -3,7 +3,7 @@ package cromwell.binding
 import java.io.File
 
 import cromwell.binding.AstTools.{AstNodeName, EnhancedAstNode, EnhancedAstSeq}
-import cromwell.binding.expression.NoFunctions
+import cromwell.binding.expression.WdlStandardLibraryFunctions
 import cromwell.binding.types._
 import cromwell.binding.values._
 import cromwell.parser.WdlParser._
@@ -107,13 +107,13 @@ case class NamespaceWithWorkflow(importedAs: Option[String],
    * For the declarations that have an expression attached to it already, evaluate the expression
    * and return the value for storage in the symbol store
    */
-  def staticDeclarationsRecursive(userInputs: WorkflowCoercedInputs): Try[WorkflowCoercedInputs] = {
+  def staticDeclarationsRecursive(userInputs: WorkflowCoercedInputs, wdlFunctions: WdlStandardLibraryFunctions): Try[WorkflowCoercedInputs] = {
     import scala.collection.mutable
     val collected = mutable.Map[String, WdlValue]()
     val allDeclarations = workflow.declarations ++ workflow.calls.flatMap {_.task.declarations}
 
     val evaluatedDeclarations = allDeclarations.filter {_.expression.isDefined}.map {decl =>
-      val value = decl.expression.get.evaluate(declarationLookupFunction(decl, collected.toMap ++ userInputs), new NoFunctions)
+      val value = decl.expression.get.evaluate(declarationLookupFunction(decl, collected.toMap ++ userInputs), wdlFunctions)
       collected += (decl.fullyQualifiedName -> value.get)
       val coercedValue = value match {
         case Success(s) => decl.wdlType.coerceRawValue(s)

src/main/scala/cromwell/binding/command/ParameterCommandPart.scala

@@ -33,7 +33,7 @@ case class ParameterCommandPart(attributes: Map[String, String], expression: Wdl
           case Some(d) => throw new UnsupportedOperationException(s"Parameter ${v.variable} is required, but no value is specified")
           case None => throw new UnsupportedOperationException(s"This should not happen: could not find declaration for ${v.variable}")
         }
-        case _ => throw new UnsupportedOperationException(s"Could not evaluate expression: ${expression.toString}")
+        case e => throw new UnsupportedOperationException(s"Could not evaluate expression: ${expression.toWdlString}", e)
       }
     }
 

src/main/scala/cromwell/binding/expression/WdlStandardLibraryFunctions.scala

@@ -17,10 +17,22 @@ trait WdlStandardLibraryFunctions extends WdlFunctions[WdlValue] {
   protected def read_object(params: Seq[Try[WdlValue]]): Try[WdlObject] = fail("read_objects")
   protected def read_objects(params: Seq[Try[WdlValue]]): Try[WdlArray] = fail("read_objects")
   protected def read_json(params: Seq[Try[WdlValue]]): Try[WdlValue] = fail("read_json")
-  protected def read_int(params: Seq[Try[WdlValue]]): Try[WdlInteger] = fail("read_int")
   protected def read_string(params: Seq[Try[WdlValue]]): Try[WdlString] = fail("read_string")
-  protected def read_float(params: Seq[Try[WdlValue]]): Try[WdlFloat] = fail("read_float")
-  protected def read_boolean(params: Seq[Try[WdlValue]]): Try[WdlBoolean] = fail("read_boolean")
+  /**
+   * Try to read an integer from the file referenced by the specified `WdlValue`.
+   */
+  protected def read_int(params: Seq[Try[WdlValue]]): Try[WdlInteger] =
+    read_string(params) map { s => WdlInteger(s.value.trim.toInt) }
+  /**
+   * Try to read a float from the file referenced by the specified `WdlValue`.
+   */
+  protected def read_float(params: Seq[Try[WdlValue]]): Try[WdlFloat] =
+    read_string(params) map { s => WdlFloat(s.value.trim.toDouble) }
+  /**
+   * Try to read a boolean from the file referenced by the specified `WdlValue`.
+   */
+  protected def read_boolean(params: Seq[Try[WdlValue]]): Try[WdlBoolean] =
+    read_string(params) map { s => WdlBoolean(java.lang.Boolean.parseBoolean(s.value.trim.toLowerCase)) }
   protected def write_lines(params: Seq[Try[WdlValue]]): Try[WdlFile] = fail("write_lines")
   protected def write_tsv(params: Seq[Try[WdlValue]]): Try[WdlFile] = fail("write_tsv")
   protected def write_map(params: Seq[Try[WdlValue]]): Try[WdlFile] = fail("write_map")

src/main/scala/cromwell/binding/types/WdlArrayType.scala

@@ -1,6 +1,6 @@
 package cromwell.binding.types
 
-import cromwell.binding.values.{WdlArray, WdlFile, WdlString}
+import cromwell.binding.values.{WdlValue, WdlArray, WdlFile, WdlString}
 import spray.json.JsArray
 
 case class WdlArrayType(memberType: WdlType) extends WdlType {
@@ -27,3 +27,14 @@ case class WdlArrayType(memberType: WdlType) extends WdlType {
     case _ => false
   }
 }
+
+object WdlArrayType {
+
+  implicit class WdlArrayEnhanced(wdlType: WdlType) extends WdlType {
+
+    override protected def coercion: PartialFunction[Any, WdlValue] = wdlType.coercion
+    override def toWdlString: String = wdlType.toWdlString
+
+    def isAnArrayOf(genericType: WdlType) = wdlType.isInstanceOf[WdlArrayType] && wdlType.asInstanceOf[WdlArrayType].memberType == genericType
+  }
+}

src/main/scala/cromwell/binding/types/WdlMapType.scala

@@ -11,10 +11,12 @@ case class WdlMapType(keyType: WdlType, valueType: WdlType) extends WdlType {
     case m: Map[_, _] if m.isEmpty => WdlMap(WdlMapType(keyType, valueType), Map())
     case js: JsObject if js.fields.nonEmpty => WdlMap.coerceMap(js.fields, this)
     case wdlMap: WdlMap => WdlMap.coerceMap(wdlMap.value, this)
+    case o: WdlObject => WdlMap.coerceMap(o.value, this)
   }
 
   override def isCoerceableFrom(otherType: WdlType): Boolean = otherType match {
     case m: WdlMapType => keyType.isCoerceableFrom(m.keyType) && valueType.isCoerceableFrom(m.valueType)
+    case WdlObjectType => keyType.isCoerceableFrom(WdlStringType) && valueType.isCoerceableFrom(WdlStringType)
     case _ => false
   }
 }

src/main/scala/cromwell/binding/types/WdlObjectType.scala

@@ -1,16 +1,52 @@
 package cromwell.binding.types
 
 import cromwell.binding.Call
-import cromwell.binding.values.{WdlCallOutputsObject, WdlObject}
+import cromwell.binding.values._
+import spray.json.JsObject
+
+import scala.util.{Try, Failure, Success}
 
 case object WdlObjectType extends WdlType {
   val toWdlString: String = "Object"
 
+  private def handleCoercionFailures(tries: Try[_]*) = {
+    val errorMessages = tries collect {
+      case Failure(f) => f.getMessage
+    } mkString ","
+
+    throw new UnsupportedOperationException(s"Coercion failed: $errorMessages")
+  }
+
   override protected def coercion = {
     case o: WdlObject => o
+    case m: WdlMap if isMapCoercable(m) =>
+      val coercedMap = m.value map {
+        case (k, v) => toWdlString(k) -> toWdlString(v)
+      } collect {
+        case (Success(k), Success(v)) => k.value -> v
+        case (k, v) => handleCoercionFailures(k, v)
+      }
+
+      WdlObject(coercedMap)
+    case js: JsObject =>
+      val coercedMap = WdlMap.coerceMap(js.fields, WdlMapType(WdlStringType, WdlAnyType)).value map {
+        // get is safe because coerceMap above would have failed already if k was not coerceable to WdlString
+        case (k, v) => toWdlString(k).get.value -> v
+      }
+
+      WdlObject(coercedMap)
+  }
+
+  private def toWdlString(v: WdlValue) = WdlStringType.coerceRawValue(v).map(_.asInstanceOf[WdlString])
+
+  override def isCoerceableFrom(otherType: WdlType) = otherType match {
+    case WdlObjectType => true
+    case t: WdlMapType if isMapTypeCoercable(t) => true
+    case _ => false
   }
 
-  override def fromWdlString(rawString: String) = ???
+  def isMapTypeCoercable(t: WdlMapType) = WdlStringType.isCoerceableFrom(t.keyType) && WdlStringType.isCoerceableFrom(t.valueType)
+  def isMapCoercable(m: WdlMap) = isMapTypeCoercable(m.wdlType)
 }
 
 case class WdlCallOutputsObjectType(call: Call) extends WdlType {

src/main/scala/cromwell/binding/types/WdlType.scala

@@ -43,6 +43,7 @@ trait WdlType {
    * @param wdlSource source code representing the WdlValue
    * @return The WdlValue
    */
+  //TODO: return a Try ?
   def fromWdlString(wdlSource: WdlSource): WdlValue = {
     val tokens = WdlType.parser.lex(wdlSource, "string")
     val terminalMap = tokens.asScala.toVector.map {(_, wdlSource)}.toMap

src/main/scala/cromwell/binding/values/WdlArray.scala

@@ -1,6 +1,6 @@
 package cromwell.binding.values
 
-import cromwell.binding.types.{WdlArrayType, WdlPrimitiveType}
+import cromwell.binding.types.{WdlObjectType, WdlArrayType, WdlPrimitiveType}
 
 import scala.util.{Failure, Success, Try}
 
@@ -25,6 +25,7 @@ case class WdlArray(wdlType: WdlArrayType, value: Seq[WdlValue]) extends WdlValu
   def tsvSerialize: Try[String] = {
     wdlType.memberType match {
       case t: WdlPrimitiveType => Success(value.map(_.valueString).mkString("\n"))
+      case WdlObjectType => WdlObject.tsvSerializeArray(value map { _.asInstanceOf[WdlObject] })
       case _ => Failure(new UnsupportedOperationException("Can only TSV serialize an Array[Primitive]"))
     }
   }