Defaults command line options

Reimplementing the options to set key-value defaults.

README.md

@@ -40,18 +40,19 @@ argument ```X```, then: ``` dx run foo -i0.X="hello world" ```
 
 Compilation can be controled with several parameters.
 
-| Option  |  Description |
-| ------  | ------------ |
-| archive | Archive older versions of applets (*dx build -a*)|
+| Option   |  Description |
+| ------   | ------------ |
+| archive  | Archive older versions of applets and workflows |
+| defaults | A file with default parameter settings. The syntax is Cromwell style. |
 | destination | Set the output folder on the platform |
-| inputs |  Use a cromwell style inputs file |
-| force   | Delete existing applets/workflows |
-| sort    | Sort call graph, to avoid forward references, used for CWL |
-| verbose | Print detailed progress information |
+| force    | Overwrite existing applets/workflows if they have changed |
+| inputs   | A cromwell style inputs file |
+| sort     | Sort call graph, to avoid forward references, used for CWL |
+| verbose  | Print detailed progress information |
 
 The `-inputs` option allows specifying a Cromwell JSON
 [format](https://software.broadinstitute.org/wdl/documentation/inputs.php)
-inputs file.  An equivalent DNAx format inputs file is generated from
+inputs file. An equivalent DNAx format inputs file is generated from
 it. For example, workflow
 [files](https://github.com/dnanexus-rnd/dxWDL/blob/master/test/files.wdl)
 has input file
@@ -88,6 +89,18 @@ The workflow can then be run with the command:
 dx run files -f test/files_input.dx.json
 ```
 
+The `-defaults` option is similar to `-inputs`. It takes a JSON file with key-value pairs,
+and compiles them as defaults into the workflow. If the `files.wdl` worklow is compiled with
+`-defaults` instead of `-inputs`
+```
+java -jar dxWDL-0.44.jar compile test/files.wdl -defaults test/files_input.json
+```
+
+It can be run without parameters, for an equivalent execution.
+```
+dx run files
+```
+
 ## Extensions (experimental)
 
 A task declaration has a runtime section where memory, cpu, and disk
@@ -164,11 +177,11 @@ is as follows:
 | Array[Float]   |   array:float |
 | Array[String]  |   array:string |
 | Array[File]    |   array:file |
-| Complex types  |   file + array:file |
+| Complex types  |   hash + array:file |
 
 Ragged arrays of files (Array[Array[File]]), and other more complex
 WDL types, are mapped to two fields: a flat array of files, and a
-file, which is a json serialized representation of the WDL value. The
+hash, which is a json serialized representation of the WDL value. The
 flat file array informs the job manager about data objects that need to
 be closed and cloned into the workspace.
 

RELEASE_NOTES.md

@@ -1,5 +1,10 @@
 # Release Notes
 
+## 0.45
+- Default workflow inputs. The `--defaults` command line argument
+embeds key-value pairs as workflow defaults. They can be overridden
+at runtime if necessary.
+
 ## 0.44
 - Use hashes instead of files for non-native dx types
 - Do not use the help field in applet input/output arguments to carry

reference_stanza.conf

@@ -1,4 +1,4 @@
 dxWDL {
-    version = "0.44"
+    version = "0.45"
     asset_ids = []
 }

src/main/scala/dxWDL/CompilerIR.scala

@@ -37,6 +37,14 @@ case class CompilerIR(gWorkflowOutputs: Option[Seq[WorkflowOutput]],
     // Environment (scope) where a call is made
     type CallEnv = Map[String, LinkedVar]
 
+    // generate a stage Id
+    var stageNum = 0
+    def genStageId() : Utils.DXWorkflowStage = {
+        val retval = Utils.DXWorkflowStage(s"stage_${stageNum}")
+        stageNum += 1
+        retval
+    }
+
     // Convert the environment to yaml, and then pretty
     // print it.
     def prettyPrint(env: CallEnv) : String = {
@@ -382,7 +390,7 @@ task Add {
                                WdlRewrite.namespace(code, Seq.empty))
         verifyWdlCodeIsLegal(applet.ns)
 
-        (IR.Stage(appletName, appletName, Vector[IR.SArg](), outputVars),
+        (IR.Stage(appletName, genStageId(), appletName, Vector[IR.SArg](), outputVars),
          applet)
     }
 
@@ -458,7 +466,7 @@ workflow w {
         // Link to the X.y original variables
         val inputs: Vector[IR.SArg] = closure.map{ case (_, lVar) => lVar.sArg }.toVector
 
-        (IR.Stage(appletName, appletName, inputs, outputVars),
+        (IR.Stage(appletName, genStageId(), appletName, inputs, outputVars),
          applet)
     }
 
@@ -543,7 +551,7 @@ workflow w {
         // Link to the X.y original variables
         val inputs: Vector[IR.SArg] = closure.map{ case (_, lVar) => lVar.sArg }.toVector
 
-        (IR.Stage(appletName, appletName, inputs, outputVars),
+        (IR.Stage(appletName, genStageId(), appletName, inputs, outputVars),
          applet)
     }
 
@@ -646,7 +654,7 @@ workflow w {
         }
 
         val stageName = callUniqueName(call)
-        IR.Stage(stageName, task.name, inputs, callee.outputs)
+        IR.Stage(stageName, genStageId(), task.name, inputs, callee.outputs)
     }
 
     // Split a block (Scatter, If, ..) into the top declarations,
@@ -945,7 +953,7 @@ workflow w {
         val sargs : Vector[IR.SArg] = closure.map {
             case (_, LinkedVar(_, sArg)) => sArg
         }.toVector
-        (IR.Stage(stageName, applet.name, sargs, outputVars),
+        (IR.Stage(stageName, genStageId(), applet.name, sargs, outputVars),
          applet)
     }
 
@@ -991,7 +999,7 @@ workflow w {
         val sargs : Vector[IR.SArg] = closure.map {
             case (_, LinkedVar(_, sArg)) => sArg
         }.toVector
-        (IR.Stage(stageName, applet.name, sargs, outputVars),
+        (IR.Stage(stageName, genStageId(), applet.name, sargs, outputVars),
          applet)
     }
 
@@ -1022,23 +1030,23 @@ workflow w {
 
         // Create a stage per call/scatter-block/declaration-block
         val subBlocks = splitIntoBlocks(wfBody)
-        val initAccu : Vector[(IR.Stage, Option[IR.Applet])] =
-            Vector((commonStage, Some(commonApplet)))
+        val initAccu : (Vector[(IR.Stage, Option[IR.Applet])]) =
+            (Vector((commonStage, Some(commonApplet))))
         val allStageInfo = subBlocks.foldLeft(initAccu) { (accu, child) =>
             val (stage, appletOpt) = child match {
                 case BlockDecl(decls) =>
-                    evalAppletNum = evalAppletNum + 1
+                    evalAppletNum += 1
                     val appletName = wf.unqualifiedName ++ "_eval" ++ evalAppletNum.toString
                     val (stage, applet) = compileEvalAndPassClosure(appletName, decls, env)
                     (stage, Some(applet))
                 case BlockIf(preDecls, cond) =>
-                    condNum = condNum + 1
+                    condNum += 1
                     val condName = Utils.IF ++ "_" ++ condNum.toString
                     val (stage, applet) = compileIf(wf.unqualifiedName, condName, preDecls,
                                                     cond, taskApplets, env)
                     (stage, Some(applet))
                 case BlockScatter(preDecls, scatter) =>
-                    scatterNum = scatterNum + 1
+                    scatterNum += 1
                     val scatterName = Utils.SCATTER ++ "_" ++ scatterNum.toString
                     val (stage, applet) = compileScatter(wf.unqualifiedName, scatterName, preDecls,
                                                          scatter, taskApplets, env)
@@ -1050,6 +1058,7 @@ workflow w {
                     throw new Exception(cef.notCurrentlySupported(
                                             x.ast, s"Workflow element type=${x}"))
             }
+
             // Add bindings for the output variables. This allows later calls to refer
             // to these results. In case of scatters, there is no block name to reference.
             for (cVar <- stage.outputs) {

src/main/scala/dxWDL/CompilerNative.scala

@@ -11,7 +11,6 @@ import scala.collection.JavaConverters._
 import spray.json._
 import Utils.{AppletLinkInfo, base64Encode, CHECKSUM_PROP, dxFileOfJsValue, DXWorkflowStage,
     INSTANCE_TYPE_DB_FILENAME, jsValueOfJsonNode, jsonNodeOfJsValue, LINK_INFO_FILENAME, trace}
-import wdl4s.parser.WdlParser.Ast
 import wdl4s.wdl.types._
 
 case class CompilerNative(dxWDLrtId: String,
@@ -57,18 +56,22 @@ case class CompilerNative(dxWDLrtId: String,
     // Ragged arrays, maps, and objects, cannot be mapped in such a trivial way.
     // These are called "Complex Types", or "Complex". They are handled
     // by passing a JSON structure and a vector of dx:files.
-    def wdlVarToSpec(varName: String,
-                     wdlType : WdlType,
-                     ast: Ast) : Vector[JsValue] = {
-        val name = Utils.encodeAppletVarName(varName)
+    def cVarToSpec(cVar: IR.CVar) : Vector[JsValue] = {
+        val name = Utils.encodeAppletVarName(cVar.dxVarName)
+        val defaultVal:Map[String, JsValue] = cVar.attrs.getDefault match {
+            case None => Map.empty
+            case Some(v) => Map("default" -> v)
+        }
         def mkPrimitive(dxType: String) : Vector[Map[String, JsValue]] = {
             Vector(Map("name" -> JsString(name),
-                       "class" -> JsString(dxType)))
+                       "class" -> JsString(dxType))
+                       ++ defaultVal)
         }
         def mkPrimitiveArray(dxType: String) : Vector[Map[String, JsValue]] = {
             Vector(Map("name" -> JsString(name),
                        "class" -> JsString("array:" ++ dxType),
-                       "optional" -> JsBoolean(true)))
+                       "optional" -> JsBoolean(true))
+                       ++ defaultVal)
         }
         def mkComplex() : Vector[Map[String,JsValue]] = {
             // A large JSON structure passed as a hash, and a
@@ -78,7 +81,8 @@ case class CompilerNative(dxWDLrtId: String,
             // so that the WdlVarLinks.loadJobInputsAsLinks method
             // will not interpret it.
             Vector(Map("name" -> JsString(name),
-                       "class" -> JsString("hash")),
+                       "class" -> JsString("hash"))
+                       ++ defaultVal,
                    Map("name" -> JsString(name + Utils.FLAT_FILES_SUFFIX),
                        "class" -> JsString("array:file"),
                        "optional" -> JsBoolean(true)))
@@ -104,8 +108,8 @@ case class CompilerNative(dxWDLrtId: String,
             }
         }
 
-        val vec: Vector[Map[String,JsValue]] = nonOptional(Utils.stripOptional(wdlType))
-        wdlType match {
+        val vec: Vector[Map[String,JsValue]] = nonOptional(Utils.stripOptional(cVar.wdlType))
+        cVar.wdlType match {
             case WdlOptionalType(t) =>
                 // An optional variable, make it an optional dx input/output
                 vec.map{ m => JsObject(m + ("optional" -> JsBoolean(true))) }
@@ -473,10 +477,10 @@ case class CompilerNative(dxWDLrtId: String,
         trace(verbose.on, s"Building /applet/new request for ${applet.name}")
 
         val inputSpec : Vector[JsValue] = applet.inputs.map(cVar =>
-            wdlVarToSpec(cVar.dxVarName, cVar.wdlType, cVar.ast)
+            cVarToSpec(cVar)
         ).flatten.toVector
         val outputSpec : Vector[JsValue] = applet.outputs.map(cVar =>
-            wdlVarToSpec(cVar.dxVarName, cVar.wdlType, cVar.ast)
+            cVarToSpec(cVar)
         ).flatten.toVector
         val runSpec : JsValue = calcRunSpec(bashScript, applet.instanceType)
 
@@ -643,16 +647,15 @@ case class CompilerNative(dxWDLrtId: String,
                 val (irApplet,dxApplet) = appletDict(stg.appletName)
                 val linkedInputs : Vector[(IR.CVar, IR.SArg)] = irApplet.inputs zip stg.inputs
                 val inputs = genStageInputs(linkedInputs, irApplet, stageDict)
-                val stgId = DXWorkflowStage(s"stage_${version}")
                 // convert the per-stage metadata into JSON
                 val stageReqDesc = JsObject(
-                    "id" -> JsString(stgId.getId),
+                    "id" -> JsString(stg.id.getId),
                     "executable" -> JsString(dxApplet.getId),
                     "name" -> JsString(stg.name),
                     "input" -> inputs)
                 (version + 1,
                  stagesReq :+ stageReqDesc,
-                 stageDict ++ Map(stg.name -> stgId))
+                 stageDict ++ Map(stg.name -> stg.id))
         }
 
         // pack all the arguments into a single API call

src/main/scala/dxWDL/DeclAttrs.scala

@@ -12,6 +12,15 @@ case class DeclAttrs(m: Map[String, JsValue]) {
             case _ => false
         }
     }
+
+    def getDefault: Option[JsValue] = {
+        m.get("default")
+    }
+
+    // add another attribute
+    def add(key:String, value:JsValue) : DeclAttrs = {
+        DeclAttrs(m + (key -> value))
+    }
 }
 
 object DeclAttrs {