Merge pull request #1183 from virtualcell/TrackSubTasksInReports

Added SubTask tracking to match biosimulators expectations.
virtualcell · Mar 8, 2024 · 20678d5 · 20678d5
2 parents 42744da + 75b4ad0
commit 20678d5
Show file tree

Hide file tree

Showing 7 changed files with 37 additions and 16 deletions.
diff --git a/vcell-cli/src/main/java/org/vcell/cli/biosimulation/BiosimulationsCommand.java b/vcell-cli/src/main/java/org/vcell/cli/biosimulation/BiosimulationsCommand.java
@@ -92,7 +92,7 @@ public Integer call() {
             logger.info("Beginning execution");
             try {
                 CLIPythonManager.getInstance().instantiatePythonProcess();
-                ExecuteImpl.singleMode(ARCHIVE, OUT_DIR, cliRecorder);
+                ExecuteImpl.singleMode(ARCHIVE, OUT_DIR, cliRecorder, true);
                 if (Tracer.hasErrors()){
                     if (!bQuiet) {
                         logger.error("Errors occurred during execution");

diff --git a/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteImpl.java b/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteImpl.java
@@ -110,13 +110,20 @@ private static void runSingleExecOmex(File inputFile, File outputDir, CLIRecorda
         Files.createDirectories(Paths.get(outputDir.getAbsolutePath() + File.separator + bioModelBaseName)); // make output subdir
         final boolean bEncapsulateOutput = true;
 
-        singleExecOmex(inputFile, outputDir, cliLogger,
-                bKeepTempFiles, bExactMatchOnly, bEncapsulateOutput, bSmallMeshOverride);
+        ExecuteImpl.singleExecOmex(inputFile, outputDir, cliLogger,
+                bKeepTempFiles, bExactMatchOnly, bEncapsulateOutput, bSmallMeshOverride, false);
     }
 
     public static void singleMode(File inputFile, File rootOutputDir, CLIRecordable cliLogger,
-            boolean bKeepTempFiles, boolean bExactMatchOnly, boolean bEncapsulateOutput, boolean bSmallMeshOverride
-    ) throws Exception {
+                                  boolean bKeepTempFiles, boolean bExactMatchOnly, boolean bEncapsulateOutput,
+                                  boolean bSmallMeshOverride) throws Exception{
+        ExecuteImpl.singleMode(inputFile, rootOutputDir, cliLogger, bKeepTempFiles, bExactMatchOnly,
+                bEncapsulateOutput, bSmallMeshOverride, false);
+    }
+
+    public static void singleMode(File inputFile, File rootOutputDir, CLIRecordable cliLogger,
+            boolean bKeepTempFiles, boolean bExactMatchOnly, boolean bEncapsulateOutput,
+          boolean bSmallMeshOverride, boolean bBioSimMode) throws Exception {
         // Build statuses
         String bioModelBaseName = FileUtils.getBaseName(inputFile.getName()); // bioModelBaseName = input file without the path
         String outputBaseDir = rootOutputDir.getAbsolutePath(); 
@@ -132,10 +139,14 @@ public static void singleMode(File inputFile, File rootOutputDir, CLIRecordable
         PythonCalls.generateStatusYaml(inputFile.getAbsolutePath(), targetOutputDir);    // generate Status YAML
 
         ExecuteImpl.singleExecOmex(inputFile, rootOutputDir, cliLogger, bKeepTempFiles, bExactMatchOnly,
-                bEncapsulateOutput, bSmallMeshOverride);
+                bEncapsulateOutput, bSmallMeshOverride, bBioSimMode);
     }
 
     public static void singleMode(File inputFile, File outputDir, CLIRecordable cliLogger) throws Exception {
+        ExecuteImpl.singleMode(inputFile, outputDir, cliLogger, false);
+    }
+
+    public static void singleMode(File inputFile, File outputDir, CLIRecordable cliLogger, boolean bioSimMode) throws Exception {
         final boolean bKeepTempFiles = false;
         final boolean bExactMatchOnly = false;
         final boolean bEncapsulateOutput = false;
@@ -145,7 +156,7 @@ public static void singleMode(File inputFile, File outputDir, CLIRecordable cliL
         try {
             span = Tracer.startSpan(Span.ContextType.OMEX_EXECUTE, inputFile.getName(), Map.of("filename", inputFile.getName()));
             ExecuteImpl.singleMode(inputFile, outputDir, cliLogger, bKeepTempFiles, bExactMatchOnly,
-                    bEncapsulateOutput, bSmallMeshOverride);
+                    bEncapsulateOutput, bSmallMeshOverride, bioSimMode);
         } catch (Exception e) {
             Tracer.failure(e, "error message");
             throw e;
@@ -218,13 +229,13 @@ public static void singleExecVcml(File vcmlFile, File outputDir, CLIRecordable c
     }
 
     private static void singleExecOmex(File inputFile, File rootOutputDir, CLIRecordable cliRecorder,
-            boolean bKeepTempFiles, boolean bExactMatchOnly, boolean bEncapsulateOutput, boolean bSmallMeshOverride)
+            boolean bKeepTempFiles, boolean bExactMatchOnly, boolean bEncapsulateOutput, boolean bSmallMeshOverride, boolean bBioSimMode)
             throws ExecutionException, PythonStreamException, IOException, InterruptedException, HDF5Exception {
 
         ExecutionJob requestedExecution = new ExecutionJob(inputFile, rootOutputDir, cliRecorder,
             bKeepTempFiles, bExactMatchOnly, bEncapsulateOutput, bSmallMeshOverride);
         requestedExecution.preprocessArchive();
-        requestedExecution.executeArchive();
+        requestedExecution.executeArchive(bBioSimMode);
         requestedExecution.postProcessessArchive();
     }
 

diff --git a/vcell-cli/src/main/java/org/vcell/cli/run/ExecutionJob.java b/vcell-cli/src/main/java/org/vcell/cli/run/ExecutionJob.java
@@ -122,9 +122,9 @@ public void preprocessArchive() throws PythonStreamException, IOException {
      * @throws IOException if there are system I/O issues
      * @throws ExecutionException if an execution specfic error occurs
      */
-    public void executeArchive() throws HDF5Exception, PythonStreamException, ExecutionException {
+    public void executeArchive(boolean isBioSimSedml) throws HDF5Exception, PythonStreamException, ExecutionException {
         try {
-            Hdf5DataContainer masterHdf5File = new Hdf5DataContainer();
+            Hdf5DataContainer masterHdf5File = new Hdf5DataContainer(isBioSimSedml);
             this.queueAllSedml();
 
             for (String sedmlLocation : this.sedmlLocations){

diff --git a/vcell-cli/src/main/java/org/vcell/cli/run/SedmlJob.java b/vcell-cli/src/main/java/org/vcell/cli/run/SedmlJob.java
@@ -382,7 +382,7 @@ private void generatePlots() throws PythonStreamException, InterruptedException,
         }
     }
 
-    private void generateHDF5(SolverHandler solverHandler, Hdf5DataContainer masterHdf5File) throws IOException {
+    private void generateHDF5(SolverHandler solverHandler, Hdf5DataContainer masterHdf5File) {
         this.logDocumentMessage += "Generating HDF5 file... ";
         logger.info("Generating HDF5 file... ");
 

diff --git a/vcell-cli/src/main/java/org/vcell/cli/run/hdf5/Hdf5DataContainer.java b/vcell-cli/src/main/java/org/vcell/cli/run/hdf5/Hdf5DataContainer.java
@@ -12,12 +12,18 @@ public class Hdf5DataContainer {
      *  Uri to results map
      */
     public Map<String, List<Hdf5SedmlResults>> uriToResultsMap;
+    public boolean trackSubSetsInReports;
 
     /**
      * Basic constructor
      */
     public Hdf5DataContainer(){
+        this(false);
+    }
+
+    public Hdf5DataContainer(boolean shouldTrackSedmlSubSetsInReports){
         this.uriToResultsMap = new HashMap<>();
+        this.trackSubSetsInReports = shouldTrackSedmlSubSetsInReports;
     }
 
     /**

diff --git a/vcell-cli/src/main/java/org/vcell/cli/run/hdf5/Hdf5DataPreparer.java b/vcell-cli/src/main/java/org/vcell/cli/run/hdf5/Hdf5DataPreparer.java
@@ -40,7 +40,7 @@ public static double[] getSpatialHdf5Attribute_Times(Hdf5SedmlResults hdf5SedmlR
      * @param datasetWrapper the data relevant to an HDF5 output file
      * @return the prepared spatial data
      */
-    public static Hdf5PreparedData prepareSpatialData (Hdf5SedmlResults datasetWrapper){
+    public static Hdf5PreparedData prepareSpatialData (Hdf5SedmlResults datasetWrapper, boolean reportSubSets){
         int totalDataVolume = 1;
         int numJobs = 1;
         double[] bigDataBuffer;
@@ -84,6 +84,7 @@ public static Hdf5PreparedData prepareSpatialData (Hdf5SedmlResults datasetWrapp
         for (int scanBound : spatialDataResults.scanBounds){
             dataDimensionList.add((long)scanBound + 1);         // ...then by scan bounds / "repeated task"
             numJobs *= (scanBound+1);
+            if (reportSubSets) dataDimensionList.add((long)1);  // ...then by num of subtasks (VCell only supports 1)
         }
         for (long dim : spaceTimeDimensions){                   // ...finally by space-time dimensions
             dataDimensionList.add(dim);
@@ -125,7 +126,7 @@ public static Hdf5PreparedData prepareSpatialData (Hdf5SedmlResults datasetWrapp
      * @param datasetWrapper the data relevant to an hdf5 output file
      * @return the prepared nonspatial data
      */
-    public static Hdf5PreparedData prepareNonspatialData(Hdf5SedmlResults datasetWrapper){
+    public static Hdf5PreparedData prepareNonspatialData(Hdf5SedmlResults datasetWrapper, boolean reportSubSets){
         long numTimePoints = 0;
         int totalDataVolume = 1;
         long numDataSets;
@@ -146,6 +147,7 @@ public static Hdf5PreparedData prepareNonspatialData(Hdf5SedmlResults datasetWra
         dataDimensionList.add(numDataSets);                     // ...first by dataSet
         for (int scanBound : dataSourceNonspatial.scanBounds){
             dataDimensionList.add((long)scanBound + 1);         // ...then by scan bounds / "repeated task" dimensions
+            if (reportSubSets) dataDimensionList.add((long)1);  // ...then by num of subtasks (VCell only supports 1)
         }
         dataDimensionList.add(numTimePoints);                   // ...finally by max time points
 

diff --git a/vcell-cli/src/main/java/org/vcell/cli/run/hdf5/Hdf5Writer.java b/vcell-cli/src/main/java/org/vcell/cli/run/hdf5/Hdf5Writer.java
@@ -62,8 +62,10 @@ public static void writeHdf5(Hdf5DataContainer hdf5DataWrapper, File outDirForCu
                 // Process the Dataset
                 for (Hdf5SedmlResults data : hdf5DataWrapper.uriToResultsMap.get(rawPath)){
                     Hdf5PreparedData preparedData;
-                    if (data.dataSource instanceof Hdf5SedmlResultsNonspatial) preparedData = Hdf5DataPreparer.prepareNonspatialData(data);
-                    else if (data.dataSource instanceof Hdf5SedmlResultsSpatial) preparedData = Hdf5DataPreparer.prepareSpatialData(data);
+                    if (data.dataSource instanceof Hdf5SedmlResultsNonspatial)
+                        preparedData = Hdf5DataPreparer.prepareNonspatialData(data, hdf5DataWrapper.trackSubSetsInReports);
+                    else if (data.dataSource instanceof Hdf5SedmlResultsSpatial)
+                        preparedData = Hdf5DataPreparer.prepareSpatialData(data, hdf5DataWrapper.trackSubSetsInReports);
                     else continue;
 
                     int currentDatasetId = masterHdf5.insertSedmlData(path, preparedData);