From b82881c09cbd4901535fc58af1750226bd6dd0fb Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Tue, 10 Dec 2024 15:14:01 -0500
Subject: [PATCH 01/45] Syntax fixes

---
 vcell-cli/src/main/java/org/vcell/cli/run/ExecuteCommand.java | 4 ++--
 vcell-cli/src/main/java/org/vcell/cli/run/ExecuteImpl.java    | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteCommand.java b/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteCommand.java
index 9fda30e845..3877411101 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteCommand.java
+++ b/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteCommand.java
@@ -16,7 +16,7 @@
 import java.util.concurrent.Callable;
 import java.util.Date;
 
-@Command(name = "execute", description = "run .vcml or .omex files via Python API")
+@Command(name = "execute", description = "run .vcml or .omex files.")
 public class ExecuteCommand implements Callable<Integer> {
 
     private final static Logger logger = org.apache.logging.log4j.LogManager.getLogger(ExecuteCommand.class);
@@ -100,7 +100,7 @@ public Integer call() {
                     bKeepTempFiles, bExactMatchOnly, bEncapsulateOutput, 
                     EXECUTABLE_MAX_WALLCLOCK_MILLIS, help, bDebug, bQuiet
             );
-            logger.trace(trace_args);
+            logger.debug(trace_args);
 
             logger.debug("Validating CLI arguments");
             if (bDebug && bQuiet) {
diff --git a/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteImpl.java b/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteImpl.java
index b78dfe3c43..858fe98022 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteImpl.java
+++ b/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteImpl.java
@@ -134,7 +134,7 @@ public static void singleMode(File inputFile, File rootOutputDir, CLIRecordable
 
         logger.info("Preparing output directory...");
         // we don't want to accidentally delete the input...
-        // if the output is a subset of the input file's housing directory, we shouldn't delete!!
+        // if the output directory is a subset of the input file's housing directory, we shouldn't delete!!
         if (!inputFile.getParentFile().getCanonicalPath().contains(adjustedOutputDir.getCanonicalPath()))
             RunUtils.removeAndMakeDirs(adjustedOutputDir);
 

From b46cbcd29e813b2fe837f7eef315ed0f7210af2a Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Tue, 10 Dec 2024 15:29:45 -0500
Subject: [PATCH 02/45] 1st attempt at individual runs

---
 .github/workflows/NightlyBMDB_CLI.yml | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index c8a84c2ef2..2f7e415dfd 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -239,7 +239,15 @@ jobs:
         # Do the execution
       - name: Begin Execution
         #$(docker image ls | grep "<none>" | awk '{print $3;}')  to get image id
-        run: docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root" -o "/root/output" --writeLogFiles # grabs image and runs
+        run: |
+          for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"; do
+            if [ -f "$file" ]; then
+              basename="$(file%.*)"
+              mkdir "$GITHUB_WORKSPACE/set_${{ matrix.sets }}/${basename}_output"
+              docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root/${file}" -o "/root/${basename}_output" --writeLogFiles
+            fi
+          done
+        #run: docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root" -o "/root/output" --writeLogFiles # grabs image and runs
   
       - name: Check For Success
         run: |

From 55395d9d522a5f0bcb352fa9da5bca85d1794ed5 Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Wed, 11 Dec 2024 11:22:20 -0500
Subject: [PATCH 03/45] [WIP] attempting to get more info about what's going on

---
 .github/workflows/NightlyBMDB_CLI.yml | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 2f7e415dfd..949e40b191 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -161,7 +161,16 @@ jobs:
         # Do the execution
       - name: Begin Execution
         #$(docker image ls | grep "<none>" | awk '{print $3;}')  to get image id
-        run: docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root" -o "/root/output" --writeLogFiles # grabs image and runs
+        run: |
+          for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"; do
+            if [ -f "$file" ]; then
+            basename="$(file%.*)"
+            mkdir "$GITHUB_WORKSPACE/set_${{ matrix.sets }}/${basename}_output"
+            echo "Running \"$file\" in output folder \"${basename}_output\""
+            docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root/${file}" -o "/root/${basename}_output" --writeLogFiles
+          fi
+          done
+        #run: docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root" -o "/root/output" --writeLogFiles # grabs image and runs
   
       - name: Check For Success
         run: |
@@ -241,9 +250,11 @@ jobs:
         #$(docker image ls | grep "<none>" | awk '{print $3;}')  to get image id
         run: |
           for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"; do
+            echo "\"$file\""
             if [ -f "$file" ]; then
               basename="$(file%.*)"
               mkdir "$GITHUB_WORKSPACE/set_${{ matrix.sets }}/${basename}_output"
+              echo "Running \"$file\" in output folder \"${basename}_output\""
               docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root/${file}" -o "/root/${basename}_output" --writeLogFiles
             fi
           done

From 05e9a209d52c1bc6ecd6bfb9c6119d3bed5ff0dd Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Wed, 11 Dec 2024 12:17:43 -0500
Subject: [PATCH 04/45] Added required wild card

---
 .github/workflows/NightlyBMDB_CLI.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 949e40b191..60fb5e3a36 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -162,7 +162,8 @@ jobs:
       - name: Begin Execution
         #$(docker image ls | grep "<none>" | awk '{print $3;}')  to get image id
         run: |
-          for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"; do
+          for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
+            echo "\"$file\""
             if [ -f "$file" ]; then
             basename="$(file%.*)"
             mkdir "$GITHUB_WORKSPACE/set_${{ matrix.sets }}/${basename}_output"
@@ -249,7 +250,7 @@ jobs:
       - name: Begin Execution
         #$(docker image ls | grep "<none>" | awk '{print $3;}')  to get image id
         run: |
-          for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"; do
+          for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}/*"; do
             echo "\"$file\""
             if [ -f "$file" ]; then
               basename="$(file%.*)"

From 8fde0f1424f8726020cc6695819a3a7ba2c46219 Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Wed, 11 Dec 2024 12:28:03 -0500
Subject: [PATCH 05/45] Syntax Fixing

---
 .github/workflows/NightlyBMDB_CLI.yml | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 60fb5e3a36..583e265a8d 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -162,13 +162,16 @@ jobs:
       - name: Begin Execution
         #$(docker image ls | grep "<none>" | awk '{print $3;}')  to get image id
         run: |
+          # The /* goes on the outside, otherwise bash just interprets a string!
           for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
             echo "\"$file\""
             if [ -f "$file" ]; then
-            basename="$(file%.*)"
-            mkdir "$GITHUB_WORKSPACE/set_${{ matrix.sets }}/${basename}_output"
-            echo "Running \"$file\" in output folder \"${basename}_output\""
-            docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root/${file}" -o "/root/${basename}_output" --writeLogFiles
+            base_name="${file%.*}"
+            mkdir "${base_name}_output"
+            echo "Running \"$file\" in output folder \"${base_name}_output\""
+            sim_input="/root/$(basename ${file})"
+            sim_output="/root/$(basename ${base_name})_output"
+            docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i $sim_input -o $sim_output --writeLogFiles
           fi
           done
         #run: docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root" -o "/root/output" --writeLogFiles # grabs image and runs
@@ -250,13 +253,16 @@ jobs:
       - name: Begin Execution
         #$(docker image ls | grep "<none>" | awk '{print $3;}')  to get image id
         run: |
-          for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}/*"; do
+          # The /* goes on the outside, otherwise bash just interprets a string!
+          for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
             echo "\"$file\""
             if [ -f "$file" ]; then
-              basename="$(file%.*)"
-              mkdir "$GITHUB_WORKSPACE/set_${{ matrix.sets }}/${basename}_output"
-              echo "Running \"$file\" in output folder \"${basename}_output\""
-              docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root/${file}" -o "/root/${basename}_output" --writeLogFiles
+              base_name="${file%.*}"
+              mkdir "${base_name}_output"
+              echo "Running \"$file\" in output folder \"${base_name}_output\""
+              sim_input="/root/$(basename ${file})"
+              sim_output="/root/$(basename ${base_name})_output"
+              docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i $sim_input -o $sim_output --writeLogFiles
             fi
           done
         #run: docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root" -o "/root/output" --writeLogFiles # grabs image and runs

From b1c633ab7fe33f31407ee5b2237e912f076f5799 Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Wed, 11 Dec 2024 13:18:34 -0500
Subject: [PATCH 06/45] Added option to guarantee good return code

---
 .github/workflows/NightlyBMDB_CLI.yml                        | 4 ++--
 .../src/main/java/org/vcell/cli/run/ExecuteCommand.java      | 5 ++++-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 583e265a8d..e504c60587 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -171,7 +171,7 @@ jobs:
             echo "Running \"$file\" in output folder \"${base_name}_output\""
             sim_input="/root/$(basename ${file})"
             sim_output="/root/$(basename ${base_name})_output"
-            docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i $sim_input -o $sim_output --writeLogFiles
+            docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i $sim_input -o $sim_output --writeLogFiles --guaranteeGoodReturnCode
           fi
           done
         #run: docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root" -o "/root/output" --writeLogFiles # grabs image and runs
@@ -262,7 +262,7 @@ jobs:
               echo "Running \"$file\" in output folder \"${base_name}_output\""
               sim_input="/root/$(basename ${file})"
               sim_output="/root/$(basename ${base_name})_output"
-              docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i $sim_input -o $sim_output --writeLogFiles
+              docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i $sim_input -o $sim_output --writeLogFiles --guaranteeGoodReturnCode
             fi
           done
         #run: docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root" -o "/root/output" --writeLogFiles # grabs image and runs
diff --git a/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteCommand.java b/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteCommand.java
index 3877411101..1026ac4cdb 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteCommand.java
+++ b/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteCommand.java
@@ -39,6 +39,9 @@ public class ExecuteCommand implements Callable<Integer> {
     @Option(names = "--keepFlushingLogs", defaultValue = "false")
     private boolean bKeepFlushingLogs = false;
 
+    @Option(names = "--guaranteeGoodReturnCode", defaultValue = "false", description = "Even on failure, return error code 0 for script purposes.")
+    private boolean bGuaranteeGoodReturnCode = false;
+
     @Option(names = "--small-mesh", defaultValue = "false", description = "force spatial simulations to have a very small mesh to make execution faster")
     private boolean bSmallMeshOverride = false;
 
@@ -135,7 +138,7 @@ public Integer call() {
             return 0;
         } catch (Exception e) { ///TODO: Break apart into specific exceptions to maximize logging.
             org.apache.logging.log4j.LogManager.getLogger(this.getClass()).error(e.getMessage(), e);
-            return 1;
+            return bGuaranteeGoodReturnCode ? 0 : 1;
         } finally {
             logger.debug("Completed all execution");
         }

From 2edb8a1ff3de1d560d35db82b6f3381674418102 Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Thu, 12 Dec 2024 09:09:19 -0500
Subject: [PATCH 07/45] Attempting to update how failures are detected

---
 .github/workflows/NightlyBMDB_CLI.yml | 34 +++++++++++++++++----------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index e504c60587..0660eff657 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -172,19 +172,24 @@ jobs:
             sim_input="/root/$(basename ${file})"
             sim_output="/root/$(basename ${base_name})_output"
             docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i $sim_input -o $sim_output --writeLogFiles --guaranteeGoodReturnCode
+          
+            # Check for individual Success
+            if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt | wc -l) -ne 0 ]
+            then 
+              # Copy error log to github workspace
+              touch ${{ github.workspace }}/errors.report
+              cat $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt) >> ${{ github.workspace }}/errors.report
+            fi
           fi
           done
         #run: docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root" -o "/root/output" --writeLogFiles # grabs image and runs
   
       - name: Check For Success
         run: |
-          errorSearch=$(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt)
-          if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt | wc -l) -eq 0 ]
-          then 
+          # If the error file exists, then we have failures to report and upload; else, we can just end here; the next job will process results
+          if [ $(find ${{ github.workspace }} -name errors.report | wc -l) -eq 0 ]
+          then
             exit 0
-          else
-            # Copy error log to github workspace
-            cp $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt) ${{ github.workspace }}/errors.report
           fi
 
       - name: Upload failures
@@ -263,19 +268,24 @@ jobs:
               sim_input="/root/$(basename ${file})"
               sim_output="/root/$(basename ${base_name})_output"
               docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i $sim_input -o $sim_output --writeLogFiles --guaranteeGoodReturnCode
+          
+              # Check for individual Success
+              if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt | wc -l) -ne 0 ]
+              then 
+                # Copy error log to github workspace
+                touch ${{ github.workspace }}/errors.report
+                cat $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt) >> ${{ github.workspace }}/errors.report
+              fi
             fi
           done
         #run: docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root" -o "/root/output" --writeLogFiles # grabs image and runs
-  
+
       - name: Check For Success
         run: |
-          errorSearch=$(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt)
-          if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt | wc -l) -eq 0 ]
+          # If the error file exists, then we have failures to report and upload; else, we can just end here; the next job will process results
+          if [ $(find ${{ github.workspace }} -name errors.report | wc -l) -eq 0 ]
           then
             exit 0
-          else
-            # Copy error log to github workspace
-            cp $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt) ${{ github.workspace }}/errors.report
           fi
 
       - name: Upload failures

From 63fb7159927586d515d0ffb1bec34c64fc6d49dc Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Thu, 12 Dec 2024 10:36:09 -0500
Subject: [PATCH 08/45] Reducing scale temporarily for testing

---
 .github/workflows/NightlyBMDB_CLI.yml | 218 +++++++++++++-------------
 1 file changed, 109 insertions(+), 109 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 0660eff657..f45bac2c25 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -116,103 +116,103 @@ jobs:
       - name: Setup tmate session 3
         uses: mxschmitt/action-tmate@v3
         if: ${{ failure() }}
-  published:
-    runs-on: ubuntu-20.04
-    needs: build
-    strategy:
-      matrix:
-        sets: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19"]
-    steps:
-      - name: Create Docker Image Dir
-        run: mkdir /tmp/docker
-
-      - name: Download Docker Image from `build` Job
-        uses: actions/download-artifact@v3
-        with:
-          name: docker-image
-          path: /tmp/docker
-
-      - name: Load Downloaded Docker Image
-        run: docker load --input /tmp/docker/$(ls /tmp/docker)
-        
-        # Get files from BMDB
-      - name: Checkout Input Files
-        uses: actions/checkout@v4
-        with:
-          repository: virtualcell/vcdb
-
-      - name: Prepare sub directory
-        run: |
-          SOURCE_DIR="$GITHUB_WORKSPACE/published/biomodel/omex/sbml"
-          TARGET_DIR="$GITHUB_WORKSPACE/set_${{ matrix.sets }}"
-          MOD_VALUE=${{ matrix.sets }}
-          file_index=0
-          
-          # make target dir
-          mkdir $TARGET_DIR
-          # loop through files in directory, increment indexing and separating files
-          for file in $SOURCE_DIR/*; do
-              file_index=$((file_index+1))
-              if [ $((file_index%${{ strategy.job-total }})) -eq $((MOD_VALUE + 0)) ]; then
-                  cp ${file} $TARGET_DIR/
-              fi
-          done
-        
-        # Do the execution
-      - name: Begin Execution
-        #$(docker image ls | grep "<none>" | awk '{print $3;}')  to get image id
-        run: |
-          # The /* goes on the outside, otherwise bash just interprets a string!
-          for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
-            echo "\"$file\""
-            if [ -f "$file" ]; then
-            base_name="${file%.*}"
-            mkdir "${base_name}_output"
-            echo "Running \"$file\" in output folder \"${base_name}_output\""
-            sim_input="/root/$(basename ${file})"
-            sim_output="/root/$(basename ${base_name})_output"
-            docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i $sim_input -o $sim_output --writeLogFiles --guaranteeGoodReturnCode
-          
-            # Check for individual Success
-            if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt | wc -l) -ne 0 ]
-            then 
-              # Copy error log to github workspace
-              touch ${{ github.workspace }}/errors.report
-              cat $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt) >> ${{ github.workspace }}/errors.report
-            fi
-          fi
-          done
-        #run: docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root" -o "/root/output" --writeLogFiles # grabs image and runs
-  
-      - name: Check For Success
-        run: |
-          # If the error file exists, then we have failures to report and upload; else, we can just end here; the next job will process results
-          if [ $(find ${{ github.workspace }} -name errors.report | wc -l) -eq 0 ]
-          then
-            exit 0
-          fi
-
-      - name: Upload failures
-        uses: actions/upload-artifact@v3
-        with:
-          name: set_${{ matrix.sets }}.report
-          path: ${{ github.workspace }}/errors.report
-
-      # Report a Problem to Slack
-      - name: Report Problem to Slack
-        if: ${{ failure() }}
-        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Oops! The *Published Step* of set ${{ matrix.sets }} in Nightly **Execution** Testing Action on GitHub Failed ( ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} ). Go see what happened!" https://slack.com/api/chat.postMessage
-
-      - name: Setup tmate session 3
-        uses: mxschmitt/action-tmate@v3
-        if: ${{ failure() }}
+#  published:
+#    runs-on: ubuntu-20.04
+#    needs: build
+#    strategy:
+#      matrix:
+#        sets: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19"]
+#    steps:
+#      - name: Create Docker Image Dir
+#        run: mkdir /tmp/docker
+#
+#      - name: Download Docker Image from `build` Job
+#        uses: actions/download-artifact@v3
+#        with:
+#          name: docker-image
+#          path: /tmp/docker
+#
+#      - name: Load Downloaded Docker Image
+#        run: docker load --input /tmp/docker/$(ls /tmp/docker)
+#
+#        # Get files from BMDB
+#      - name: Checkout Input Files
+#        uses: actions/checkout@v4
+#        with:
+#          repository: virtualcell/vcdb
+#
+#      - name: Prepare sub directory
+#        run: |
+#          SOURCE_DIR="$GITHUB_WORKSPACE/published/biomodel/omex/sbml"
+#          TARGET_DIR="$GITHUB_WORKSPACE/set_${{ matrix.sets }}"
+#          MOD_VALUE=${{ matrix.sets }}
+#          file_index=0
+#
+#          # make target dir
+#          mkdir $TARGET_DIR
+#          # loop through files in directory, increment indexing and separating files
+#          for file in $SOURCE_DIR/*; do
+#              file_index=$((file_index+1))
+#              if [ $((file_index%${{ strategy.job-total }})) -eq $((MOD_VALUE + 0)) ]; then
+#                  cp ${file} $TARGET_DIR/
+#              fi
+#          done
+#
+#        # Do the execution
+#      - name: Begin Execution
+#        #$(docker image ls | grep "<none>" | awk '{print $3;}')  to get image id
+#        run: |
+#          # The /* goes on the outside, otherwise bash just interprets a string!
+#          for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
+#            echo "\"$file\""
+#            if [ -f "$file" ]; then
+#            base_name="${file%.*}"
+#            mkdir "${base_name}_output"
+#            echo "Running \"$file\" in output folder \"${base_name}_output\""
+#            sim_input="/root/$(basename ${file})"
+#            sim_output="/root/$(basename ${base_name})_output"
+#            docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i $sim_input -o $sim_output --writeLogFiles --guaranteeGoodReturnCode
+#
+#            # Check for individual Success
+#            if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt | wc -l) -ne 0 ]
+#            then
+#              # Copy error log to github workspace
+#              touch ${{ github.workspace }}/errors.report
+#              cat $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt) >> ${{ github.workspace }}/errors.report
+#            fi
+#          fi
+#          done
+#        #run: docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root" -o "/root/output" --writeLogFiles # grabs image and runs
+#
+#      - name: Check For Success
+#        run: |
+#          # If the error file exists, then we have failures to report and upload; else, we can just end here; the next job will process results
+#          if [ $(find ${{ github.workspace }} -name errors.report | wc -l) -eq 0 ]
+#          then
+#            exit 0
+#          fi
+#
+#      - name: Upload failures
+#        uses: actions/upload-artifact@v3
+#        with:
+#          name: set_${{ matrix.sets }}.report
+#          path: ${{ github.workspace }}/errors.report
+#
+#      # Report a Problem to Slack
+#      - name: Report Problem to Slack
+#        if: ${{ failure() }}
+#        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Oops! The *Published Step* of set ${{ matrix.sets }} in Nightly **Execution** Testing Action on GitHub Failed ( ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} ). Go see what happened!" https://slack.com/api/chat.postMessage
+#
+#      - name: Setup tmate session 3
+#        uses: mxschmitt/action-tmate@v3
+#        if: ${{ failure() }}
 
   bmdb:
     runs-on: ubuntu-20.04
     needs: build
     strategy:
       matrix:
-        sets: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19"]
+        sets: ["2", "4"]
     steps:
       - name: Create Docker Image Dir
         run: mkdir /tmp/docker
@@ -303,24 +303,24 @@ jobs:
         uses: mxschmitt/action-tmate@v3
         if: ${{ failure() }}
 
-  published_results:
-    runs-on: ubuntu-20.04
-    needs: published
-
-    steps:
-      - name: Gather Artifacts
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ github.workspace }}/published-results
-
-      - name: Combine .report artifacts into one file
-        run: cat $(find $GITHUB_WORKSPACE/published-results -name 'errors.report') > $GITHUB_WORKSPACE/combined.txt
-
-      - name: Post results to slack part 1
-        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Here're the $(cat ${GITHUB_WORKSPACE}/combined.txt | wc -l) published models that didn't pass:" https://slack.com/api/chat.postMessage
-
-      - name: Post results to slack part 2
-        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(cat ${GITHUB_WORKSPACE}/combined.txt)" https://slack.com/api/chat.postMessage
+#  published_results:
+#    runs-on: ubuntu-20.04
+#    needs: published
+#
+#    steps:
+#      - name: Gather Artifacts
+#        uses: actions/download-artifact@v3
+#        with:
+#          path: ${{ github.workspace }}/published-results
+#
+#      - name: Combine .report artifacts into one file
+#        run: cat $(find $GITHUB_WORKSPACE/published-results -name 'errors.report') > $GITHUB_WORKSPACE/combined.txt
+#
+#      - name: Post results to slack part 1
+#        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Here're the $(cat ${GITHUB_WORKSPACE}/combined.txt | wc -l) published models that didn't pass:" https://slack.com/api/chat.postMessage
+#
+#      - name: Post results to slack part 2
+#        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(cat ${GITHUB_WORKSPACE}/combined.txt)" https://slack.com/api/chat.postMessage
 
 
   bmdb_results:

From 5ecfdea1bd7adac1d9e0e135c3bd362248debd42 Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Thu, 12 Dec 2024 10:58:39 -0500
Subject: [PATCH 09/45] Attempting different way to bypass "extra" matrix jobs

---
 .github/workflows/NightlyBMDB_CLI.yml | 43 ++++++++++++++-------------
 1 file changed, 23 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index f45bac2c25..138bd24e39 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -212,7 +212,7 @@ jobs:
     needs: build
     strategy:
       matrix:
-        sets: ["2", "4"]
+        sets: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19"]
     steps:
       - name: Create Docker Image Dir
         run: mkdir /tmp/docker
@@ -258,26 +258,29 @@ jobs:
       - name: Begin Execution
         #$(docker image ls | grep "<none>" | awk '{print $3;}')  to get image id
         run: |
-          # The /* goes on the outside, otherwise bash just interprets a string!
-          for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
-            echo "\"$file\""
-            if [ -f "$file" ]; then
-              base_name="${file%.*}"
-              mkdir "${base_name}_output"
-              echo "Running \"$file\" in output folder \"${base_name}_output\""
-              sim_input="/root/$(basename ${file})"
-              sim_output="/root/$(basename ${base_name})_output"
-              docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i $sim_input -o $sim_output --writeLogFiles --guaranteeGoodReturnCode
-          
-              # Check for individual Success
-              if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt | wc -l) -ne 0 ]
-              then 
-                # Copy error log to github workspace
-                touch ${{ github.workspace }}/errors.report
-                cat $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt) >> ${{ github.workspace }}/errors.report
+          if [ ${{ matrix.sets }} -eq 4 ] || [ ${{ matrix.sets }} -eq 2 ]; then
+            # The /* goes on the outside, otherwise bash just interprets a string!
+            for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
+              echo "\"$file\""
+              if [ -f "$file" ]; then
+                base_name="${file%.*}"
+                mkdir "${base_name}_output"
+                echo "Running \"$file\" in output folder \"${base_name}_output\""
+                sim_input="/root/$(basename ${file})"
+                sim_output="/root/$(basename ${base_name})_output"
+                docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i $sim_input -o $sim_output --writeLogFiles --guaranteeGoodReturnCode
+            
+                # Check for individual Success
+                if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt | wc -l) -ne 0 ]
+                then
+                  # Copy error log to github workspace
+                  touch ${{ github.workspace }}/errors.report
+                  cat $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt) >> ${{ github.workspace }}/errors.report
+                fi
               fi
-            fi
-          done
+            done
+          fi
+
         #run: docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root" -o "/root/output" --writeLogFiles # grabs image and runs
 
       - name: Check For Success

From 1317b53f91216382802956d9afbb53f0e90d4b4e Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Fri, 13 Dec 2024 11:49:41 -0500
Subject: [PATCH 10/45] More context information

---
 .github/workflows/NightlyBMDB_CLI.yml | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 138bd24e39..d9ccbd4485 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -237,19 +237,29 @@ jobs:
           SOURCE_DIR="$GITHUB_WORKSPACE/omex_files"
           TARGET_DIR="$GITHUB_WORKSPACE/set_${{ matrix.sets }}"
           MOD_VALUE=${{ matrix.sets }}
-          file_index=0
           
           # make target dir
           mkdir $TARGET_DIR
           # loop through files in directory, increment indexing and separating files
+          
+          echo "Extracting tests"
+          
           for file in $SOURCE_DIR/*; do
-              file_index=$((file_index+1))
-              if [[ $file == *1008* ]]; then
+              # Is the file of the correct form?
+              base_name=$(basename $file)
+              if ! [[ $base_name =~ ^BIOMD[0-9]+\.omex$ ]]; then
+                echo "Input is invalid. Expecting `BIOMD##########.omex` format, got `${base_name}`."
+                continue
+              fi
+              # ...is it the infamous BioModel 1008?
+              if [[ $base_name == *1008* ]]; then
                   echo -n 'skipping '
-                  echo $file
+                  echo $base_name
                   continue
               fi
-              if [ $((file_index%${{ strategy.job-total }})) -eq $((MOD_VALUE + 0)) ]; then
+              number=$(echo "$base_name" | sed -n 's/^BIOMD\([0-9]\+\)\.omex$/\1/p' | sed 's/^0*//')
+              if [ $((number % ${{ strategy.job-total }})) -eq $((MOD_VALUE + 0)) ]; then
+                  echo "Accepting ${base_name}"
                   cp ${file} $TARGET_DIR/
               fi
           done
@@ -269,7 +279,7 @@ jobs:
                 sim_input="/root/$(basename ${file})"
                 sim_output="/root/$(basename ${base_name})_output"
                 docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i $sim_input -o $sim_output --writeLogFiles --guaranteeGoodReturnCode
-            
+                echo -e "\n\n\n\n\n" # give ourselves some line separation
                 # Check for individual Success
                 if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt | wc -l) -ne 0 ]
                 then
@@ -290,6 +300,8 @@ jobs:
           then
             exit 0
           fi
+          echo "Models that failed:"
+          cat ${{ github.workspace }}/errors.report
 
       - name: Upload failures
         uses: actions/upload-artifact@v3

From abb7e3aa8321e45915ad6829d3dc25f573bedd02 Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Fri, 13 Dec 2024 13:33:02 -0500
Subject: [PATCH 11/45] Fixed location of finding error file to be more
 specific

---
 .github/workflows/NightlyBMDB_CLI.yml | 37 +++++++++++++++------------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index d9ccbd4485..69c132f5d7 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -246,20 +246,20 @@ jobs:
           
           for file in $SOURCE_DIR/*; do
               # Is the file of the correct form?
-              base_name=$(basename $file)
-              if ! [[ $base_name =~ ^BIOMD[0-9]+\.omex$ ]]; then
-                echo "Input is invalid. Expecting `BIOMD##########.omex` format, got `${base_name}`."
+              file_name=$(basename $file)
+              if ! [[ $file_name =~ ^BIOMD[0-9]+\.omex$ ]]; then
+                echo "Input is invalid. Expecting 'BIOMD##########.omex format, got '${file_name}'."
                 continue
               fi
               # ...is it the infamous BioModel 1008?
-              if [[ $base_name == *1008* ]]; then
+              if [[ $file_name == *1008* ]]; then
                   echo -n 'skipping '
-                  echo $base_name
+                  echo $file_name
                   continue
               fi
-              number=$(echo "$base_name" | sed -n 's/^BIOMD\([0-9]\+\)\.omex$/\1/p' | sed 's/^0*//')
+              number=$(echo "$file_name" | sed -n 's/^BIOMD\([0-9]\+\)\.omex$/\1/p' | sed 's/^0*//')
               if [ $((number % ${{ strategy.job-total }})) -eq $((MOD_VALUE + 0)) ]; then
-                  echo "Accepting ${base_name}"
+                  echo "Accepting ${file_name}"
                   cp ${file} $TARGET_DIR/
               fi
           done
@@ -268,24 +268,29 @@ jobs:
       - name: Begin Execution
         #$(docker image ls | grep "<none>" | awk '{print $3;}')  to get image id
         run: |
-          if [ ${{ matrix.sets }} -eq 4 ] || [ ${{ matrix.sets }} -eq 2 ]; then
+          if [ ${{ matrix.sets }} -eq 4 ] || [ ${{ matrix.sets }} -eq 6 ]; then
             # The /* goes on the outside, otherwise bash just interprets a string!
             for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
               echo "\"$file\""
               if [ -f "$file" ]; then
-                base_name="${file%.*}"
-                mkdir "${base_name}_output"
-                echo "Running \"$file\" in output folder \"${base_name}_output\""
-                sim_input="/root/$(basename ${file})"
-                sim_output="/root/$(basename ${base_name})_output"
-                docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i $sim_input -o $sim_output --writeLogFiles --guaranteeGoodReturnCode
+                extensionless_name="${file%.*}"
+                base_name="$(basename ${file})"
+                base_extless_name="$(basename ${extensionless_name})"
+                mkdir "${extensionless_name}_output"
+                echo "Running \"${file}\" in output folder \"${base_extless_name}_output\""
+                sim_input="/root/${base_name}"
+                sim_output="/root/${base_extless_name}_output"
+                docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i ${sim_input} -o ${sim_output} --writeLogFiles --guaranteeGoodReturnCode
+                
                 echo -e "\n\n\n\n\n" # give ourselves some line separation
                 # Check for individual Success
-                if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt | wc -l) -ne 0 ]
+                if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name errorLog.txt | wc -l) -ne 0 ]
                 then
                   # Copy error log to github workspace
                   touch ${{ github.workspace }}/errors.report
-                  cat $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt) >> ${{ github.workspace }}/errors.report
+                  cat $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name errorLog.txt) >> ${{ github.workspace }}/errors.report
+                else
+                  echo "No errors found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
                 fi
               fi
             done

From a8d5b3e0b156b15f213f2d3a225ce7141a9cfff8 Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Mon, 16 Dec 2024 16:32:30 -0500
Subject: [PATCH 12/45] Sorting output and unlocking full spread

---
 .github/workflows/NightlyBMDB_CLI.yml | 275 +++++++++++++-------------
 1 file changed, 140 insertions(+), 135 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 69c132f5d7..523eb8ea92 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -116,96 +116,103 @@ jobs:
       - name: Setup tmate session 3
         uses: mxschmitt/action-tmate@v3
         if: ${{ failure() }}
-#  published:
-#    runs-on: ubuntu-20.04
-#    needs: build
-#    strategy:
-#      matrix:
-#        sets: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19"]
-#    steps:
-#      - name: Create Docker Image Dir
-#        run: mkdir /tmp/docker
-#
-#      - name: Download Docker Image from `build` Job
-#        uses: actions/download-artifact@v3
-#        with:
-#          name: docker-image
-#          path: /tmp/docker
-#
-#      - name: Load Downloaded Docker Image
-#        run: docker load --input /tmp/docker/$(ls /tmp/docker)
-#
-#        # Get files from BMDB
-#      - name: Checkout Input Files
-#        uses: actions/checkout@v4
-#        with:
-#          repository: virtualcell/vcdb
-#
-#      - name: Prepare sub directory
-#        run: |
-#          SOURCE_DIR="$GITHUB_WORKSPACE/published/biomodel/omex/sbml"
-#          TARGET_DIR="$GITHUB_WORKSPACE/set_${{ matrix.sets }}"
-#          MOD_VALUE=${{ matrix.sets }}
-#          file_index=0
-#
-#          # make target dir
-#          mkdir $TARGET_DIR
-#          # loop through files in directory, increment indexing and separating files
-#          for file in $SOURCE_DIR/*; do
-#              file_index=$((file_index+1))
-#              if [ $((file_index%${{ strategy.job-total }})) -eq $((MOD_VALUE + 0)) ]; then
-#                  cp ${file} $TARGET_DIR/
-#              fi
-#          done
-#
-#        # Do the execution
-#      - name: Begin Execution
-#        #$(docker image ls | grep "<none>" | awk '{print $3;}')  to get image id
-#        run: |
-#          # The /* goes on the outside, otherwise bash just interprets a string!
-#          for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
-#            echo "\"$file\""
-#            if [ -f "$file" ]; then
-#            base_name="${file%.*}"
-#            mkdir "${base_name}_output"
-#            echo "Running \"$file\" in output folder \"${base_name}_output\""
-#            sim_input="/root/$(basename ${file})"
-#            sim_output="/root/$(basename ${base_name})_output"
-#            docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i $sim_input -o $sim_output --writeLogFiles --guaranteeGoodReturnCode
-#
-#            # Check for individual Success
-#            if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt | wc -l) -ne 0 ]
-#            then
-#              # Copy error log to github workspace
-#              touch ${{ github.workspace }}/errors.report
-#              cat $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }} -name errorLog.txt) >> ${{ github.workspace }}/errors.report
-#            fi
-#          fi
-#          done
-#        #run: docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root" -o "/root/output" --writeLogFiles # grabs image and runs
-#
-#      - name: Check For Success
-#        run: |
-#          # If the error file exists, then we have failures to report and upload; else, we can just end here; the next job will process results
-#          if [ $(find ${{ github.workspace }} -name errors.report | wc -l) -eq 0 ]
-#          then
-#            exit 0
-#          fi
-#
-#      - name: Upload failures
-#        uses: actions/upload-artifact@v3
-#        with:
-#          name: set_${{ matrix.sets }}.report
-#          path: ${{ github.workspace }}/errors.report
-#
-#      # Report a Problem to Slack
-#      - name: Report Problem to Slack
-#        if: ${{ failure() }}
-#        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Oops! The *Published Step* of set ${{ matrix.sets }} in Nightly **Execution** Testing Action on GitHub Failed ( ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} ). Go see what happened!" https://slack.com/api/chat.postMessage
-#
-#      - name: Setup tmate session 3
-#        uses: mxschmitt/action-tmate@v3
-#        if: ${{ failure() }}
+
+  published:
+    runs-on: ubuntu-20.04
+    needs: build
+    strategy:
+      matrix:
+        sets: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19"]
+    steps:
+      - name: Create Docker Image Dir
+        run: mkdir /tmp/docker
+
+      - name: Download Docker Image from `build` Job
+        uses: actions/download-artifact@v3
+        with:
+          name: docker-image
+          path: /tmp/docker
+
+      - name: Load Downloaded Docker Image
+        run: docker load --input /tmp/docker/$(ls /tmp/docker)
+
+        # Get files from BMDB
+      - name: Checkout Input Files
+        uses: actions/checkout@v4
+        with:
+          repository: virtualcell/vcdb
+
+      - name: Prepare sub directory
+        run: |
+          SOURCE_DIR="$GITHUB_WORKSPACE/published/biomodel/omex/sbml"
+          TARGET_DIR="$GITHUB_WORKSPACE/set_${{ matrix.sets }}"
+          MOD_VALUE=${{ matrix.sets }}
+          file_index=0
+
+          # make target dir
+          mkdir $TARGET_DIR
+          # loop through files in directory, increment indexing and separating files
+          for file in $SOURCE_DIR/*; do
+              file_index=$((file_index+1))
+              if [ $((file_index%${{ strategy.job-total }})) -eq $((MOD_VALUE + 0)) ]; then
+                  cp ${file} $TARGET_DIR/
+              fi
+          done
+
+        # Do the execution
+      - name: Begin Execution
+        #$(docker image ls | grep "<none>" | awk '{print $3;}')  to get image id
+        run: |
+          # The /* goes on the outside, otherwise bash just interprets a string!
+          for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
+            echo "\"$file\""
+            if [ -f "$file" ]; then
+              extensionless_name="${file%.*}"
+              base_name="$(basename ${file})"
+              base_extless_name="$(basename ${extensionless_name})"
+              mkdir "${extensionless_name}_output"
+              echo "Running \"${file}\" in output folder \"${base_extless_name}_output\""
+              sim_input="/root/${base_name}"
+              sim_output="/root/${base_extless_name}_output"
+              docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i ${sim_input} -o ${sim_output} --writeLogFiles --guaranteeGoodReturnCode
+          
+              echo -e "\n\n\n\n\n" # give ourselves some line separation
+              # Check for individual Success
+              if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name errorLog.txt | wc -l) -ne 0 ]
+              then
+                # Copy error log to github workspace
+                touch ${{ github.workspace }}/errors.report
+                cat $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name errorLog.txt) >> ${{ github.workspace }}/errors.report
+              else
+                echo "No errors found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
+              fi
+            fi
+          done
+
+      - name: Check For Success
+        run: |
+          # If the error file exists, then we have failures to report and upload; else, we can just end here; the next job will process results
+          if [ $(find ${{ github.workspace }} -name errors.report | wc -l) -eq 0 ]
+          then
+            exit 0
+          fi
+          echo "Models that failed:"
+          cat ${{ github.workspace }}/errors.report
+
+      - name: Upload failures
+        uses: actions/upload-artifact@v3
+        with:
+          name: set_${{ matrix.sets }}.report
+          path: ${{ github.workspace }}/errors.report
+
+      # Report a Problem to Slack
+      - name: Report Problem to Slack
+        if: ${{ failure() }}
+        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Oops! The *Published Step* of set ${{ matrix.sets }} in Nightly **Execution** Testing Action on GitHub Failed ( ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} ). Go see what happened!" https://slack.com/api/chat.postMessage
+
+      - name: Setup tmate session 3
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ failure() }}
 
   bmdb:
     runs-on: ubuntu-20.04
@@ -268,33 +275,31 @@ jobs:
       - name: Begin Execution
         #$(docker image ls | grep "<none>" | awk '{print $3;}')  to get image id
         run: |
-          if [ ${{ matrix.sets }} -eq 4 ] || [ ${{ matrix.sets }} -eq 6 ]; then
-            # The /* goes on the outside, otherwise bash just interprets a string!
-            for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
-              echo "\"$file\""
-              if [ -f "$file" ]; then
-                extensionless_name="${file%.*}"
-                base_name="$(basename ${file})"
-                base_extless_name="$(basename ${extensionless_name})"
-                mkdir "${extensionless_name}_output"
-                echo "Running \"${file}\" in output folder \"${base_extless_name}_output\""
-                sim_input="/root/${base_name}"
-                sim_output="/root/${base_extless_name}_output"
-                docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i ${sim_input} -o ${sim_output} --writeLogFiles --guaranteeGoodReturnCode
-                
-                echo -e "\n\n\n\n\n" # give ourselves some line separation
-                # Check for individual Success
-                if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name errorLog.txt | wc -l) -ne 0 ]
-                then
-                  # Copy error log to github workspace
-                  touch ${{ github.workspace }}/errors.report
-                  cat $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name errorLog.txt) >> ${{ github.workspace }}/errors.report
-                else
-                  echo "No errors found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
-                fi
+          # The /* goes on the outside, otherwise bash just interprets a string!
+          for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
+            echo "\"$file\""
+            if [ -f "$file" ]; then
+              extensionless_name="${file%.*}"
+              base_name="$(basename ${file})"
+              base_extless_name="$(basename ${extensionless_name})"
+              mkdir "${extensionless_name}_output"
+              echo "Running \"${file}\" in output folder \"${base_extless_name}_output\""
+              sim_input="/root/${base_name}"
+              sim_output="/root/${base_extless_name}_output"
+              docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i ${sim_input} -o ${sim_output} --writeLogFiles --guaranteeGoodReturnCode
+        
+              echo -e "\n\n\n\n\n" # give ourselves some line separation
+              # Check for individual Success
+              if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name errorLog.txt | wc -l) -ne 0 ]
+              then
+                # Copy error log to github workspace
+                touch ${{ github.workspace }}/errors.report
+                cat $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name errorLog.txt) >> ${{ github.workspace }}/errors.report
+              else
+                echo "No errors found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
               fi
-            done
-          fi
+            fi
+          done
 
         #run: docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root" -o "/root/output" --writeLogFiles # grabs image and runs
 
@@ -323,24 +328,24 @@ jobs:
         uses: mxschmitt/action-tmate@v3
         if: ${{ failure() }}
 
-#  published_results:
-#    runs-on: ubuntu-20.04
-#    needs: published
-#
-#    steps:
-#      - name: Gather Artifacts
-#        uses: actions/download-artifact@v3
-#        with:
-#          path: ${{ github.workspace }}/published-results
-#
-#      - name: Combine .report artifacts into one file
-#        run: cat $(find $GITHUB_WORKSPACE/published-results -name 'errors.report') > $GITHUB_WORKSPACE/combined.txt
-#
-#      - name: Post results to slack part 1
-#        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Here're the $(cat ${GITHUB_WORKSPACE}/combined.txt | wc -l) published models that didn't pass:" https://slack.com/api/chat.postMessage
-#
-#      - name: Post results to slack part 2
-#        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(cat ${GITHUB_WORKSPACE}/combined.txt)" https://slack.com/api/chat.postMessage
+  published_results:
+    runs-on: ubuntu-20.04
+    needs: published
+
+    steps:
+      - name: Gather Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ github.workspace }}/published-results
+
+      - name: Combine .report artifacts into one file
+        run: cat $(find $GITHUB_WORKSPACE/published-results -name 'errors.report') > $GITHUB_WORKSPACE/combined.txt
+
+      - name: Post results to slack part 1
+        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Here're the $(cat ${GITHUB_WORKSPACE}/combined.txt | wc -l) published models that didn't pass:" https://slack.com/api/chat.postMessage
+
+      - name: Post results to slack part 2
+        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(sort ${GITHUB_WORKSPACE}/combined.txt)" https://slack.com/api/chat.postMessage
 
 
   bmdb_results:
@@ -360,4 +365,4 @@ jobs:
         run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Here're the $(cat ${GITHUB_WORKSPACE}/combined.txt | wc -l) BMDB models that didn't pass:" https://slack.com/api/chat.postMessage
 
       - name: Post results to slack part 2
-        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(cat ${GITHUB_WORKSPACE}/combined.txt)" https://slack.com/api/chat.postMessage
+        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(sort ${GITHUB_WORKSPACE}/combined.txt)" https://slack.com/api/chat.postMessage

From 911cc3330cc0105b6fe45fa06ec078b8a6135976 Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Tue, 17 Dec 2024 10:34:19 -0500
Subject: [PATCH 13/45] Updating to use new command

---
 .github/workflows/NightlyBMDB_CLI.yml | 341 ++++++++++++++++----------
 docker_run.sh                         |   5 +
 2 files changed, 222 insertions(+), 124 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 523eb8ea92..bc57f73b18 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -117,102 +117,129 @@ jobs:
         uses: mxschmitt/action-tmate@v3
         if: ${{ failure() }}
 
-  published:
-    runs-on: ubuntu-20.04
-    needs: build
-    strategy:
-      matrix:
-        sets: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19"]
-    steps:
-      - name: Create Docker Image Dir
-        run: mkdir /tmp/docker
-
-      - name: Download Docker Image from `build` Job
-        uses: actions/download-artifact@v3
-        with:
-          name: docker-image
-          path: /tmp/docker
-
-      - name: Load Downloaded Docker Image
-        run: docker load --input /tmp/docker/$(ls /tmp/docker)
-
-        # Get files from BMDB
-      - name: Checkout Input Files
-        uses: actions/checkout@v4
-        with:
-          repository: virtualcell/vcdb
-
-      - name: Prepare sub directory
-        run: |
-          SOURCE_DIR="$GITHUB_WORKSPACE/published/biomodel/omex/sbml"
-          TARGET_DIR="$GITHUB_WORKSPACE/set_${{ matrix.sets }}"
-          MOD_VALUE=${{ matrix.sets }}
-          file_index=0
-
-          # make target dir
-          mkdir $TARGET_DIR
-          # loop through files in directory, increment indexing and separating files
-          for file in $SOURCE_DIR/*; do
-              file_index=$((file_index+1))
-              if [ $((file_index%${{ strategy.job-total }})) -eq $((MOD_VALUE + 0)) ]; then
-                  cp ${file} $TARGET_DIR/
-              fi
-          done
-
-        # Do the execution
-      - name: Begin Execution
-        #$(docker image ls | grep "<none>" | awk '{print $3;}')  to get image id
-        run: |
-          # The /* goes on the outside, otherwise bash just interprets a string!
-          for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
-            echo "\"$file\""
-            if [ -f "$file" ]; then
-              extensionless_name="${file%.*}"
-              base_name="$(basename ${file})"
-              base_extless_name="$(basename ${extensionless_name})"
-              mkdir "${extensionless_name}_output"
-              echo "Running \"${file}\" in output folder \"${base_extless_name}_output\""
-              sim_input="/root/${base_name}"
-              sim_output="/root/${base_extless_name}_output"
-              docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i ${sim_input} -o ${sim_output} --writeLogFiles --guaranteeGoodReturnCode
-          
-              echo -e "\n\n\n\n\n" # give ourselves some line separation
-              # Check for individual Success
-              if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name errorLog.txt | wc -l) -ne 0 ]
-              then
-                # Copy error log to github workspace
-                touch ${{ github.workspace }}/errors.report
-                cat $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name errorLog.txt) >> ${{ github.workspace }}/errors.report
-              else
-                echo "No errors found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
-              fi
-            fi
-          done
-
-      - name: Check For Success
-        run: |
-          # If the error file exists, then we have failures to report and upload; else, we can just end here; the next job will process results
-          if [ $(find ${{ github.workspace }} -name errors.report | wc -l) -eq 0 ]
-          then
-            exit 0
-          fi
-          echo "Models that failed:"
-          cat ${{ github.workspace }}/errors.report
-
-      - name: Upload failures
-        uses: actions/upload-artifact@v3
-        with:
-          name: set_${{ matrix.sets }}.report
-          path: ${{ github.workspace }}/errors.report
-
-      # Report a Problem to Slack
-      - name: Report Problem to Slack
-        if: ${{ failure() }}
-        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Oops! The *Published Step* of set ${{ matrix.sets }} in Nightly **Execution** Testing Action on GitHub Failed ( ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} ). Go see what happened!" https://slack.com/api/chat.postMessage
-
-      - name: Setup tmate session 3
-        uses: mxschmitt/action-tmate@v3
-        if: ${{ failure() }}
+#  published:
+#    runs-on: ubuntu-20.04
+#    needs: build
+#    strategy:
+#      matrix:
+#        sets: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19"]
+#    steps:
+#      - name: Create Docker Image Dir
+#        run: mkdir /tmp/docker
+#
+#      - name: Download Docker Image from `build` Job
+#        uses: actions/download-artifact@v3
+#        with:
+#          name: docker-image
+#          path: /tmp/docker
+#
+#      - name: Load Downloaded Docker Image
+#        run: docker load --input /tmp/docker/$(ls /tmp/docker)
+#
+#        # Get files from BMDB
+#      - name: Checkout Input Files
+#        uses: actions/checkout@v4
+#        with:
+#          repository: virtualcell/vcdb
+#
+#      - name: Prepare sub directory
+#        run: |
+#          SOURCE_DIR="$GITHUB_WORKSPACE/published/biomodel/omex/sbml"
+#          TARGET_DIR="$GITHUB_WORKSPACE/set_${{ matrix.sets }}"
+#          MOD_VALUE=${{ matrix.sets }}
+#          file_index=0
+#
+#          # make target dir
+#          mkdir $TARGET_DIR
+#          # loop through files in directory, increment indexing and separating files
+#          for file in $SOURCE_DIR/*; do
+#              file_index=$((file_index+1))
+#              if [ $((file_index%${{ strategy.job-total }})) -eq $((MOD_VALUE + 0)) ]; then
+#                  cp ${file} $TARGET_DIR/
+#              fi
+#          done
+#
+#        # Do the execution
+#      - name: Begin Execution
+#        #$(docker image ls | grep "<none>" | awk '{print $3;}')  to get image id
+#        run: |
+#          # Prepare files
+#          echo "[]" > ${{ github.workspace }}/total_exec_summary.json
+#          echo "[]" > ${{ github.workspace }}/full_tracer.json
+#          # The /* goes on the outside, otherwise bash just interprets a string!
+#          for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
+#            echo "\"$file\""
+#            if [ -f "$file" ]; then
+#              extensionless_name="${file%.*}"
+#              base_name="$(basename ${file})"
+#              base_extless_name="$(basename ${extensionless_name})"
+#              mkdir "${extensionless_name}_output"
+#              echo "Running \"${file}\" in output folder \"${base_extless_name}_output\""
+#              sim_input="/root/${base_name}"
+#              sim_output="/root/${base_extless_name}_output"
+#              vcell_result=$(docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute-omex -d -i ${sim_input} -o ${sim_output} --timeout_ms=300000)
+#
+#              echo -e "\n\n\n\n\n" # give ourselves some line separation
+#
+#              # Grab Summary
+#              if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json | wc -l) -ne 0 ]
+#              then
+#                # Append to master summary in github workspace
+#                jq '. + [inputs]' ${{ github.workspace }}/total_exec_summary.json $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json) > temp.json
+#                mv temp.json ${{ github.workspace }}/total_exec_summary.json
+#              else
+#                echo "No summary found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
+#              fi
+#
+#              # Check for individual Failure (a.k.a. if we failed early)
+#              status=$(cat $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json) | jq -r '.status')
+#              if [[ "$status" == "FAILED" ]] || [ ${vcell_result} -ne 0 ]
+#              then
+#                # Copy error log to github workspace
+#                echo "${base_extless_name}" >> $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output/errorLog.txt
+#                touch ${{ github.workspace }}/errors.report
+#                cat $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output/errorLog.txt >> ${{ github.workspace }}/errors.report
+#              else
+#                echo "Run of ${base_extless_name} succeeded (set_${{ matrix.sets }})"
+#              fi
+#
+#              # Grab Trace
+#              # Check for individual Success
+#              if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name tracer.json | wc -l) -ne 0 ]
+#              then
+#                # Append to master trace in github workspace
+#                jq '. + inputs' ${{ github.workspace }}/full_tracer.json $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name tracer.json) > temp_2.json
+#                mv temp_2.json ${{ github.workspace }}/full_tracer.json
+#              else
+#                echo "No trace found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
+#              fi
+#            fi
+#          done
+#
+#      - name: Check For Success
+#        run: |
+#          # If the error file exists, then we have failures to report and upload; else, we can just end here; the next job will process results
+#          if [ $(find ${{ github.workspace }} -name errors.report | wc -l) -eq 0 ]
+#          then
+#            exit 0
+#          fi
+#          echo "Models that failed:"
+#          cat ${{ github.workspace }}/errors.report
+#
+#      - name: Upload failures
+#        uses: actions/upload-artifact@v3
+#        with:
+#          name: set_${{ matrix.sets }}.report
+#          path: ${{ github.workspace }}/errors.report
+#
+#      # Report a Problem to Slack
+#      - name: Report Problem to Slack
+#        if: ${{ failure() }}
+#        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Oops! The *Published Step* of set ${{ matrix.sets }} in Nightly **Execution** Testing Action on GitHub Failed ( ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} ). Go see what happened!" https://slack.com/api/chat.postMessage
+#
+#      - name: Setup tmate session 3
+#        uses: mxschmitt/action-tmate@v3
+#        if: ${{ failure() }}
 
   bmdb:
     runs-on: ubuntu-20.04
@@ -271,37 +298,76 @@ jobs:
               fi
           done
 
+      - name: Install JQ
+        run: sudo apt update && sudo apt install jq -y
+
         # Do the execution
       - name: Begin Execution
         #$(docker image ls | grep "<none>" | awk '{print $3;}')  to get image id
         run: |
+          # Prepare files
+          set +e
+          echo "[]" > ${{ github.workspace }}/total_exec_summary.json
+          echo "[]" > ${{ github.workspace }}/full_tracer.json
           # The /* goes on the outside, otherwise bash just interprets a string!
           for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
-            echo "\"$file\""
             if [ -f "$file" ]; then
               extensionless_name="${file%.*}"
               base_name="$(basename ${file})"
               base_extless_name="$(basename ${extensionless_name})"
               mkdir "${extensionless_name}_output"
-              echo "Running \"${file}\" in output folder \"${base_extless_name}_output\""
+              
               sim_input="/root/${base_name}"
               sim_output="/root/${base_extless_name}_output"
-              docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i ${sim_input} -o ${sim_output} --writeLogFiles --guaranteeGoodReturnCode
-        
-              echo -e "\n\n\n\n\n" # give ourselves some line separation
-              # Check for individual Success
-              if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name errorLog.txt | wc -l) -ne 0 ]
+              echo -n "Running \"${file}\" in output folder \"${base_extless_name}_output\"..."
+              vcell_result=$(docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute-omex -d -i ${sim_input} -o ${sim_output} --timeout_ms=300000)
+              echo "Done!"
+              # Grab Summary
+              echo -n "Obtaining Summary..."
+              if [ -f "${GITHUB_WORKSPACE}/set_${{ matrix.sets }}/${base_extless_name}_output/exec_summary.json" ]
               then
+                # Append to master summary in github workspace
+                jq '. + [inputs]' ${{ github.workspace }}/total_exec_summary.json $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json) > temp.json
+                mv temp.json ${{ github.workspace }}/total_exec_summary.json
+                echo "Done!"
+              else
+                echo "No summary found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
+              fi
+          
+              # Check for individual Failure (a.k.a. if we failed early)
+              echo -n "Obtaining Execution Result..."
+              status=$(cat "${GITHUB_WORKSPACE}/set_${{ matrix.sets }}/${base_extless_name}_output/exec_summary.json" | jq -r '.status')
+              echo "${status}"
+              # Note: `cat`ing "$vcell_result" will be the runting logging! 
+              if [[ "${status}" == "FAILED" ]]; then
                 # Copy error log to github workspace
+                echo -n "Copying Summary..."
+                echo "${base_extless_name}" >> $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output/errorLog.txt
                 touch ${{ github.workspace }}/errors.report
-                cat $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name errorLog.txt) >> ${{ github.workspace }}/errors.report
+                cat $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output/errorLog.txt >> ${{ github.workspace }}/errors.report
+                echo "Done!"
               else
-                echo "No errors found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
+                echo "Run of ${base_extless_name} succeeded (set_${{ matrix.sets }})"
+              fi
+          
+              # Grab Trace
+              # Check for individual Success
+              echo -n "Obtaining Execution Tracer Logs..."
+              if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name tracer.json | wc -l) -ne 0 ]
+              then
+                echo -n "Copying Tracer Logs..."
+                # Append to master trace in github workspace
+                jq '. + inputs' ${{ github.workspace }}/full_tracer.json $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name tracer.json) > temp_2.json
+                mv temp_2.json ${{ github.workspace }}/full_tracer.json
+                echo "Done!"
+              else
+                echo "No trace found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
               fi
             fi
+          echo "Completed processing of \"${file}\"!"
+          echo "\n\n-----------------------------------------------------------------------------------------------\n\n"
           done
 
-        #run: docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute -d -i "/root" -o "/root/output" --writeLogFiles # grabs image and runs
 
       - name: Check For Success
         run: |
@@ -319,6 +385,18 @@ jobs:
           name: set_${{ matrix.sets }}.report
           path: ${{ github.workspace }}/errors.report
 
+      - name: Upload summary
+        uses: actions/upload-artifact@v3
+        with:
+          name: set_${{ matrix.sets }}.summary
+          path: ${{ github.workspace }}/total_exec_summary.json
+
+      - name: Upload tracer
+        uses: actions/upload-artifact@v3
+        with:
+          name: set_${{ matrix.sets }}.tracer
+          path: ${{ github.workspace }}/full_tracer.json
+
       # Report a Problem to Slack
       - name: Report Problem to Slack
         if: ${{ failure() }}
@@ -328,24 +406,24 @@ jobs:
         uses: mxschmitt/action-tmate@v3
         if: ${{ failure() }}
 
-  published_results:
-    runs-on: ubuntu-20.04
-    needs: published
-
-    steps:
-      - name: Gather Artifacts
-        uses: actions/download-artifact@v3
-        with:
-          path: ${{ github.workspace }}/published-results
-
-      - name: Combine .report artifacts into one file
-        run: cat $(find $GITHUB_WORKSPACE/published-results -name 'errors.report') > $GITHUB_WORKSPACE/combined.txt
-
-      - name: Post results to slack part 1
-        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Here're the $(cat ${GITHUB_WORKSPACE}/combined.txt | wc -l) published models that didn't pass:" https://slack.com/api/chat.postMessage
-
-      - name: Post results to slack part 2
-        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(sort ${GITHUB_WORKSPACE}/combined.txt)" https://slack.com/api/chat.postMessage
+#  published_results:
+#    runs-on: ubuntu-20.04
+#    needs: published
+#
+#    steps:
+#      - name: Gather Artifacts
+#        uses: actions/download-artifact@v3
+#        with:
+#          path: ${{ github.workspace }}/published-results
+#
+#      - name: Combine .report artifacts into one file
+#        run: cat $(find $GITHUB_WORKSPACE/published-results -name 'errors.report') > $GITHUB_WORKSPACE/combined.txt
+#
+#      - name: Post results to slack part 1
+#        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Here're the $(cat ${GITHUB_WORKSPACE}/combined.txt | wc -l) published models that didn't pass:" https://slack.com/api/chat.postMessage
+#
+#      - name: Post results to slack part 2
+#        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(sort ${GITHUB_WORKSPACE}/combined.txt)" https://slack.com/api/chat.postMessage
 
 
   bmdb_results:
@@ -358,11 +436,26 @@ jobs:
         with:
           path: ${{ github.workspace }}/bmdb-results
 
+      - name: Install JQ
+        run: sudo apt update && sudo apt install jq -y
+
       - name: Combine .report artifacts into one file
         run: cat $(find $GITHUB_WORKSPACE/bmdb-results -name 'errors.report') > $GITHUB_WORKSPACE/combined.txt
 
+      - name: Combine .summary artifacts into one file
+        run: jq -s 'add' $(find $GITHUB_WORKSPACE/bmdb-results -name 'total_exec_summary.json') > $GITHUB_WORKSPACE/summary.json
+
+      - name: Combine .summary artifacts into one file
+        run: jq -s 'add' $(find $GITHUB_WORKSPACE/bmdb-results -name 'full_tracer.json') > $GITHUB_WORKSPACE/tracer.json
+
       - name: Post results to slack part 1
         run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Here're the $(cat ${GITHUB_WORKSPACE}/combined.txt | wc -l) BMDB models that didn't pass:" https://slack.com/api/chat.postMessage
 
       - name: Post results to slack part 2
         run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(sort ${GITHUB_WORKSPACE}/combined.txt)" https://slack.com/api/chat.postMessage
+
+      - name: Post results to slack part 3
+        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F file=$GITHUB_WORKSPACE/summary.json https://slack.com/api/files.upload
+
+      - name: Post results to slack part 4
+        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F file=$GITHUB_WORKSPACE/tracer.json https://slack.com/api/files.upload
\ No newline at end of file
diff --git a/docker_run.sh b/docker_run.sh
index b46dff546d..c86f2b53ab 100755
--- a/docker_run.sh
+++ b/docker_run.sh
@@ -47,6 +47,11 @@ case "$rawCommand" in
     command="execute"
     shift
     ;;
+  "execute-omex")
+      echo 'special testing execution mode requested'
+      command="execute-omex"
+      shift
+      ;;
   "validate")
     echo 'validate mode requested'
     command="validate"

From c868cbb0af75be178afd9d3f7c58616abb46e919 Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Fri, 20 Dec 2024 01:48:01 -0500
Subject: [PATCH 14/45] accommodate path prefix clash, add reporting

---
 .../java/org/vcell/cli/CLIStandalone.java     |   2 +
 .../vcell/cli/testsupport/OmexTestCase.java   |  16 +-
 .../vcell/cli/testsupport/OmexTestReport.java | 134 ++++++++++++++
 .../cli/testsupport/OmexTestingDatabase.java  |  45 +++--
 .../org/vcell/cli/testsupport/PathUtils.java  |  43 +++++
 .../cli/testsupport/TestReportCommand.java    |  62 +++++++
 .../testsupport/TestOmexTestingDatabase.java  | 175 +++++++++++++++---
 7 files changed, 432 insertions(+), 45 deletions(-)
 create mode 100644 vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java
 create mode 100644 vcell-cli/src/main/java/org/vcell/cli/testsupport/PathUtils.java
 create mode 100644 vcell-cli/src/main/java/org/vcell/cli/testsupport/TestReportCommand.java

diff --git a/vcell-cli/src/main/java/org/vcell/cli/CLIStandalone.java b/vcell-cli/src/main/java/org/vcell/cli/CLIStandalone.java
index 05d2fc8e2e..e4b79ba1e8 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/CLIStandalone.java
+++ b/vcell-cli/src/main/java/org/vcell/cli/CLIStandalone.java
@@ -4,6 +4,7 @@
 import org.vcell.cli.run.ExecuteCommand;
 import org.vcell.cli.run.ExecuteOmexCommand;
 import org.vcell.cli.sbml.ModelCommand;
+import org.vcell.cli.testsupport.TestReportCommand;
 import org.vcell.cli.vcml.*;
 
 import org.apache.logging.log4j.LogManager;
@@ -24,6 +25,7 @@
         ImportOmexCommand.class,
         ImportOmexBatchCommand.class,
         ExecuteOmexCommand.class,
+        TestReportCommand.class,
         ExecuteCommand.class,
         VersionCommand.class,
         ModelCommand.class,
diff --git a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestCase.java b/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestCase.java
index 6a1f44a3ce..c312dacec6 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestCase.java
+++ b/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestCase.java
@@ -15,6 +15,18 @@ public enum Status {
     public FailureType known_failure_type;
     public String known_failure_desc;
 
+    public OmexTestCase(OmexTestingDatabase.TestCollection testCollection, String filePath, Boolean shouldFail, Status status, FailureType failureType, String failureDesc) {
+        this.test_collection = testCollection;
+        this.file_path = filePath;
+        this.should_fail = shouldFail;
+        this.known_status = status;
+        this.known_failure_type = failureType;
+        this.known_failure_desc = failureDesc;
+    }
+
+    public OmexTestCase() {
+    }
+
     @Override
     public String toString() {
         return "OmexTestCase{" +
@@ -27,8 +39,4 @@ public String toString() {
                 '}';
     }
 
-    public boolean matchFileSuffix(String path) {
-        String fullPath = test_collection.pathPrefix + "/" + file_path;
-        return fullPath.endsWith(path);
-    }
 }
diff --git a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java b/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java
new file mode 100644
index 0000000000..3add3226f4
--- /dev/null
+++ b/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java
@@ -0,0 +1,134 @@
+package org.vcell.cli.testsupport;
+
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
+public class OmexTestReport {
+
+    public static class OmexTestCaseChange {
+        public final OmexTestCase original;
+        public final OmexTestCase updated;
+
+        public OmexTestCaseChange(OmexTestCase original, OmexTestCase updated) {
+            this.original = original;
+            this.updated = updated;
+        }
+    }
+
+    public static class OmexTestStatistics {
+        public int testCaseCount;
+        public int testCaseChangeCount;
+        public int unmatchedTestCaseCount;
+        public int unmatchedExecutionsCount;
+        public int failedExecutionsCount;
+        public int passedExecutionsCount;
+        public int totalExecutions;
+    }
+
+    public final List<OmexTestCaseChange> testCaseChanges = new ArrayList<>();
+    public final List<OmexExecSummary> unmatchedExecSummaries = new ArrayList<>();
+    public final List<OmexTestCase> unmatchedTestCases = new ArrayList<>();
+    public final OmexTestStatistics statistics = new OmexTestStatistics();
+
+    public OmexTestReport(List<OmexTestCase> testCases, List<OmexExecSummary> execSummaries) {
+        // find common path prefix of list of Path objects
+        Path commonPrefix = PathUtils.findCommonPrefix(execSummaries.stream().map(tc -> Paths.get(tc.file_path)).toList());
+
+        unmatchedTestCases.addAll(testCases);
+        for (OmexExecSummary execSummary : execSummaries) {
+            List<OmexTestCase> matchingTestCases = OmexTestingDatabase.queryOmexTestCase(testCases, Paths.get(execSummary.file_path), commonPrefix);
+            if (matchingTestCases.isEmpty()) {
+                System.out.println("No test case found for: " + execSummary.file_path);
+                unmatchedExecSummaries.add(execSummary);
+                continue;
+            }
+            if (matchingTestCases.size()>1){
+                throw new RuntimeException("Multiple test cases "+matchingTestCases+" found for: " + execSummary.file_path);
+            }
+            OmexTestCase matchingTestCase = matchingTestCases.get(0);
+            unmatchedTestCases.remove(matchingTestCase);
+            OmexTestCase updatedTestCase = null;
+            if (execSummary.status == OmexExecSummary.ActualStatus.PASSED){
+                if (matchingTestCase.known_status == OmexTestCase.Status.FAIL || matchingTestCase.known_status == OmexTestCase.Status.SKIP || matchingTestCase.known_status == null) {
+                    System.out.println("Test case marked as FAIL or SKIP, type="+matchingTestCase.known_failure_type+", but passed: " + execSummary.file_path);
+                    updatedTestCase = new OmexTestCase(matchingTestCase.test_collection, matchingTestCase.file_path,
+                                    matchingTestCase.should_fail, OmexTestCase.Status.PASS, null, null);
+                }
+            }
+            if (execSummary.status == OmexExecSummary.ActualStatus.FAILED){
+                if (matchingTestCase.known_status == OmexTestCase.Status.PASS || matchingTestCase.known_status == null) {
+                    System.out.println("Test case marked as "+matchingTestCase.known_status+", but failed "+execSummary.failure_type+": "+execSummary.failure_desc+", "+execSummary.file_path);
+                    updatedTestCase = new OmexTestCase(matchingTestCase.test_collection, matchingTestCase.file_path,
+                            matchingTestCase.should_fail, OmexTestCase.Status.FAIL,
+                            execSummary.failure_type, execSummary.failure_desc);
+                } else if (matchingTestCase.known_status == OmexTestCase.Status.SKIP) {
+                    System.out.println("Test case marked as SKIP and failed with "+execSummary.failure_type+": "+execSummary.failure_desc+", "+execSummary.file_path);
+                    updatedTestCase = new OmexTestCase(matchingTestCase.test_collection, matchingTestCase.file_path,
+                            matchingTestCase.should_fail, OmexTestCase.Status.SKIP,
+                            execSummary.failure_type, execSummary.failure_desc);
+                } else if (matchingTestCase.known_status == OmexTestCase.Status.FAIL) {
+                    if (matchingTestCase.known_failure_type == null || !matchingTestCase.known_failure_type.equals(execSummary.failure_type)) {
+                        System.out.println("Test case marked as FAIL with different type "+matchingTestCase.known_failure_type+", but failed with "+execSummary.failure_type+": "+execSummary.failure_desc+", "+execSummary.file_path);
+                        updatedTestCase = new OmexTestCase(matchingTestCase.test_collection, matchingTestCase.file_path,
+                                matchingTestCase.should_fail, OmexTestCase.Status.FAIL,
+                                execSummary.failure_type, execSummary.failure_desc);
+                    }
+                }
+            }
+            if (updatedTestCase != null) {
+                testCaseChanges.add(new OmexTestCaseChange(matchingTestCase, updatedTestCase));
+            }
+        }
+        statistics.testCaseCount = testCases.size();
+        statistics.totalExecutions = execSummaries.size();
+        statistics.failedExecutionsCount = (int) execSummaries.stream().filter(s -> s.status == OmexExecSummary.ActualStatus.FAILED).count();
+        statistics.passedExecutionsCount = (int) execSummaries.stream().filter(s -> s.status == OmexExecSummary.ActualStatus.PASSED).count();
+        statistics.testCaseChangeCount = testCaseChanges.size();
+        statistics.unmatchedTestCaseCount = unmatchedTestCases.size();
+        statistics.unmatchedExecutionsCount = unmatchedExecSummaries.size();
+    }
+
+    // find test cases which were not run, they don't have execSummaries
+    public List<OmexTestCase> findUnexecutedTestCases() {
+        return unmatchedTestCases;
+    }
+
+    // find execSummaries which don't have matching test cases
+    public List<OmexExecSummary> findUnmatchedExecSummaries() {
+        return unmatchedExecSummaries;
+    }
+
+    // find test cases which should be updated
+    public List<OmexTestCaseChange> findTestCaseChanges() {
+        return testCaseChanges;
+    }
+
+    public OmexTestStatistics getStatistics() {
+        return statistics;
+    }
+
+    public String toJson() {
+        ObjectMapper objectMapper = new ObjectMapper();
+        try {
+            return objectMapper.writeValueAsString(this);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public String toYaml() {
+        ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());
+        try {
+            return objectMapper.writeValueAsString(this);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+}
diff --git a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestingDatabase.java b/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestingDatabase.java
index d11c3840b5..dcbfbf55ca 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestingDatabase.java
+++ b/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestingDatabase.java
@@ -6,13 +6,14 @@
 import org.vcell.sbml.vcell.SBMLImportException;
 import org.vcell.trace.Span;
 import org.vcell.trace.TraceEvent;
-import org.vcell.trace.Tracer;
 
 import java.io.*;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 public class OmexTestingDatabase {
 
@@ -35,16 +36,19 @@ public enum TestCollection {
         }
     }
 
-    public static List<OmexTestCase> queryOmexTestCase(List<OmexTestCase> omexTestCases, String path) {
-        return omexTestCases.stream().filter(tc -> tc.matchFileSuffix(path)).toList();
+    public static List<OmexTestCase> queryOmexTestCase(List<OmexTestCase> omexTestCases, Path path, Path commonPrefix) {
+        if (commonPrefix != null){
+            path = path.subpath(commonPrefix.getNameCount(), path.getNameCount());
+        }
+        final Path finalPath = path;
+        return omexTestCases.stream().filter(tc -> {
+            Path fullPath = Paths.get(tc.test_collection.pathPrefix, tc.file_path);
+            return fullPath.endsWith(finalPath);
+        }).toList();
     }
 
-    public String generateReport(List<OmexTestCase> omexTestCases, List<OmexExecSummary> execSummaries) {
-        StringBuilder report = new StringBuilder();
-        for (OmexTestCase testCase : omexTestCases) {
-            report.append(testCase.toString()).append("\n");
-        }
-        return report.toString();
+    public static OmexTestReport generateReport(List<OmexTestCase> omexTestCases, List<OmexExecSummary> execSummaries) {
+        return new OmexTestReport(omexTestCases, execSummaries);
     }
 
     public static List<OmexExecSummary> loadOmexExecSummaries(String execSummariesNdjson) throws IOException {
@@ -61,19 +65,20 @@ public static List<OmexExecSummary> loadOmexExecSummaries(String execSummariesNd
 
     // read a newline-delimited json file into a list of OmexTextCase objects
     public static List<OmexTestCase> loadOmexTestCases() throws IOException {
-        List<OmexTestCase> testCases = new ArrayList<>();
         String fileName = "test_cases.ndjson";
+        try (InputStream inputStream = OmexTestingDatabase.class.getResourceAsStream("/"+fileName);
+             BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))){
+            String testCasesNdjson = reader.lines().collect(Collectors.joining(System.lineSeparator()));
+            return parseOmexTestCases(testCasesNdjson);
+        }
+    }
+
+    public static List<OmexTestCase> parseOmexTestCases(String testCasesNdjson) throws IOException {
+        List<OmexTestCase> testCases = new ArrayList<>();
         ObjectMapper objectMapper = new ObjectMapper();
-        try (InputStream inputStream = OmexTestingDatabase.class.getResourceAsStream("/"+fileName);){
-            if (inputStream == null) {
-                throw new FileNotFoundException("file not found! " + fileName);
-            }
-            try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
-                try (MappingIterator<OmexTestCase> it = objectMapper.readerFor(OmexTestCase.class).readValues(reader)) {
-                    while (it.hasNext()) {
-                        testCases.add(it.next());
-                    }
-                }
+        try (MappingIterator<OmexTestCase> it = objectMapper.readerFor(OmexTestCase.class).readValues(testCasesNdjson)) {
+            while (it.hasNext()) {
+                testCases.add(it.next());
             }
         }
         return testCases;
diff --git a/vcell-cli/src/main/java/org/vcell/cli/testsupport/PathUtils.java b/vcell-cli/src/main/java/org/vcell/cli/testsupport/PathUtils.java
new file mode 100644
index 0000000000..add1be3fad
--- /dev/null
+++ b/vcell-cli/src/main/java/org/vcell/cli/testsupport/PathUtils.java
@@ -0,0 +1,43 @@
+package org.vcell.cli.testsupport;
+
+import java.nio.file.Path;
+import java.util.List;
+
+public class PathUtils {
+    public static Path findCommonPrefix(List<Path> paths) {
+        System.out.println(paths.get(0).subpath(0, 1));
+        if (paths == null || paths.isEmpty()) {
+            return null;
+        }
+
+        Path commonPrefix = paths.get(0);
+
+        for (Path path : paths) {
+            commonPrefix = findCommonPrefix(commonPrefix, path);
+            if (commonPrefix == null) {
+                break;
+            }
+        }
+
+        return commonPrefix;
+    }
+
+    private static Path findCommonPrefix(Path path1, Path path2) {
+        int minLength = Math.min(path1.getNameCount(), path2.getNameCount());
+        Path commonPrefix = path1.getRoot();
+
+        for (int i = 0; i < minLength; i++) {
+            if (path1.getName(i).equals(path2.getName(i))) {
+                if (commonPrefix == null) {
+                    commonPrefix = path1.getName(i);
+                } else {
+                    commonPrefix = commonPrefix.resolve(path1.getName(i));
+                }
+            } else {
+                break;
+            }
+        }
+
+        return commonPrefix;
+    }
+}
diff --git a/vcell-cli/src/main/java/org/vcell/cli/testsupport/TestReportCommand.java b/vcell-cli/src/main/java/org/vcell/cli/testsupport/TestReportCommand.java
new file mode 100644
index 0000000000..c877d4d96e
--- /dev/null
+++ b/vcell-cli/src/main/java/org/vcell/cli/testsupport/TestReportCommand.java
@@ -0,0 +1,62 @@
+package org.vcell.cli.testsupport;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.LoggerContext;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.concurrent.Callable;
+
+@Command(name = "test-report", description = "create test reports for a test suite")
+public class TestReportCommand implements Callable<Integer> {
+
+    private final static Logger logger = LogManager.getLogger(TestReportCommand.class);
+
+    @Option(names = { "-t", "--test-cases" }, description = "[optional] test cases file - defaults to embedded test cases")
+    private File testCasesNdjsonFile;
+
+    @Option(names = { "-e", "--exec-summaries" }, required = true, description = "test results file e.g. exec_summary.ndjson")
+    private File execSummaryNdjsonFile;
+
+    @Option(names = { "-r", "--report" }, required = false, description = "filename for generated test report (e.g. ./report.md)")
+    private File reportFile;
+
+    // flag weather output should be json or yaml
+    @Option(names = { "-j", "--json" }, description = "output report in json format")
+    private boolean bJson = false;
+
+    @Option(names = { "-d", "--debug" }, description = "enable debug logging")
+    private boolean bDebug = false;
+
+    public Integer call() {
+        Level logLevel = bDebug ? Level.DEBUG : logger.getLevel(); 
+        
+        LoggerContext config = (LoggerContext)(LogManager.getContext(false));
+        config.getConfiguration().getLoggerConfig(LogManager.getLogger("org.vcell").getName()).setLevel(logLevel);
+        config.getConfiguration().getLoggerConfig(LogManager.getLogger("cbit").getName()).setLevel(logLevel);
+        config.updateLoggers();
+
+        try {
+            List<OmexTestCase> testCaseList = OmexTestingDatabase.loadOmexTestCases();
+            // read fully text file into a string from file 'execSummaryNdjsonFile'
+            String exec_summary_contents = Files.readString(execSummaryNdjsonFile.toPath());
+            List<OmexExecSummary> execSummaries = OmexTestingDatabase.loadOmexExecSummaries(exec_summary_contents);
+            OmexTestReport report = OmexTestingDatabase.generateReport(testCaseList, execSummaries);
+            if (reportFile != null) {
+                Files.writeString(reportFile.toPath(), bJson ? report.toJson() : report.toYaml());
+            } else {
+                System.out.println(bJson ? report.toJson() : report.toYaml());
+            }
+            return 0;
+        } catch (Exception e) {
+            throw new RuntimeException(e.getMessage(), e);
+        } finally {
+            logger.debug("Completed all exports");
+        }
+    }
+}
diff --git a/vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestingDatabase.java b/vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestingDatabase.java
index 6e16ed9fe7..a23ea19408 100644
--- a/vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestingDatabase.java
+++ b/vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestingDatabase.java
@@ -4,6 +4,8 @@
 import org.junit.jupiter.api.Test;
 
 import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.List;
 
 
@@ -30,34 +32,165 @@ void execSummariesLoad() throws IOException {
 
     @Test
     void queryOmexTestCase() throws IOException {
-        String[] shouldFindUnique = {
-                "sedml/SimulatorSupportsAddReplaceRemoveModelElementChanges/2.execution-should-succeed.omex",
-                "bsts-omex/misc-projects/BIOMD0000000842.omex",
-                "omex_files/BIOMD0000000842.omex"
-        };
-        String[] shouldFindMany = {
-                "11.omex",
-                "BIOMD0000000842.omex",
-                "2.execution-should-succeed.omex"
-        };
-        String[] shouldNotFind = {
-                "11111.omex",
-                "2.execution-should-succeed",
-                "sedml/SimulatorSupportsAddReplaceRemoveModelElementChanges"
-        };
+        List<Path> shouldFindUnique = List.of(
+                Paths.get("sedml/SimulatorSupportsAddReplaceRemoveModelElementChanges/2.execution-should-succeed.omex"),
+                Paths.get("bsts-omex/misc-projects/BIOMD0000000842.omex"),
+                Paths.get("omex_files/BIOMD0000000842.omex")
+        );
+        List<Path> shouldFindMany = List.of(
+                Paths.get("/root/BIOMD0000000842.omex"),
+                Paths.get("/root/2.execution-should-succeed.omex")
+        );
+        List<Path> shouldNotFind = List.of(
+                Paths.get("11111.omex"),
+                Paths.get("2.execution-should-succeed"),
+                Paths.get("sedml/SimulatorSupportsAddReplaceRemoveModelElementChanges")
+        );
         List<OmexTestCase> allTestCases = OmexTestingDatabase.loadOmexTestCases();
-        for (String path : shouldFindUnique) {
-            List<OmexTestCase> omexTestCase = OmexTestingDatabase.queryOmexTestCase(allTestCases, path);
+        for (Path path : shouldFindUnique) {
+            List<OmexTestCase> omexTestCase = OmexTestingDatabase.queryOmexTestCase(allTestCases, path, PathUtils.findCommonPrefix(shouldFindUnique));
             Assertions.assertEquals(1, omexTestCase.size());
         }
-        for (String path : shouldFindMany) {
-            List<OmexTestCase> omexTestCase = OmexTestingDatabase.queryOmexTestCase(allTestCases, path);
+        for (Path path : shouldFindMany) {
+            List<OmexTestCase> omexTestCase = OmexTestingDatabase.queryOmexTestCase(allTestCases, path, PathUtils.findCommonPrefix(shouldFindMany));
             Assertions.assertTrue(omexTestCase.size() > 1);
         }
-        for (String path : shouldNotFind) {
-            List<OmexTestCase> omexTestCase = OmexTestingDatabase.queryOmexTestCase(allTestCases, path);
+        for (Path path : shouldNotFind) {
+            List<OmexTestCase> omexTestCase = OmexTestingDatabase.queryOmexTestCase(allTestCases, path, PathUtils.findCommonPrefix(shouldNotFind));
             Assertions.assertTrue(omexTestCase.isEmpty());
         }
     }
 
+    // test generateReport
+    @Test
+    void generateReport() throws IOException {
+
+        String test_cases_ndjson = """
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000039.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000060.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000080.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000100.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000120.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000140.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000160.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000180.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000200.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000220.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000240.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000260.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000280.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000300.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000320.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000340.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000360.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000380.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000400.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000420.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000421.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000440.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000460.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000480.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000500.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000520.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000540.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000560.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000580.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000600.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000620.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000640.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000660.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000680.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000700.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000720.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000740.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000760.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000780.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000800.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000820.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000840.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000860.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000880.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000900.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000920.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000940.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000960.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000980.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001000.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001020.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001040.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001060.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001080.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+                                
+                """;
+        String exec_summary_ndjson = """
+                {  "file_path": "/root/BIOMD0000000020.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000040.omex",  "status": "FAILED",  "failure_type": "SBML_IMPORT_FAILURE",  "failure_desc": "Unable to initialize bioModel for the given selection: Failed to translate SBML model into BioModel: org.vcell.sbml.vcell.SBMLImportException: Non-numeric stoichiometry ('f' for product 'Br' in reaction 'Reaction5') not handled in VCell at this time. org.vcell.sbml.vcell.SBMLImportException: Failed to translate SBML model into BioModel: org.vcell.sbml.vcell.SBMLImportException: Non-numeric stoichiometry ('f' for product 'Br' in reaction 'Reaction5') not handled in VCell at this time." }
+                {  "file_path": "/root/BIOMD0000000060.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000080.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000100.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000120.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000140.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000240.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000260.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000280.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000300.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000320.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000340.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000360.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000380.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000400.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000420.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000440.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000460.omex",  "status": "FAILED",  "failure_type": "SBML_IMPORT_FAILURE",  "failure_desc": "Unable to initialize bioModel for the given selection: couldn't find SBase with sid=null in SBMLSymbolMapping org.vcell.sbml.vcell.SBMLImportException: couldn't find SBase with sid=null in SBMLSymbolMapping" }
+                {  "file_path": "/root/BIOMD0000000480.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000500.omex",  "status": "FAILED",  "failure_type": "UNCATETORIZED_FAULT",  "failure_desc": "Unable to initialize bioModel for the given selection: Cannot invoke \\"cbit.vcell.solver.Simulation.getImportedTaskID()\\" because \\"simulation\\" is null java.lang.NullPointerException: Cannot invoke \\"cbit.vcell.solver.Simulation.getImportedTaskID()\\" because \\"simulation\\" is null" }
+                {  "file_path": "/root/BIOMD0000000520.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000540.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000560.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000580.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000600.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000620.omex",  "status": "FAILED",  "failure_type": "UNCATETORIZED_FAULT",  "failure_desc": "Failed execution: Model 'BIOMD0000000620_BIOMD0000000620_url.sedml_BIOMD0000000620_url' Task 'task1'. java.lang.RuntimeException: Could not execute code: CVODE solver failed : at time 5.5, discontinuity (t == Anakinra_dose_counter) evaluated to TRUE, solver assumed FALSE\\n\\n\\n\\n(/usr/local/app/vcell/installDir/localsolvers/linux64/SundialsSolverStandalone_x64 /tmp/VCell_CLI_193e0a5db7a967308657490923326/BIOMD0000000620/BIOMD0000000620_url.sedml/SimID_411786192_0_.cvodeInput /tmp/VCell_CLI_193e0a5db7a967308657490923326/BIOMD0000000620/BIOMD0000000620_url.sedml/SimID_411786192_0_.ida) " }
+                {  "file_path": "/root/BIOMD0000000640.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000660.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000680.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000700.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000720.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000740.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000760.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000780.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000800.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000820.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000840.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000860.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000880.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000900.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000920.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000940.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000960.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000000980.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000001000.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000001020.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000001040.omex",  "status": "FAILED",  "failure_type": "SBML_IMPORT_FAILURE",  "failure_desc": "Unable to initialize bioModel for the given selection: Failed to translate SBML model into BioModel: Error binding global parameter 'Summary_flux_to_RBC' to model: 'Vin' is either not found in your model or is not allowed to be used in the current context. Check that you have provided the correct and full name (e.g. Ca_Cytosol). org.vcell.sbml.vcell.SBMLImportException: Failed to translate SBML model into BioModel: Error binding global parameter 'Summary_flux_to_RBC' to model: 'Vin' is either not found in your model or is not allowed to be used in the current context. Check that you have provided the correct and full name (e.g. Ca_Cytosol)." }
+                {  "file_path": "/root/BIOMD0000001060.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                {  "file_path": "/root/BIOMD0000001080.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+                """;
+
+        List<OmexTestCase> testCases = OmexTestingDatabase.parseOmexTestCases(test_cases_ndjson);
+        List<OmexExecSummary> omexExecSummaries = OmexTestingDatabase.loadOmexExecSummaries(exec_summary_ndjson);
+        OmexTestReport report = OmexTestingDatabase.generateReport(testCases, omexExecSummaries);
+        Assertions.assertEquals(50, report.getStatistics().totalExecutions);
+        Assertions.assertEquals(54, report.getStatistics().testCaseCount);
+        Assertions.assertEquals(5, report.getStatistics().failedExecutionsCount);
+        Assertions.assertEquals(45, report.getStatistics().passedExecutionsCount);
+        Assertions.assertEquals(2, report.getStatistics().unmatchedExecutionsCount);
+        Assertions.assertEquals(6, report.getStatistics().unmatchedTestCaseCount);
+        Assertions.assertEquals(4, report.getStatistics().testCaseChangeCount);
+        Assertions.assertEquals(2, report.unmatchedExecSummaries.size());
+        Assertions.assertEquals(6, report.unmatchedTestCases.size());
+        Assertions.assertEquals(4, report.testCaseChanges.size());
+
+        String reportYaml = report.toYaml();
+        System.out.println(reportYaml);
+    }
+
 }

From ef678e5f0c24342fc7555c10c35ac04374e2d895 Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Fri, 20 Dec 2024 11:32:23 -0500
Subject: [PATCH 15/45] Fixing syntax

---
 .github/workflows/NightlyBMDB_CLI.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index bc57f73b18..01a2e15149 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -341,13 +341,11 @@ jobs:
               # Note: `cat`ing "$vcell_result" will be the runting logging! 
               if [[ "${status}" == "FAILED" ]]; then
                 # Copy error log to github workspace
-                echo -n "Copying Summary..."
+                echo -n "Copying Failure Report..."
                 echo "${base_extless_name}" >> $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output/errorLog.txt
                 touch ${{ github.workspace }}/errors.report
                 cat $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output/errorLog.txt >> ${{ github.workspace }}/errors.report
                 echo "Done!"
-              else
-                echo "Run of ${base_extless_name} succeeded (set_${{ matrix.sets }})"
               fi
           
               # Grab Trace

From 370e1a3b2d8654e14588f52e1818bd8bad6b144d Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Mon, 23 Dec 2024 13:06:41 -0500
Subject: [PATCH 16/45] Remove Published and attempt to upload logs to Slack

---
 .github/workflows/NightlyBMDB_CLI.yml | 149 +-------------------------
 1 file changed, 1 insertion(+), 148 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 01a2e15149..f376fd85cb 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -117,130 +117,6 @@ jobs:
         uses: mxschmitt/action-tmate@v3
         if: ${{ failure() }}
 
-#  published:
-#    runs-on: ubuntu-20.04
-#    needs: build
-#    strategy:
-#      matrix:
-#        sets: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19"]
-#    steps:
-#      - name: Create Docker Image Dir
-#        run: mkdir /tmp/docker
-#
-#      - name: Download Docker Image from `build` Job
-#        uses: actions/download-artifact@v3
-#        with:
-#          name: docker-image
-#          path: /tmp/docker
-#
-#      - name: Load Downloaded Docker Image
-#        run: docker load --input /tmp/docker/$(ls /tmp/docker)
-#
-#        # Get files from BMDB
-#      - name: Checkout Input Files
-#        uses: actions/checkout@v4
-#        with:
-#          repository: virtualcell/vcdb
-#
-#      - name: Prepare sub directory
-#        run: |
-#          SOURCE_DIR="$GITHUB_WORKSPACE/published/biomodel/omex/sbml"
-#          TARGET_DIR="$GITHUB_WORKSPACE/set_${{ matrix.sets }}"
-#          MOD_VALUE=${{ matrix.sets }}
-#          file_index=0
-#
-#          # make target dir
-#          mkdir $TARGET_DIR
-#          # loop through files in directory, increment indexing and separating files
-#          for file in $SOURCE_DIR/*; do
-#              file_index=$((file_index+1))
-#              if [ $((file_index%${{ strategy.job-total }})) -eq $((MOD_VALUE + 0)) ]; then
-#                  cp ${file} $TARGET_DIR/
-#              fi
-#          done
-#
-#        # Do the execution
-#      - name: Begin Execution
-#        #$(docker image ls | grep "<none>" | awk '{print $3;}')  to get image id
-#        run: |
-#          # Prepare files
-#          echo "[]" > ${{ github.workspace }}/total_exec_summary.json
-#          echo "[]" > ${{ github.workspace }}/full_tracer.json
-#          # The /* goes on the outside, otherwise bash just interprets a string!
-#          for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
-#            echo "\"$file\""
-#            if [ -f "$file" ]; then
-#              extensionless_name="${file%.*}"
-#              base_name="$(basename ${file})"
-#              base_extless_name="$(basename ${extensionless_name})"
-#              mkdir "${extensionless_name}_output"
-#              echo "Running \"${file}\" in output folder \"${base_extless_name}_output\""
-#              sim_input="/root/${base_name}"
-#              sim_output="/root/${base_extless_name}_output"
-#              vcell_result=$(docker run -v $GITHUB_WORKSPACE/set_${{ matrix.sets }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') execute-omex -d -i ${sim_input} -o ${sim_output} --timeout_ms=300000)
-#
-#              echo -e "\n\n\n\n\n" # give ourselves some line separation
-#
-#              # Grab Summary
-#              if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json | wc -l) -ne 0 ]
-#              then
-#                # Append to master summary in github workspace
-#                jq '. + [inputs]' ${{ github.workspace }}/total_exec_summary.json $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json) > temp.json
-#                mv temp.json ${{ github.workspace }}/total_exec_summary.json
-#              else
-#                echo "No summary found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
-#              fi
-#
-#              # Check for individual Failure (a.k.a. if we failed early)
-#              status=$(cat $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json) | jq -r '.status')
-#              if [[ "$status" == "FAILED" ]] || [ ${vcell_result} -ne 0 ]
-#              then
-#                # Copy error log to github workspace
-#                echo "${base_extless_name}" >> $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output/errorLog.txt
-#                touch ${{ github.workspace }}/errors.report
-#                cat $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output/errorLog.txt >> ${{ github.workspace }}/errors.report
-#              else
-#                echo "Run of ${base_extless_name} succeeded (set_${{ matrix.sets }})"
-#              fi
-#
-#              # Grab Trace
-#              # Check for individual Success
-#              if [ $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name tracer.json | wc -l) -ne 0 ]
-#              then
-#                # Append to master trace in github workspace
-#                jq '. + inputs' ${{ github.workspace }}/full_tracer.json $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name tracer.json) > temp_2.json
-#                mv temp_2.json ${{ github.workspace }}/full_tracer.json
-#              else
-#                echo "No trace found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
-#              fi
-#            fi
-#          done
-#
-#      - name: Check For Success
-#        run: |
-#          # If the error file exists, then we have failures to report and upload; else, we can just end here; the next job will process results
-#          if [ $(find ${{ github.workspace }} -name errors.report | wc -l) -eq 0 ]
-#          then
-#            exit 0
-#          fi
-#          echo "Models that failed:"
-#          cat ${{ github.workspace }}/errors.report
-#
-#      - name: Upload failures
-#        uses: actions/upload-artifact@v3
-#        with:
-#          name: set_${{ matrix.sets }}.report
-#          path: ${{ github.workspace }}/errors.report
-#
-#      # Report a Problem to Slack
-#      - name: Report Problem to Slack
-#        if: ${{ failure() }}
-#        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Oops! The *Published Step* of set ${{ matrix.sets }} in Nightly **Execution** Testing Action on GitHub Failed ( ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} ). Go see what happened!" https://slack.com/api/chat.postMessage
-#
-#      - name: Setup tmate session 3
-#        uses: mxschmitt/action-tmate@v3
-#        if: ${{ failure() }}
-
   bmdb:
     runs-on: ubuntu-20.04
     needs: build
@@ -404,26 +280,6 @@ jobs:
         uses: mxschmitt/action-tmate@v3
         if: ${{ failure() }}
 
-#  published_results:
-#    runs-on: ubuntu-20.04
-#    needs: published
-#
-#    steps:
-#      - name: Gather Artifacts
-#        uses: actions/download-artifact@v3
-#        with:
-#          path: ${{ github.workspace }}/published-results
-#
-#      - name: Combine .report artifacts into one file
-#        run: cat $(find $GITHUB_WORKSPACE/published-results -name 'errors.report') > $GITHUB_WORKSPACE/combined.txt
-#
-#      - name: Post results to slack part 1
-#        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Here're the $(cat ${GITHUB_WORKSPACE}/combined.txt | wc -l) published models that didn't pass:" https://slack.com/api/chat.postMessage
-#
-#      - name: Post results to slack part 2
-#        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(sort ${GITHUB_WORKSPACE}/combined.txt)" https://slack.com/api/chat.postMessage
-
-
   bmdb_results:
     runs-on: ubuntu-20.04
     needs: bmdb
@@ -453,7 +309,4 @@ jobs:
         run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(sort ${GITHUB_WORKSPACE}/combined.txt)" https://slack.com/api/chat.postMessage
 
       - name: Post results to slack part 3
-        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F file=$GITHUB_WORKSPACE/summary.json https://slack.com/api/files.upload
-
-      - name: Post results to slack part 4
-        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F file=$GITHUB_WORKSPACE/tracer.json https://slack.com/api/files.upload
\ No newline at end of file
+        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="To see detailed logs of the BMDB results, see - ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}. Go see what happened!" https://slack.com/api/chat.postMessage
\ No newline at end of file

From 3072b341064f5e3d5c79661018d9dda16786e627 Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Mon, 23 Dec 2024 13:09:32 -0500
Subject: [PATCH 17/45] Upload processed logs to GitHub

---
 .github/workflows/NightlyBMDB_CLI.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index f376fd85cb..e5a98c1191 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -302,6 +302,18 @@ jobs:
       - name: Combine .summary artifacts into one file
         run: jq -s 'add' $(find $GITHUB_WORKSPACE/bmdb-results -name 'full_tracer.json') > $GITHUB_WORKSPACE/tracer.json
 
+      - name: Upload complete, compiled summary
+        uses: actions/upload-artifact@v3
+        with:
+          name: complete_summary.json
+          path: ${{ github.workspace }}/summary.json
+
+      - name: Upload  complete, compiled tracer
+        uses: actions/upload-artifact@v3
+        with:
+          name: complete_tracer.json
+          path: ${{ github.workspace }}/tracer.json
+
       - name: Post results to slack part 1
         run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="Here're the $(cat ${GITHUB_WORKSPACE}/combined.txt | wc -l) BMDB models that didn't pass:" https://slack.com/api/chat.postMessage
 

From ae9c724867d5bc27718e724f4b813a0276fb992f Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Mon, 23 Dec 2024 13:25:54 -0500
Subject: [PATCH 18/45] Name Change

---
 .github/workflows/NightlyBMDB_CLI.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index e5a98c1191..4433f05e1c 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -1,4 +1,4 @@
-name: Nightly CLI Execution Tests (Published + BMDB)
+name: Nightly CLI Execution Tests (BMDB)
 on:
   schedule:
     - cron: "0 7 * * *"

From 443b7ec795fffabf521c159c9bbdcaf8e8966b69 Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Mon, 23 Dec 2024 18:40:54 -0500
Subject: [PATCH 19/45] sort failures by BMDB id

---
 .github/workflows/NightlyBMDB_CLI.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 4433f05e1c..8753cd28e2 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -297,8 +297,8 @@ jobs:
         run: cat $(find $GITHUB_WORKSPACE/bmdb-results -name 'errors.report') > $GITHUB_WORKSPACE/combined.txt
 
       - name: Combine .summary artifacts into one file
-        run: jq -s 'add' $(find $GITHUB_WORKSPACE/bmdb-results -name 'total_exec_summary.json') > $GITHUB_WORKSPACE/summary.json
-
+        run: jq -s 'add | sort_by(.file_path)' $(find "$GITHUB_WORKSPACE/bmdb-results" -name 'total_exec_summary.json') > $GITHUB_WORKSPACE/summary.json
+        
       - name: Combine .summary artifacts into one file
         run: jq -s 'add' $(find $GITHUB_WORKSPACE/bmdb-results -name 'full_tracer.json') > $GITHUB_WORKSPACE/tracer.json
 

From f3f934820d0546e9fb082322200f96f47d33a383 Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Mon, 23 Dec 2024 22:38:22 -0500
Subject: [PATCH 20/45] add markdown exec-report CLI output

---
 .../vcell/cli/testsupport/OmexTestReport.java |  47 +++++++
 .../cli/testsupport/TestReportCommand.java    |  41 ++++--
 .../testsupport/TestOmexTestingDatabase.java  | 120 ++----------------
 .../testsupport/fake_exec_summaries.ndjson    |  50 ++++++++
 .../cli/testsupport/fake_test_cases.ndjson    |  54 ++++++++
 5 files changed, 191 insertions(+), 121 deletions(-)
 create mode 100644 vcell-cli/src/test/resources/org/vcell/cli/testsupport/fake_exec_summaries.ndjson
 create mode 100644 vcell-cli/src/test/resources/org/vcell/cli/testsupport/fake_test_cases.ndjson

diff --git a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java b/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java
index 3add3226f4..e53670845b 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java
+++ b/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java
@@ -1,6 +1,7 @@
 package org.vcell.cli.testsupport;
 
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 
@@ -19,10 +20,13 @@ public OmexTestCaseChange(OmexTestCase original, OmexTestCase updated) {
             this.original = original;
             this.updated = updated;
         }
+
     }
 
     public static class OmexTestStatistics {
         public int testCaseCount;
+        public int testCaseFailCount;
+        public int testCasePassCount;
         public int testCaseChangeCount;
         public int unmatchedTestCaseCount;
         public int unmatchedExecutionsCount;
@@ -86,6 +90,8 @@ public OmexTestReport(List<OmexTestCase> testCases, List<OmexExecSummary> execSu
             }
         }
         statistics.testCaseCount = testCases.size();
+        statistics.testCasePassCount = (int) testCases.stream().filter(tc -> tc.known_status == OmexTestCase.Status.PASS).count();
+        statistics.testCaseFailCount = (int) testCases.stream().filter(tc -> tc.known_status == OmexTestCase.Status.FAIL).count();
         statistics.totalExecutions = execSummaries.size();
         statistics.failedExecutionsCount = (int) execSummaries.stream().filter(s -> s.status == OmexExecSummary.ActualStatus.FAILED).count();
         statistics.passedExecutionsCount = (int) execSummaries.stream().filter(s -> s.status == OmexExecSummary.ActualStatus.PASSED).count();
@@ -131,4 +137,45 @@ public String toYaml() {
         }
     }
 
+    public String toMarkdown() throws JsonProcessingException {
+        ObjectMapper objectMapper = new ObjectMapper();
+        StringBuilder sb = new StringBuilder();
+        sb.append("# Test Report\n");
+        int testCasesFailed = statistics.testCaseFailCount;
+        int testCasesPassed = statistics.testCasePassCount;
+        int testCasesUnknown = statistics.testCaseCount - testCasesFailed - testCasesPassed;
+        sb.append("## Historical Test Case Records: "+statistics.testCaseCount+" (KNOWN PASSES = "+statistics.testCasePassCount+", KNOWN FAILURES = "+statistics.testCaseFailCount+", STATUS NOT RECORDED = "+testCasesUnknown).append(")\n");
+        sb.append("## New Test Executions: "+statistics.totalExecutions+" (PASSES = "+statistics.passedExecutionsCount+", FAILURES = "+statistics.failedExecutionsCount).append(")\n");
+
+        if (statistics.unmatchedTestCaseCount > 0) {
+            sb.append("## Historical Test Case Records without matching Test Executions: ").append(statistics.unmatchedTestCaseCount).append("\n");
+            for (OmexTestCase testCase : unmatchedTestCases) {
+                sb.append(" - ").append(testCase.test_collection).append(" : ").append(testCase.file_path).append("\n");
+            }
+        } else {
+            sb.append("## All Historical Test Case Records have matching Test Executions\n");
+        }
+
+        if (statistics.unmatchedExecutionsCount > 0) {
+            sb.append("## New Test Executions without matching Historical Test Case Records: ").append(statistics.unmatchedExecutionsCount).append("\n");
+            for (OmexExecSummary execSummary : unmatchedExecSummaries) {
+                sb.append(" - ").append(execSummary.file_path).append("\n");
+            }
+        } else {
+            sb.append("## All New Test Executions have matching Historical Test Case Records\n");
+        }
+
+        if (!testCaseChanges.isEmpty()) {
+            sb.append("## New Test Executions which differ from Historical Test Case Records: ").append(statistics.testCaseChangeCount).append("\n");
+            for (OmexTestCaseChange testCaseChange : testCaseChanges) {
+                String diff = "(" + testCaseChange.original.known_status + ":" + testCaseChange.original.known_failure_type + ") -> (" + testCaseChange.updated.known_status + ":" + testCaseChange.updated.known_failure_type + ")";
+                sb.append(" - ").append(testCaseChange.original.file_path).append(": " + diff + " ===New==> ").append(objectMapper.writeValueAsString(testCaseChange.updated)).append("`\n");
+            }
+        } else {
+            sb.append("## No New Test Executions differ from Historical Test Case Records\n");
+        }
+
+        return sb.toString();
+    }
+
 }
diff --git a/vcell-cli/src/main/java/org/vcell/cli/testsupport/TestReportCommand.java b/vcell-cli/src/main/java/org/vcell/cli/testsupport/TestReportCommand.java
index c877d4d96e..8a51169e8d 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/testsupport/TestReportCommand.java
+++ b/vcell-cli/src/main/java/org/vcell/cli/testsupport/TestReportCommand.java
@@ -11,24 +11,28 @@
 import java.nio.file.Files;
 import java.util.List;
 import java.util.concurrent.Callable;
+import java.util.function.Predicate;
 
 @Command(name = "test-report", description = "create test reports for a test suite")
 public class TestReportCommand implements Callable<Integer> {
 
     private final static Logger logger = LogManager.getLogger(TestReportCommand.class);
 
-    @Option(names = { "-t", "--test-cases" }, description = "[optional] test cases file - defaults to embedded test cases")
+    @Option(names = { "-t", "--test-cases" }, required = false, description = "[optional] test cases file - defaults to embedded test cases")
     private File testCasesNdjsonFile;
 
     @Option(names = { "-e", "--exec-summaries" }, required = true, description = "test results file e.g. exec_summary.ndjson")
     private File execSummaryNdjsonFile;
 
-    @Option(names = { "-r", "--report" }, required = false, description = "filename for generated test report (e.g. ./report.md)")
-    private File reportFile;
+    // list of OmexTestingDatabase.TestCollection objects to include
+    @Option(names = { "-c", "--collections" }, required = true, description = "list of test collections to include")
+    private List<OmexTestingDatabase.TestCollection> collections;
 
-    // flag weather output should be json or yaml
-    @Option(names = { "-j", "--json" }, description = "output report in json format")
-    private boolean bJson = false;
+    @Option(names = { "--report-md" }, required = true, description = "filename for generated markdown test report (e.g. ./report.md)")
+    private File reportFile_md;
+
+    @Option(names = { "--report-json" }, required = false, description = "filename for generated json test report (e.g. ./report.json)")
+    private File reportFile_json;
 
     @Option(names = { "-d", "--debug" }, description = "enable debug logging")
     private boolean bDebug = false;
@@ -42,15 +46,28 @@ public Integer call() {
         config.updateLoggers();
 
         try {
-            List<OmexTestCase> testCaseList = OmexTestingDatabase.loadOmexTestCases();
-            // read fully text file into a string from file 'execSummaryNdjsonFile'
+            // read test cases and filter by collections
+            Predicate<OmexTestCase> omexTestCasePredicate = tc -> collections.contains(tc.test_collection);
+            final List<OmexTestCase> testCaseList;
+            if (testCasesNdjsonFile != null) {
+                String test_cases_contents = Files.readString(testCasesNdjsonFile.toPath());
+                testCaseList = OmexTestingDatabase.parseOmexTestCases(test_cases_contents).stream().filter(omexTestCasePredicate).toList();
+            } else {
+                // if file not specified, use embedded test cases
+                testCaseList = OmexTestingDatabase.loadOmexTestCases().stream().filter(omexTestCasePredicate).toList();
+            }
+
+            // read exec summaries
             String exec_summary_contents = Files.readString(execSummaryNdjsonFile.toPath());
             List<OmexExecSummary> execSummaries = OmexTestingDatabase.loadOmexExecSummaries(exec_summary_contents);
+
+            // generate report
             OmexTestReport report = OmexTestingDatabase.generateReport(testCaseList, execSummaries);
-            if (reportFile != null) {
-                Files.writeString(reportFile.toPath(), bJson ? report.toJson() : report.toYaml());
-            } else {
-                System.out.println(bJson ? report.toJson() : report.toYaml());
+            if (reportFile_json != null) {
+                Files.writeString(reportFile_json.toPath(), report.toJson());
+            }
+            if (reportFile_md != null) {
+                Files.writeString(reportFile_md.toPath(), report.toMarkdown());
             }
             return 0;
         } catch (Exception e) {
diff --git a/vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestingDatabase.java b/vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestingDatabase.java
index a23ea19408..8b8a70e6a9 100644
--- a/vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestingDatabase.java
+++ b/vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestingDatabase.java
@@ -1,12 +1,15 @@
 package org.vcell.cli.testsupport;
 
+import org.apache.commons.io.IOUtils;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
+import java.util.Objects;
 
 
 public class TestOmexTestingDatabase {
@@ -64,118 +67,14 @@ void queryOmexTestCase() throws IOException {
     // test generateReport
     @Test
     void generateReport() throws IOException {
+        // read the line delimited json file in resources path /org/vcell/cli/testsupport/fake_test_cases.ndjson into a String
+        String test_cases_ndjson = new String(getClass().getResourceAsStream("/org/vcell/cli/testsupport/fake_test_cases.ndjson").readAllBytes());
 
-        String test_cases_ndjson = """
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000039.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000060.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000080.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000100.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000120.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000140.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000160.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000180.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000200.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000220.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000240.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000260.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000280.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000300.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000320.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000340.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000360.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000380.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000400.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000420.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000421.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000440.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000460.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000480.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000500.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000520.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000540.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000560.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000580.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000600.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000620.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000640.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000660.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000680.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000700.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000720.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000740.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000760.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000780.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000800.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000820.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000840.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000860.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000880.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000900.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000920.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000940.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000960.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000980.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001000.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001020.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001040.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001060.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001080.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-                                
-                """;
-        String exec_summary_ndjson = """
-                {  "file_path": "/root/BIOMD0000000020.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000040.omex",  "status": "FAILED",  "failure_type": "SBML_IMPORT_FAILURE",  "failure_desc": "Unable to initialize bioModel for the given selection: Failed to translate SBML model into BioModel: org.vcell.sbml.vcell.SBMLImportException: Non-numeric stoichiometry ('f' for product 'Br' in reaction 'Reaction5') not handled in VCell at this time. org.vcell.sbml.vcell.SBMLImportException: Failed to translate SBML model into BioModel: org.vcell.sbml.vcell.SBMLImportException: Non-numeric stoichiometry ('f' for product 'Br' in reaction 'Reaction5') not handled in VCell at this time." }
-                {  "file_path": "/root/BIOMD0000000060.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000080.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000100.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000120.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000140.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000240.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000260.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000280.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000300.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000320.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000340.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000360.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000380.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000400.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000420.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000440.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000460.omex",  "status": "FAILED",  "failure_type": "SBML_IMPORT_FAILURE",  "failure_desc": "Unable to initialize bioModel for the given selection: couldn't find SBase with sid=null in SBMLSymbolMapping org.vcell.sbml.vcell.SBMLImportException: couldn't find SBase with sid=null in SBMLSymbolMapping" }
-                {  "file_path": "/root/BIOMD0000000480.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000500.omex",  "status": "FAILED",  "failure_type": "UNCATETORIZED_FAULT",  "failure_desc": "Unable to initialize bioModel for the given selection: Cannot invoke \\"cbit.vcell.solver.Simulation.getImportedTaskID()\\" because \\"simulation\\" is null java.lang.NullPointerException: Cannot invoke \\"cbit.vcell.solver.Simulation.getImportedTaskID()\\" because \\"simulation\\" is null" }
-                {  "file_path": "/root/BIOMD0000000520.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000540.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000560.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000580.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000600.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000620.omex",  "status": "FAILED",  "failure_type": "UNCATETORIZED_FAULT",  "failure_desc": "Failed execution: Model 'BIOMD0000000620_BIOMD0000000620_url.sedml_BIOMD0000000620_url' Task 'task1'. java.lang.RuntimeException: Could not execute code: CVODE solver failed : at time 5.5, discontinuity (t == Anakinra_dose_counter) evaluated to TRUE, solver assumed FALSE\\n\\n\\n\\n(/usr/local/app/vcell/installDir/localsolvers/linux64/SundialsSolverStandalone_x64 /tmp/VCell_CLI_193e0a5db7a967308657490923326/BIOMD0000000620/BIOMD0000000620_url.sedml/SimID_411786192_0_.cvodeInput /tmp/VCell_CLI_193e0a5db7a967308657490923326/BIOMD0000000620/BIOMD0000000620_url.sedml/SimID_411786192_0_.ida) " }
-                {  "file_path": "/root/BIOMD0000000640.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000660.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000680.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000700.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000720.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000740.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000760.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000780.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000800.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000820.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000840.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000860.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000880.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000900.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000920.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000940.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000960.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000000980.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000001000.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000001020.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000001040.omex",  "status": "FAILED",  "failure_type": "SBML_IMPORT_FAILURE",  "failure_desc": "Unable to initialize bioModel for the given selection: Failed to translate SBML model into BioModel: Error binding global parameter 'Summary_flux_to_RBC' to model: 'Vin' is either not found in your model or is not allowed to be used in the current context. Check that you have provided the correct and full name (e.g. Ca_Cytosol). org.vcell.sbml.vcell.SBMLImportException: Failed to translate SBML model into BioModel: Error binding global parameter 'Summary_flux_to_RBC' to model: 'Vin' is either not found in your model or is not allowed to be used in the current context. Check that you have provided the correct and full name (e.g. Ca_Cytosol)." }
-                {  "file_path": "/root/BIOMD0000001060.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                {  "file_path": "/root/BIOMD0000001080.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-                """;
+        String exec_summary_ndjson = new String(getClass().getResourceAsStream("/org/vcell/cli/testsupport/fake_exec_summaries.ndjson").readAllBytes());
 
         List<OmexTestCase> testCases = OmexTestingDatabase.parseOmexTestCases(test_cases_ndjson);
+        OmexTestingDatabase.TestCollection testCollection = OmexTestingDatabase.TestCollection.SYSBIO_BIOMD;
+        testCases = testCases.stream().filter(c -> c.test_collection == testCollection).toList();
         List<OmexExecSummary> omexExecSummaries = OmexTestingDatabase.loadOmexExecSummaries(exec_summary_ndjson);
         OmexTestReport report = OmexTestingDatabase.generateReport(testCases, omexExecSummaries);
         Assertions.assertEquals(50, report.getStatistics().totalExecutions);
@@ -191,6 +90,9 @@ void generateReport() throws IOException {
 
         String reportYaml = report.toYaml();
         System.out.println(reportYaml);
+
+        String reportMarkdown = report.toMarkdown();
+        System.out.println(reportMarkdown);
     }
 
 }
diff --git a/vcell-cli/src/test/resources/org/vcell/cli/testsupport/fake_exec_summaries.ndjson b/vcell-cli/src/test/resources/org/vcell/cli/testsupport/fake_exec_summaries.ndjson
new file mode 100644
index 0000000000..3662bf4cc7
--- /dev/null
+++ b/vcell-cli/src/test/resources/org/vcell/cli/testsupport/fake_exec_summaries.ndjson
@@ -0,0 +1,50 @@
+{  "file_path": "/root/BIOMD0000000020.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000040.omex",  "status": "FAILED",  "failure_type": "SBML_IMPORT_FAILURE",  "failure_desc": "Unable to initialize bioModel for the given selection: Failed to translate SBML model into BioModel: org.vcell.sbml.vcell.SBMLImportException: Non-numeric stoichiometry ('f' for product 'Br' in reaction 'Reaction5') not handled in VCell at this time. org.vcell.sbml.vcell.SBMLImportException: Failed to translate SBML model into BioModel: org.vcell.sbml.vcell.SBMLImportException: Non-numeric stoichiometry ('f' for product 'Br' in reaction 'Reaction5') not handled in VCell at this time." }
+{  "file_path": "/root/BIOMD0000000060.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000080.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000100.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000120.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000140.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000240.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000260.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000280.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000300.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000320.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000340.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000360.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000380.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000400.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000420.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000440.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000460.omex",  "status": "FAILED",  "failure_type": "SBML_IMPORT_FAILURE",  "failure_desc": "Unable to initialize bioModel for the given selection: couldn't find SBase with sid=null in SBMLSymbolMapping org.vcell.sbml.vcell.SBMLImportException: couldn't find SBase with sid=null in SBMLSymbolMapping" }
+{  "file_path": "/root/BIOMD0000000480.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000500.omex",  "status": "FAILED",  "failure_type": "UNCATETORIZED_FAULT",  "failure_desc": "Unable to initialize bioModel for the given selection: Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null java.lang.NullPointerException: Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null" }
+{  "file_path": "/root/BIOMD0000000520.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000540.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000560.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000580.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000600.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000620.omex",  "status": "FAILED",  "failure_type": "UNCATETORIZED_FAULT",  "failure_desc": "Failed execution: Model 'BIOMD0000000620_BIOMD0000000620_url.sedml_BIOMD0000000620_url' Task 'task1'. java.lang.RuntimeException: Could not execute code: CVODE solver failed : at time 5.5, discontinuity (t == Anakinra_dose_counter) evaluated to TRUE, solver assumed FALSE\n\n\n\n(/usr/local/app/vcell/installDir/localsolvers/linux64/SundialsSolverStandalone_x64 /tmp/VCell_CLI_193e0a5db7a967308657490923326/BIOMD0000000620/BIOMD0000000620_url.sedml/SimID_411786192_0_.cvodeInput /tmp/VCell_CLI_193e0a5db7a967308657490923326/BIOMD0000000620/BIOMD0000000620_url.sedml/SimID_411786192_0_.ida) " }
+{  "file_path": "/root/BIOMD0000000640.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000660.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000680.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000700.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000720.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000740.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000760.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000780.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000800.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000820.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000840.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000860.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000880.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000900.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000920.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000940.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000960.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000980.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000001000.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000001020.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000001040.omex",  "status": "FAILED",  "failure_type": "SBML_IMPORT_FAILURE",  "failure_desc": "Unable to initialize bioModel for the given selection: Failed to translate SBML model into BioModel: Error binding global parameter 'Summary_flux_to_RBC' to model: 'Vin' is either not found in your model or is not allowed to be used in the current context. Check that you have provided the correct and full name (e.g. Ca_Cytosol). org.vcell.sbml.vcell.SBMLImportException: Failed to translate SBML model into BioModel: Error binding global parameter 'Summary_flux_to_RBC' to model: 'Vin' is either not found in your model or is not allowed to be used in the current context. Check that you have provided the correct and full name (e.g. Ca_Cytosol)." }
+{  "file_path": "/root/BIOMD0000001060.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000001080.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
diff --git a/vcell-cli/src/test/resources/org/vcell/cli/testsupport/fake_test_cases.ndjson b/vcell-cli/src/test/resources/org/vcell/cli/testsupport/fake_test_cases.ndjson
new file mode 100644
index 0000000000..f531c08715
--- /dev/null
+++ b/vcell-cli/src/test/resources/org/vcell/cli/testsupport/fake_test_cases.ndjson
@@ -0,0 +1,54 @@
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000039.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000060.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000080.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000100.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000120.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000140.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000160.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000180.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000200.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000220.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000240.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000260.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000280.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000300.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000320.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000340.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000360.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000380.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000400.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000420.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000421.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000440.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000460.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000480.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000500.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000520.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000540.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000560.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000580.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000600.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000620.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000640.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000660.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000680.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000700.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000720.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000740.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000760.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000780.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000800.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000820.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000840.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000860.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000880.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000900.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000920.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000940.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000960.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000980.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001000.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001020.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001040.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001060.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001080.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}

From 5602be00e69485adbec4b1825f0b66286d46e9ee Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Mon, 23 Dec 2024 23:01:07 -0500
Subject: [PATCH 21/45] generate summary report.md and push to slack

---
 .github/workflows/NightlyBMDB_CLI.yml | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 4433f05e1c..72aa589495 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -205,6 +205,7 @@ jobs:
                 # Append to master summary in github workspace
                 jq '. + [inputs]' ${{ github.workspace }}/total_exec_summary.json $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json) > temp.json
                 mv temp.json ${{ github.workspace }}/total_exec_summary.json
+                echo $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json) >> ${{ github.workspace }}/exec_summary.ndjson
                 echo "Done!"
               else
                 echo "No summary found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
@@ -299,9 +300,28 @@ jobs:
       - name: Combine .summary artifacts into one file
         run: jq -s 'add' $(find $GITHUB_WORKSPACE/bmdb-results -name 'total_exec_summary.json') > $GITHUB_WORKSPACE/summary.json
 
+      - name: Combine all exec_summary.ndjson files (from each of the matrix jobs) into one exec_summary.ndjson file
+        run: echo $(find $GITHUB_WORKSPACE/bmdb-results -name 'total_exec_summary.ndjson') > $GITHUB_WORKSPACE/exec_summary.ndjson
+
+      - name: Upload combined exec_summary.ndjson file
+        uses: actions/upload-artifact@v3
+        with:
+          name: exec_summary.ndjson
+          path: ${{ github.workspace }}/exec_summary.ndjson
+
       - name: Combine .summary artifacts into one file
         run: jq -s 'add' $(find $GITHUB_WORKSPACE/bmdb-results -name 'full_tracer.json') > $GITHUB_WORKSPACE/tracer.json
 
+      # run exec-report CLI command via docker to generate report using the exec_summary.ndjson file and the embedded test_cases.ndjson file
+      - name: Run Docker test-report command
+        run: docker run -v ${{ github.workspace }}:/root $(docker image ls | grep "<none>" | awk '{print $3;}') test-report -c SYSBIO_BIOMD --exec-summaries /root/exec_summary.ndjson --report-md /root/report.md
+
+      - name: Upload generated report.md
+        uses: actions/upload-artifact@v3
+        with:
+          name: report.md
+          path: ${{ github.workspace }}/report.md
+
       - name: Upload complete, compiled summary
         uses: actions/upload-artifact@v3
         with:
@@ -320,5 +340,8 @@ jobs:
       - name: Post results to slack part 2
         run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(sort ${GITHUB_WORKSPACE}/combined.txt)" https://slack.com/api/chat.postMessage
 
-      - name: Post results to slack part 3
-        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="To see detailed logs of the BMDB results, see - ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}. Go see what happened!" https://slack.com/api/chat.postMessage
\ No newline at end of file
+      - name: Post results to slack part 3 (report.md from exec-report CLI command)
+        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="$(cat ${GITHUB_WORKSPACE}/report.md)" https://slack.com/api/chat.postMessage
+
+      - name: Post results to slack part 4
+        run: curl -X POST -F token="${{ secrets.SLACK_BOT_TOKEN }}" -F channel=${{ secrets.SLACK_CHANNEL_VCELL_DEV_TOKEN }} -F text="To see detailed logs of the BMDB results, see - ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}. Go see what happened!" https://slack.com/api/chat.postMessage

From cad82eb162ea1847cbf68f282c27c3ca12812614 Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Mon, 23 Dec 2024 23:01:23 -0500
Subject: [PATCH 22/45] WIP: test with fewer tests and workers

---
 .github/workflows/NightlyBMDB_CLI.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 72aa589495..10cd219acc 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -122,7 +122,8 @@ jobs:
     needs: build
     strategy:
       matrix:
-        sets: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19"]
+        # sets: [ "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19" ]
+        sets: [ "98", "99" ]
     steps:
       - name: Create Docker Image Dir
         run: mkdir /tmp/docker

From 92eca002c86b950ee85ce32a538bf60fec47f0f5 Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Mon, 23 Dec 2024 23:21:45 -0500
Subject: [PATCH 23/45] download docker image for test-report CLI command

---
 .github/workflows/NightlyBMDB_CLI.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index b591e0d917..841cd60340 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -287,6 +287,18 @@ jobs:
     needs: bmdb
 
     steps:
+      - name: Create Docker Image Dir
+        run: mkdir /tmp/docker
+
+      - name: Download Docker Image from `build` Job
+        uses: actions/download-artifact@v3
+        with:
+          name: docker-image
+          path: /tmp/docker
+
+      - name: Load Downloaded Docker Image
+        run: docker load --input /tmp/docker/$(ls /tmp/docker)
+
       - name: Gather Artifacts
         uses: actions/download-artifact@v3
         with:

From 35afa88629f8e8a8e6b8eae5944aab6fd87defbc Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Mon, 23 Dec 2024 23:38:25 -0500
Subject: [PATCH 24/45] restore full testing

---
 .github/workflows/NightlyBMDB_CLI.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 841cd60340..3bc6bd41c2 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -122,8 +122,7 @@ jobs:
     needs: build
     strategy:
       matrix:
-        # sets: [ "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19" ]
-        sets: [ "98", "99" ]
+        sets: [ "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19" ]
     steps:
       - name: Create Docker Image Dir
         run: mkdir /tmp/docker

From 9024108f1407da07e57f44189e3bf1a522af20a6 Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Tue, 24 Dec 2024 04:52:07 -0500
Subject: [PATCH 25/45] enable test-report CLI command in docker_run.sh for
 Dockerfile

---
 docker_run.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docker_run.sh b/docker_run.sh
index c86f2b53ab..81f01d4e05 100755
--- a/docker_run.sh
+++ b/docker_run.sh
@@ -22,6 +22,11 @@ case "$rawCommand" in
     command="export-omex"
     shift
     ;;
+  "test-report")
+    echo 'test-report mode requested'
+    command="test-report"
+    shift
+    ;;
   "export-omex-batch")
     echo 'export-omex-batch mode requested'
     command="export-omex-batch"

From 8ef8e4d163b8b18e341be390a673baabc9871567 Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Tue, 24 Dec 2024 05:37:48 -0500
Subject: [PATCH 26/45] fix bash processing of ndjson files

---
 .github/workflows/NightlyBMDB_CLI.yml | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 3bc6bd41c2..38e591f086 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -205,7 +205,8 @@ jobs:
                 # Append to master summary in github workspace
                 jq '. + [inputs]' ${{ github.workspace }}/total_exec_summary.json $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json) > temp.json
                 mv temp.json ${{ github.workspace }}/total_exec_summary.json
-                echo $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json) >> ${{ github.workspace }}/exec_summary.ndjson
+                # note that the exec_summary.json file is already only a single line of json text, so simple concatenation is fine
+                find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json -exec cat {} + | sort > ${{ github.workspace }}/total_exec_summary.ndjson
                 echo "Done!"
               else
                 echo "No summary found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"
@@ -260,12 +261,18 @@ jobs:
           name: set_${{ matrix.sets }}.report
           path: ${{ github.workspace }}/errors.report
 
-      - name: Upload summary
+      - name: Upload summary as json
         uses: actions/upload-artifact@v3
         with:
           name: set_${{ matrix.sets }}.summary
           path: ${{ github.workspace }}/total_exec_summary.json
 
+      - name: Upload summary as ndjson
+        uses: actions/upload-artifact@v3
+        with:
+          name: set_${{ matrix.sets }}.summary_ndjson
+          path: ${{ github.workspace }}/total_exec_summary.ndjson
+
       - name: Upload tracer
         uses: actions/upload-artifact@v3
         with:
@@ -312,8 +319,9 @@ jobs:
       - name: Combine .summary artifacts into one file
         run: jq -s 'add | sort_by(.file_path)' $(find "$GITHUB_WORKSPACE/bmdb-results" -name 'total_exec_summary.json') > $GITHUB_WORKSPACE/summary.json
         
-      - name: Combine all exec_summary.ndjson files (from each of the matrix jobs) into one exec_summary.ndjson file
-        run: echo $(find $GITHUB_WORKSPACE/bmdb-results -name 'total_exec_summary.ndjson') > $GITHUB_WORKSPACE/exec_summary.ndjson
+      - name: Combine all .summary_ndjson artifacts into one file
+        # note that the total_exec_summary.ndjson files are already line delimited json text, so simple concatenation is fine
+        run: find $GITHUB_WORKSPACE/bmdb-results -name total_exec_summary.ndjson -exec cat {} + | sort > $GITHUB_WORKSPACE/exec_summary.ndjson
 
       - name: Upload combined exec_summary.ndjson file
         uses: actions/upload-artifact@v3

From e7461020b8c37afa4ccb28aaf89dcbdcf6b4588d Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Tue, 24 Dec 2024 07:04:30 -0500
Subject: [PATCH 27/45] add elapsed_time_ms to OmexExecSummary

---
 .../org/vcell/cli/run/ExecuteOmexCommand.java |   6 +-
 .../cli/testsupport/OmexExecSummary.java      |   2 +
 .../cli/testsupport/OmexTestingDatabase.java  |   3 +-
 .../testsupport/fake_exec_summaries.ndjson    | 100 +++++++++---------
 4 files changed, 58 insertions(+), 53 deletions(-)

diff --git a/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteOmexCommand.java b/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteOmexCommand.java
index 2b946e89bb..100836c48f 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteOmexCommand.java
+++ b/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteOmexCommand.java
@@ -63,6 +63,7 @@ public class ExecuteOmexCommand implements Callable<Integer> {
     public Integer call() {
 
         CLIRecordable cliTracer = new CliTracer();
+        long startTime_ms = System.currentTimeMillis();
         try {
             if (bDebug && bQuiet) {
                 System.err.println("cannot specify both debug and quiet, try --help for usage");
@@ -106,18 +107,19 @@ public Integer call() {
             FileUtils.copyDirectoryContents(tmpDir, outputFilePath, true, null);
             final OmexExecSummary omexExecSummary;
             if (Tracer.hasErrors()){
-                omexExecSummary = OmexTestingDatabase.summarize(inputFilePath,(Exception)null,Tracer.getErrors());
+                omexExecSummary = OmexTestingDatabase.summarize(inputFilePath,(Exception)null,Tracer.getErrors(), System.currentTimeMillis() - startTime_ms);
             } else {
                 omexExecSummary = new OmexExecSummary();
                 omexExecSummary.file_path = String.valueOf(inputFilePath);
                 omexExecSummary.status = OmexExecSummary.ActualStatus.PASSED;
+                omexExecSummary.elapsed_time_ms = System.currentTimeMillis() - startTime_ms;
             }
             new ObjectMapper().writeValue(new File(outputFilePath, "exec_summary.json"), omexExecSummary);
             new ObjectMapper().writeValue(new File(outputFilePath, "tracer.json"), Tracer.getTraceEvents());
             return 0;
         } catch (Exception e) { ///TODO: Break apart into specific exceptions to maximize logging.
             LogManager.getLogger(this.getClass()).error(e.getMessage(), e);
-            OmexExecSummary omexExecSummary = OmexTestingDatabase.summarize(inputFilePath,e,Tracer.getErrors());
+            OmexExecSummary omexExecSummary = OmexTestingDatabase.summarize(inputFilePath,e,Tracer.getErrors(),System.currentTimeMillis() - startTime_ms);
             try {
                 new ObjectMapper().writeValue(new File(outputFilePath, "exec_summary.json"), omexExecSummary);
                 new ObjectMapper().writeValue(new File(outputFilePath, "tracer.json"), Tracer.getTraceEvents());
diff --git a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexExecSummary.java b/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexExecSummary.java
index d8a8053fc4..d16f4820de 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexExecSummary.java
+++ b/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexExecSummary.java
@@ -11,6 +11,7 @@ public enum ActualStatus {
     public ActualStatus status;
     public FailureType failure_type;
     public String failure_desc;
+    public long elapsed_time_ms;
 
     @Override
     public String toString() {
@@ -19,6 +20,7 @@ public String toString() {
                 ", status=" + status +
                 ", failure_type=" + failure_type +
                 ", failure_desc="+((failure_desc!=null)?('\'' + failure_desc + '\''):null) +
+                ", elapsed_time_ms=" + elapsed_time_ms +
                 '}';
     }
 }
diff --git a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestingDatabase.java b/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestingDatabase.java
index dcbfbf55ca..81ee2ec940 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestingDatabase.java
+++ b/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestingDatabase.java
@@ -84,10 +84,11 @@ public static List<OmexTestCase> parseOmexTestCases(String testCasesNdjson) thro
         return testCases;
     }
 
-    public static OmexExecSummary summarize(File inputFilePath, Exception exception, List<TraceEvent> errorEvents) {
+    public static OmexExecSummary summarize(File inputFilePath, Exception exception, List<TraceEvent> errorEvents, long elapsedTime_ms) {
         OmexExecSummary execSummary = new OmexExecSummary();
         execSummary.file_path = inputFilePath.toString();
         execSummary.status = OmexExecSummary.ActualStatus.FAILED;
+        execSummary.elapsed_time_ms = elapsedTime_ms;
         if (exception != null || !errorEvents.isEmpty()) {
             execSummary.failure_type = determineFault(exception, errorEvents);
             execSummary.failure_desc = null;
diff --git a/vcell-cli/src/test/resources/org/vcell/cli/testsupport/fake_exec_summaries.ndjson b/vcell-cli/src/test/resources/org/vcell/cli/testsupport/fake_exec_summaries.ndjson
index 3662bf4cc7..dc3789438c 100644
--- a/vcell-cli/src/test/resources/org/vcell/cli/testsupport/fake_exec_summaries.ndjson
+++ b/vcell-cli/src/test/resources/org/vcell/cli/testsupport/fake_exec_summaries.ndjson
@@ -1,50 +1,50 @@
-{  "file_path": "/root/BIOMD0000000020.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000040.omex",  "status": "FAILED",  "failure_type": "SBML_IMPORT_FAILURE",  "failure_desc": "Unable to initialize bioModel for the given selection: Failed to translate SBML model into BioModel: org.vcell.sbml.vcell.SBMLImportException: Non-numeric stoichiometry ('f' for product 'Br' in reaction 'Reaction5') not handled in VCell at this time. org.vcell.sbml.vcell.SBMLImportException: Failed to translate SBML model into BioModel: org.vcell.sbml.vcell.SBMLImportException: Non-numeric stoichiometry ('f' for product 'Br' in reaction 'Reaction5') not handled in VCell at this time." }
-{  "file_path": "/root/BIOMD0000000060.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000080.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000100.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000120.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000140.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000240.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000260.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000280.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000300.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000320.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000340.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000360.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000380.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000400.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000420.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000440.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000460.omex",  "status": "FAILED",  "failure_type": "SBML_IMPORT_FAILURE",  "failure_desc": "Unable to initialize bioModel for the given selection: couldn't find SBase with sid=null in SBMLSymbolMapping org.vcell.sbml.vcell.SBMLImportException: couldn't find SBase with sid=null in SBMLSymbolMapping" }
-{  "file_path": "/root/BIOMD0000000480.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000500.omex",  "status": "FAILED",  "failure_type": "UNCATETORIZED_FAULT",  "failure_desc": "Unable to initialize bioModel for the given selection: Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null java.lang.NullPointerException: Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null" }
-{  "file_path": "/root/BIOMD0000000520.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000540.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000560.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000580.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000600.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000620.omex",  "status": "FAILED",  "failure_type": "UNCATETORIZED_FAULT",  "failure_desc": "Failed execution: Model 'BIOMD0000000620_BIOMD0000000620_url.sedml_BIOMD0000000620_url' Task 'task1'. java.lang.RuntimeException: Could not execute code: CVODE solver failed : at time 5.5, discontinuity (t == Anakinra_dose_counter) evaluated to TRUE, solver assumed FALSE\n\n\n\n(/usr/local/app/vcell/installDir/localsolvers/linux64/SundialsSolverStandalone_x64 /tmp/VCell_CLI_193e0a5db7a967308657490923326/BIOMD0000000620/BIOMD0000000620_url.sedml/SimID_411786192_0_.cvodeInput /tmp/VCell_CLI_193e0a5db7a967308657490923326/BIOMD0000000620/BIOMD0000000620_url.sedml/SimID_411786192_0_.ida) " }
-{  "file_path": "/root/BIOMD0000000640.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000660.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000680.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000700.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000720.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000740.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000760.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000780.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000800.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000820.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000840.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000860.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000880.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000900.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000920.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000940.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000960.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000000980.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000001000.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000001020.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000001040.omex",  "status": "FAILED",  "failure_type": "SBML_IMPORT_FAILURE",  "failure_desc": "Unable to initialize bioModel for the given selection: Failed to translate SBML model into BioModel: Error binding global parameter 'Summary_flux_to_RBC' to model: 'Vin' is either not found in your model or is not allowed to be used in the current context. Check that you have provided the correct and full name (e.g. Ca_Cytosol). org.vcell.sbml.vcell.SBMLImportException: Failed to translate SBML model into BioModel: Error binding global parameter 'Summary_flux_to_RBC' to model: 'Vin' is either not found in your model or is not allowed to be used in the current context. Check that you have provided the correct and full name (e.g. Ca_Cytosol)." }
-{  "file_path": "/root/BIOMD0000001060.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
-{  "file_path": "/root/BIOMD0000001080.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null }
+{  "file_path": "/root/BIOMD0000000020.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000040.omex",  "status": "FAILED",  "failure_type": "SBML_IMPORT_FAILURE",  "failure_desc": "Unable to initialize bioModel for the given selection: Failed to translate SBML model into BioModel: org.vcell.sbml.vcell.SBMLImportException: Non-numeric stoichiometry ('f' for product 'Br' in reaction 'Reaction5') not handled in VCell at this time. org.vcell.sbml.vcell.SBMLImportException: Failed to translate SBML model into BioModel: org.vcell.sbml.vcell.SBMLImportException: Non-numeric stoichiometry ('f' for product 'Br' in reaction 'Reaction5') not handled in VCell at this time.", "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000060.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000080.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000100.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000120.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000140.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000240.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000260.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000280.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000300.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000320.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000340.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000360.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000380.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000400.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000420.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000440.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000460.omex",  "status": "FAILED",  "failure_type": "SBML_IMPORT_FAILURE",  "failure_desc": "Unable to initialize bioModel for the given selection: couldn't find SBase with sid=null in SBMLSymbolMapping org.vcell.sbml.vcell.SBMLImportException: couldn't find SBase with sid=null in SBMLSymbolMapping", "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000480.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000500.omex",  "status": "FAILED",  "failure_type": "UNCATETORIZED_FAULT",  "failure_desc": "Unable to initialize bioModel for the given selection: Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null java.lang.NullPointerException: Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null", "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000520.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000540.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000560.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000580.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000600.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000620.omex",  "status": "FAILED",  "failure_type": "UNCATETORIZED_FAULT",  "failure_desc": "Failed execution: Model 'BIOMD0000000620_BIOMD0000000620_url.sedml_BIOMD0000000620_url' Task 'task1'. java.lang.RuntimeException: Could not execute code: CVODE solver failed : at time 5.5, discontinuity (t == Anakinra_dose_counter) evaluated to TRUE, solver assumed FALSE\n\n\n\n(/usr/local/app/vcell/installDir/localsolvers/linux64/SundialsSolverStandalone_x64 /tmp/VCell_CLI_193e0a5db7a967308657490923326/BIOMD0000000620/BIOMD0000000620_url.sedml/SimID_411786192_0_.cvodeInput /tmp/VCell_CLI_193e0a5db7a967308657490923326/BIOMD0000000620/BIOMD0000000620_url.sedml/SimID_411786192_0_.ida) ", "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000640.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000660.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000680.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000700.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000720.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000740.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000760.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000780.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000800.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000820.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000840.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000860.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000880.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000900.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000920.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000940.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000960.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000000980.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000001000.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000001020.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000001040.omex",  "status": "FAILED",  "failure_type": "SBML_IMPORT_FAILURE",  "failure_desc": "Unable to initialize bioModel for the given selection: Failed to translate SBML model into BioModel: Error binding global parameter 'Summary_flux_to_RBC' to model: 'Vin' is either not found in your model or is not allowed to be used in the current context. Check that you have provided the correct and full name (e.g. Ca_Cytosol). org.vcell.sbml.vcell.SBMLImportException: Failed to translate SBML model into BioModel: Error binding global parameter 'Summary_flux_to_RBC' to model: 'Vin' is either not found in your model or is not allowed to be used in the current context. Check that you have provided the correct and full name (e.g. Ca_Cytosol).", "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000001060.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }
+{  "file_path": "/root/BIOMD0000001080.omex",  "status": "PASSED",  "failure_type": null,  "failure_desc": null, "elapsed_time_ms": 1234 }

From b548df0e3af2508d048944cf614f7cd59c85faca Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Tue, 24 Dec 2024 07:10:20 -0500
Subject: [PATCH 28/45] only include tests between 100 and 200, includes a
 failure

---
 .github/workflows/NightlyBMDB_CLI.yml | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 38e591f086..524ca606f7 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -168,6 +168,19 @@ jobs:
                   continue
               fi
               number=$(echo "$file_name" | sed -n 's/^BIOMD\([0-9]\+\)\.omex$/\1/p' | sed 's/^0*//')
+
+              # TODO: remove later - if number is greater than MAX_TEST_NUMBER or less than MIN_TEST_NUMBER then continue
+              MIN_TEST_NUMBER=100
+              MAX_TEST_NUMBER=200
+              if [ $number -lt $MIN_TEST_NUMBER ]; then
+                  echo "Skipping ${file_name}"
+                  continue
+              fi
+              if [ $number -gt $MAX_TEST_NUMBER ]; then
+                  echo "Skipping ${file_name}"
+                  continue
+              fi
+
               if [ $((number % ${{ strategy.job-total }})) -eq $((MOD_VALUE + 0)) ]; then
                   echo "Accepting ${file_name}"
                   cp ${file} $TARGET_DIR/

From eb3af7e845d817ab3ccf4b88bb60cc787ddd6cef Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Tue, 24 Dec 2024 07:46:52 -0500
Subject: [PATCH 29/45] fix ndjson accumulation, expand to test cases [000-200]

---
 .github/workflows/NightlyBMDB_CLI.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 524ca606f7..ae127909e6 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -170,7 +170,7 @@ jobs:
               number=$(echo "$file_name" | sed -n 's/^BIOMD\([0-9]\+\)\.omex$/\1/p' | sed 's/^0*//')
 
               # TODO: remove later - if number is greater than MAX_TEST_NUMBER or less than MIN_TEST_NUMBER then continue
-              MIN_TEST_NUMBER=100
+              MIN_TEST_NUMBER=0
               MAX_TEST_NUMBER=200
               if [ $number -lt $MIN_TEST_NUMBER ]; then
                   echo "Skipping ${file_name}"
@@ -197,6 +197,7 @@ jobs:
           # Prepare files
           set +e
           echo "[]" > ${{ github.workspace }}/total_exec_summary.json
+          touch ${{ github.workspace }}/total_exec_summary.ndjson
           echo "[]" > ${{ github.workspace }}/full_tracer.json
           # The /* goes on the outside, otherwise bash just interprets a string!
           for file in "$GITHUB_WORKSPACE/set_${{ matrix.sets }}"/*; do
@@ -219,7 +220,8 @@ jobs:
                 jq '. + [inputs]' ${{ github.workspace }}/total_exec_summary.json $(find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json) > temp.json
                 mv temp.json ${{ github.workspace }}/total_exec_summary.json
                 # note that the exec_summary.json file is already only a single line of json text, so simple concatenation is fine
-                find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json -exec cat {} + | sort > ${{ github.workspace }}/total_exec_summary.ndjson
+                find $GITHUB_WORKSPACE/set_${{ matrix.sets }}/${base_extless_name}_output -name exec_summary.json -exec cat {} + | sort > temp.ndjson
+                cat temp.ndjson >> ${{ github.workspace }}/total_exec_summary.ndjson
                 echo "Done!"
               else
                 echo "No summary found in set $GITHUB_WORKSPACE/set_${{ matrix.sets }}"

From e34f7a4d2f04b2286b99e9f23336e354f25decc8 Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Tue, 24 Dec 2024 09:04:43 -0500
Subject: [PATCH 30/45] record failures in 000-200, test 200-400, truncate long
 markdown

---
 .github/workflows/NightlyBMDB_CLI.yml         |  4 +--
 .../vcell/cli/testsupport/OmexTestReport.java | 14 +++++++-
 .../src/main/resources/test_cases.ndjson      | 36 +++++++++----------
 3 files changed, 33 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index ae127909e6..ad7f85a213 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -170,8 +170,8 @@ jobs:
               number=$(echo "$file_name" | sed -n 's/^BIOMD\([0-9]\+\)\.omex$/\1/p' | sed 's/^0*//')
 
               # TODO: remove later - if number is greater than MAX_TEST_NUMBER or less than MIN_TEST_NUMBER then continue
-              MIN_TEST_NUMBER=0
-              MAX_TEST_NUMBER=200
+              MIN_TEST_NUMBER=200
+              MAX_TEST_NUMBER=400
               if [ $number -lt $MIN_TEST_NUMBER ]; then
                   echo "Skipping ${file_name}"
                   continue
diff --git a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java b/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java
index e53670845b..ca539ce996 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java
+++ b/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java
@@ -149,8 +149,14 @@ public String toMarkdown() throws JsonProcessingException {
 
         if (statistics.unmatchedTestCaseCount > 0) {
             sb.append("## Historical Test Case Records without matching Test Executions: ").append(statistics.unmatchedTestCaseCount).append("\n");
+            int count = 0;
             for (OmexTestCase testCase : unmatchedTestCases) {
                 sb.append(" - ").append(testCase.test_collection).append(" : ").append(testCase.file_path).append("\n");
+                count++;
+                if (count > 10) {
+                    sb.append(" - ...").append("\n");
+                    break;
+                }
             }
         } else {
             sb.append("## All Historical Test Case Records have matching Test Executions\n");
@@ -158,8 +164,14 @@ public String toMarkdown() throws JsonProcessingException {
 
         if (statistics.unmatchedExecutionsCount > 0) {
             sb.append("## New Test Executions without matching Historical Test Case Records: ").append(statistics.unmatchedExecutionsCount).append("\n");
+            int count = 0;
             for (OmexExecSummary execSummary : unmatchedExecSummaries) {
                 sb.append(" - ").append(execSummary.file_path).append("\n");
+                count++;
+                if (count > 10) {
+                    sb.append(" - ...").append("\n");
+                    break;
+                }
             }
         } else {
             sb.append("## All New Test Executions have matching Historical Test Case Records\n");
@@ -169,7 +181,7 @@ public String toMarkdown() throws JsonProcessingException {
             sb.append("## New Test Executions which differ from Historical Test Case Records: ").append(statistics.testCaseChangeCount).append("\n");
             for (OmexTestCaseChange testCaseChange : testCaseChanges) {
                 String diff = "(" + testCaseChange.original.known_status + ":" + testCaseChange.original.known_failure_type + ") -> (" + testCaseChange.updated.known_status + ":" + testCaseChange.updated.known_failure_type + ")";
-                sb.append(" - ").append(testCaseChange.original.file_path).append(": " + diff + " ===New==> ").append(objectMapper.writeValueAsString(testCaseChange.updated)).append("`\n");
+                sb.append(" - ").append(testCaseChange.original.file_path).append(": " + diff + " ===New==> `").append(objectMapper.writeValueAsString(testCaseChange.updated)).append("`\n");
             }
         } else {
             sb.append("## No New Test Executions differ from Historical Test Case Records\n");
diff --git a/vcell-cli/src/main/resources/test_cases.ndjson b/vcell-cli/src/main/resources/test_cases.ndjson
index 23267b18ab..3f56c19df1 100644
--- a/vcell-cli/src/main/resources/test_cases.ndjson
+++ b/vcell-cli/src/main/resources/test_cases.ndjson
@@ -125,8 +125,8 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000021.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000022.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000023.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000024.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000025.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000024.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: unsupported SBML element 'delay' in expression"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000025.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: unsupported SBML element 'delay' in expression"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000026.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000027.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000028.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -135,13 +135,13 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000031.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000032.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000033.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000034.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000034.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: unsupported SBML element 'delay' in expression"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000035.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000036.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000037.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000038.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000039.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000040.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000039.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('0.25' for reactant 'CaER' in reaction 'v1') not handled in VCell at this time"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000040.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-numeric stoichiometry ('f' for product 'Br' in reaction 'Reaction5') not handled in VCell at this time."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000041.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000042.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000043.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -155,16 +155,16 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000051.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000052.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000053.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000054.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000054.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000054_BIOMD0000000054_url.sedml_BIOMD0000000054_url' Task 'task1'.  java.lang.RuntimeException: CVODE solver could not generate input file: infinite loop in eliminating function nesting "}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000055.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000056.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000057.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000058.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000059.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000059.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('0.01' for reactant 'Ca_cyt' in reaction 'Calcium_cyt_Jerp') not handled in VCell at this time."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000060.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000061.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000062.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000063.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000063.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('0.5' for reactant 'FDP' in reaction 'Vgol') not handled in VCell at this time."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000064.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000065.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000066.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -182,7 +182,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000078.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000079.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000080.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000081.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000081.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('9.967E-4' for reactant 'ATP_C' in reaction 'PIP5kinase') not handled in VCell at this time."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000082.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000083.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000084.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -213,7 +213,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000109.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000110.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000111.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000112.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000112.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Failed to translate SBML model into BioModel: Cannot invoke \"org.sbml.jsbml.SBase.getAnnotation()\" because \"sbase\" is null"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000113.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000114.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000115.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -238,7 +238,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000134.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000135.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000136.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000137.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000137.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000137_BIOMD0000000137_url.sedml_BIOMD0000000137_url' Task 'task1'.  java.lang.RuntimeException: Could not execute code: CVODE solver failed : at time 0.01, discontinuity ((x6 + x7 + x8) > 100) evaluated to TRUE, solver assumed FALSE"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000138.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000139.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000140.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -246,17 +246,17 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000142.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000143.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000144.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000145.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000145.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('0.001' for reactant 'Ca_ER' in reaction 'R9') not handled in VCell at this time."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000146.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000147.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000148.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000149.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000150.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000151.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000151.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('0.5' for product 'x13' in reaction 'R23') not handled in VCell at this time."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000152.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000153.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000154.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000155.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000154.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: unsupported SBML element 'delay' in expression"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000155.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: unsupported SBML element 'delay' in expression"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000156.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000157.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000158.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -276,7 +276,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000172.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000173.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000174.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000175.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000175.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"MATH_GENERATION_FAILURE","known_failure_desc":"failed to generate math: Unable to sort, unknown identifier I_Net_E44PPI3K_binding"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000176.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000177.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000178.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -297,10 +297,10 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000193.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000194.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000195.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000196.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000196.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: unsupported SBML element 'delay' in expression"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000197.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000198.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000199.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000199.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('0.5' for reactant 'NADPH' in reaction 'r4') not handled in VCell at this time"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000200.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000201.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000202.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}

From 5d46637e8a8d6e5d825e782baaadfe55d21e02e5 Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Tue, 24 Dec 2024 10:41:24 -0500
Subject: [PATCH 31/45] add summary of known failures to report

---
 .../vcell/cli/testsupport/OmexTestReport.java    | 16 ++++++++++++++++
 .../vcell/cli/testsupport/fake_test_cases.ndjson |  2 +-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java b/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java
index ca539ce996..ac6bd753ad 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java
+++ b/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java
@@ -8,7 +8,9 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 public class OmexTestReport {
 
@@ -39,12 +41,22 @@ public static class OmexTestStatistics {
     public final List<OmexExecSummary> unmatchedExecSummaries = new ArrayList<>();
     public final List<OmexTestCase> unmatchedTestCases = new ArrayList<>();
     public final OmexTestStatistics statistics = new OmexTestStatistics();
+    public final Map<FailureType,Integer> failureTypeCounts = new HashMap<>();
 
     public OmexTestReport(List<OmexTestCase> testCases, List<OmexExecSummary> execSummaries) {
         // find common path prefix of list of Path objects
         Path commonPrefix = PathUtils.findCommonPrefix(execSummaries.stream().map(tc -> Paths.get(tc.file_path)).toList());
 
         unmatchedTestCases.addAll(testCases);
+
+        for (OmexTestCase testCase : testCases) {
+            if (testCase.known_status != OmexTestCase.Status.FAIL) {
+                continue;
+            }
+            FailureType failureType = (testCase.known_failure_type != null) ? testCase.known_failure_type : FailureType.UNCATETORIZED_FAULT;
+            failureTypeCounts.put(failureType, failureTypeCounts.getOrDefault(failureType, 0) + 1);
+        }
+
         for (OmexExecSummary execSummary : execSummaries) {
             List<OmexTestCase> matchingTestCases = OmexTestingDatabase.queryOmexTestCase(testCases, Paths.get(execSummary.file_path), commonPrefix);
             if (matchingTestCases.isEmpty()) {
@@ -145,6 +157,10 @@ public String toMarkdown() throws JsonProcessingException {
         int testCasesPassed = statistics.testCasePassCount;
         int testCasesUnknown = statistics.testCaseCount - testCasesFailed - testCasesPassed;
         sb.append("## Historical Test Case Records: "+statistics.testCaseCount+" (KNOWN PASSES = "+statistics.testCasePassCount+", KNOWN FAILURES = "+statistics.testCaseFailCount+", STATUS NOT RECORDED = "+testCasesUnknown).append(")\n");
+        // print failure type counts
+        for (Map.Entry<FailureType,Integer> entry : failureTypeCounts.entrySet()) {
+            sb.append(" - ").append(entry.getKey()).append(": ").append(entry.getValue()).append("\n");
+        }
         sb.append("## New Test Executions: "+statistics.totalExecutions+" (PASSES = "+statistics.passedExecutionsCount+", FAILURES = "+statistics.failedExecutionsCount).append(")\n");
 
         if (statistics.unmatchedTestCaseCount > 0) {
diff --git a/vcell-cli/src/test/resources/org/vcell/cli/testsupport/fake_test_cases.ndjson b/vcell-cli/src/test/resources/org/vcell/cli/testsupport/fake_test_cases.ndjson
index f531c08715..a392c626b3 100644
--- a/vcell-cli/src/test/resources/org/vcell/cli/testsupport/fake_test_cases.ndjson
+++ b/vcell-cli/src/test/resources/org/vcell/cli/testsupport/fake_test_cases.ndjson
@@ -49,6 +49,6 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000980.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001000.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001020.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001040.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001040.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"DIVIDE_BY_ZERO","known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001060.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001080.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}

From e0da337b7df887ce61a2db915c916252eefcdd87 Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Tue, 24 Dec 2024 10:42:03 -0500
Subject: [PATCH 32/45] record known faults for BMDB tests 200-400 (including 2
 timeouts)

---
 .github/workflows/NightlyBMDB_CLI.yml         |  5 ++
 .../src/main/resources/test_cases.ndjson      | 58 +++++++++----------
 2 files changed, 34 insertions(+), 29 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index ad7f85a213..e6f5cf8a6f 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -180,6 +180,11 @@ jobs:
                   echo "Skipping ${file_name}"
                   continue
               fi
+              # if number is in the list of [125, 246, 678, 1008] then continue
+              if [ $number -eq 235 ] || [ $number -eq 255 ] || [ $number -eq 1008 ]; then
+                  echo "Skipping ${file_name} - too slow or otherwise misbehaved"
+                  continue
+              fi
 
               if [ $((number % ${{ strategy.job-total }})) -eq $((MOD_VALUE + 0)) ]; then
                   echo "Accepting ${file_name}"
diff --git a/vcell-cli/src/main/resources/test_cases.ndjson b/vcell-cli/src/main/resources/test_cases.ndjson
index 3f56c19df1..2ddccca06a 100644
--- a/vcell-cli/src/main/resources/test_cases.ndjson
+++ b/vcell-cli/src/main/resources/test_cases.ndjson
@@ -307,7 +307,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000203.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000204.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000205.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000206.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000206.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('0.1' for product 's6o' in reaction 'v10') not handled in VCell at this time."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000207.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000208.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000209.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -333,9 +333,9 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000229.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000230.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000231.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000232.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000232.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('0.5' for reactant 'O2' in reaction 'vresp') not handled in VCell at this time."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000233.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000234.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000234.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"1626.047 seconds - timeout - Failed execution: Model 'BIOMD0000000234_GemcitabineCorrected.sedml_model' Task 'task1'.  java.lang.RuntimeException: Could not execute code: Unexpected error: Process timed out in SundialsSolverStandalone_x64"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000235.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000236.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000237.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -345,19 +345,19 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000241.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000242.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000243.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000244.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000245.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000246.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000244.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('0.5' for reactant 'FBP' in reaction 'e_Emp') not handled in VCell at this time."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000245.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('0.978' for product 's_pyr' in reaction 'r1') not handled in VCell at this time."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000246.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('0.001' for product 'Ca_in' in reaction 'vo') not handled in VCell at this time."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000247.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000248.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000248.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Failed to translate SBML model into BioModel: Error binding global parameter 'ATPase_flux_mM' to model: 'ATPase' is either not found in your model or is not allowed to be used in the current context."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000249.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000250.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000251.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000252.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000253.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000254.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000255.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000256.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000255.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":"1221.598 seconds - didn't timeout but very close"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000256.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Failed to translate SBML model into BioModel: Error binding global parameter 'XIAP_ini' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000257.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000258.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000259.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -377,8 +377,8 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000273.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000274.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000275.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000276.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000277.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000276.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000276_BIOMD0000000276_url.sedml_BIOMD0000000276_url' Task 'task1'.  java.lang.RuntimeException: Could not execute code: \n[CVODE ERROR]  CVode\n  The right-hand side routine failed at the first call.\n\nCVODE solver failed : CV_FIRST_RHSFUNC_ERR: The right-hand side routine failed at the first call : FunctionRangeException : FunctionRangeException : Evaluated to infinity in \"(9.8436754999999998006 - ((0.00049463777420084603827 + (0.048942788387963273578 / (1 + ((((1.2549999999999998934 * (__D_B_0 == 1)) + ((1.2549999999999998934 - (0.1817000000000000004 * (1 - exp( - (0.044200000000000003175 * ( - 575 + t)))))) * !((__D_B_0 == 1)))) / (1.2549999999999998934 * (0.16863731981259902359 ^ (1 / (15 + (112.51999999999999602 / (1 + exp( - (1000000 * (1.2161999999999999478 - ((1.2549999999999998934 * (__D_B_0 == 1)) + ((1.2549999999999998934 - (0.1817000000000000004 * (1 - exp( - (0.044200000000000003175 * ( - 575 + t)))))) * !((__D_B_0 == 1)))))))))))))) ^ (15 + (112.51999999999999602 / (1 + exp( - (1000000 * (1.2161999999999999478 - ((1.2549999999999998934 * (__D_B_0 == 1)) + ((1.2549999999999998934 - (0.1817000000000000004 * (1 - exp( - (0.044200000000000003175 * ( - 575 + t)))))) * !((__D_B_0 == 1)))))))))))))) * x1) - (0.012500000000000000694 * x1))\"\n  where:\n\t__D_B_0 = 1.000000\n\tt = 0.000000\n\tx1 = 0.000000"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000277.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000277_BIOMD0000000277_url.sedml_BIOMD0000000277_url' Task 'task1'.  java.lang.RuntimeException: Could not execute code: \n[CVODE ERROR]  CVode\n  The right-hand side routine failed at the first call.\n\nCVODE solver failed : CV_FIRST_RHSFUNC_ERR: The right-hand side routine failed at the first call : FunctionRangeException : FunctionRangeException : Evaluated to infinity in \"(9.8436754999999998006 - ((0.00049463777420084603827 + (0.048942788387963273578 / (1 + ((((1.2199999999999999734 * (__D_B_0 == 1)) + ((1.2199999999999999734 + (0.26240000000000002212 * (1 - exp( - (0.05689999999999999919 * ( - 575 + t)))))) * !((__D_B_0 == 1)))) / (1.2549999999999998934 * (0.16863731981259902359 ^ (1 / (15 + (112.51999999999999602 / (1 + exp( - (1000000 * (1.2161999999999999478 - ((1.2199999999999999734 * (__D_B_0 == 1)) + ((1.2199999999999999734 + (0.26240000000000002212 * (1 - exp( - (0.05689999999999999919 * ( - 575 + t)))))) * !((__D_B_0 == 1)))))))))))))) ^ (15 + (112.51999999999999602 / (1 + exp( - (1000000 * (1.2161999999999999478 - ((1.2199999999999999734 * (__D_B_0 == 1)) + ((1.2199999999999999734 + (0.26240000000000002212 * (1 - exp( - (0.05689999999999999919 * ( - 575 + t)))))) * !((__D_B_0 == 1)))))))))))))) * x1) - (0.012500000000000000694 * x1))\"\n  where:\n\t__D_B_0 = 1.000000\n\tt = 0.000000\n\tx1 = 0.000000"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000278.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000279.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000280.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -402,11 +402,11 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000298.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000299.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000300.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000301.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000302.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000301.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000301_BIOMD0000000301_url.sedml_BIOMD0000000301_url' Task 'task1'.  java.lang.RuntimeException: Could not execute code: CVODE solver failed : at time 0.01, discontinuity (t >= 0.010000000000000000208) evaluated to TRUE, solver assumed FALSE"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000302.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"MATH_GENERATION_FAILURE","known_failure_desc":"MappingException occurred: failed to generate math: generated an invalid mathDescription: Initial condition for variable 'h_post' references variable 'V_post'. Initial conditions cannot reference variables"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000303.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000304.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000305.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000305.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'V' to model: 'Fw_1st_step' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000306.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000307.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000308.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -420,7 +420,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000316.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000317.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000318.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000319.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000319.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('0.02' for product 'gamma' in reaction 'r3') not handled in VCell at this time"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000320.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000321.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000322.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -439,13 +439,13 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000335.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000336.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000337.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000338.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000339.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000338.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error adding Feature to vcModel org.vcell.sbml.vcell.SBMLImportException: compartment 'compartment_1' has constant attribute set to False, not currently supported"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000339.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error adding Feature to vcModel org.vcell.sbml.vcell.SBMLImportException: compartment 'compartment_1' has constant attribute set to False, not currently supported"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000340.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000341.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000342.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000342.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error adding Feature to vcModel org.vcell.sbml.vcell.SBMLImportException: compartment 'Vmed' has constant attribute set to False, not currently supported"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000343.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000344.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000344.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000345.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000346.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000347.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -454,9 +454,9 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000350.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000351.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000352.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000353.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000354.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000355.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000353.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('12345.7' for reactant 'cpd_C00369Glc_CS' in reaction 'rn_R02112CS_G2') not handled in VCell at this time"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000354.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error adding Feature to vcModel org.vcell.sbml.vcell.SBMLImportException: compartment 'cytosol' has constant attribute set to False, not currently supported"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000355.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error adding Feature to vcModel org.vcell.sbml.vcell.SBMLImportException: compartment 'cytosol' has constant attribute set to False, not currently supported"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000356.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000357.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000358.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -484,16 +484,16 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000380.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000381.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000382.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000383.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000384.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000385.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000386.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000387.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000388.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000383.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('1.5' for product 'PGA' in reaction 'PGA_prod_Vo') not handled in VCell at this time."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000384.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('1.5' for product 'PGA' in reaction 'PGA_prod_Vo') not handled in VCell at this time."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000385.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('1.5' for product 'PGA' in reaction 'PGA_prod_Vo') not handled in VCell at this time."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000386.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('1.5' for product 'PGA' in reaction 'PGA_prod_Vo') not handled in VCell at this time."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000387.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('1.5' for product 'PGA' in reaction 'PGA_prod_Vo') not handled in VCell at this time."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000388.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('0.6' for product 'Ru5P' in reaction 'GAP2Ru5P') not handled in VCell at this time."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000389.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000390.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000391.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000392.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000392.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('0.5' for reactant 'ATP' in reaction 'RuBisCO_6_O2') not handled in VCell at this time."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000393.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000394.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000395.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}

From 0cc76373468afddc70fec72e00ebef56841115e8 Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Tue, 24 Dec 2024 10:43:24 -0500
Subject: [PATCH 33/45] process BMDB tests 400 to 600

---
 .github/workflows/NightlyBMDB_CLI.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index e6f5cf8a6f..3d6036f048 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -170,8 +170,8 @@ jobs:
               number=$(echo "$file_name" | sed -n 's/^BIOMD\([0-9]\+\)\.omex$/\1/p' | sed 's/^0*//')
 
               # TODO: remove later - if number is greater than MAX_TEST_NUMBER or less than MIN_TEST_NUMBER then continue
-              MIN_TEST_NUMBER=200
-              MAX_TEST_NUMBER=400
+              MIN_TEST_NUMBER=400
+              MAX_TEST_NUMBER=600
               if [ $number -lt $MIN_TEST_NUMBER ]; then
                   echo "Skipping ${file_name}"
                   continue

From 0cb55283601861bd75dbd530e75637224af5bfa6 Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Tue, 24 Dec 2024 11:10:41 -0500
Subject: [PATCH 34/45] disable BMDB 595 (> 12 minutes runtime)

---
 .github/workflows/NightlyBMDB_CLI.yml          | 3 +--
 vcell-cli/src/main/resources/test_cases.ndjson | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 3d6036f048..4fecd684b0 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -180,8 +180,7 @@ jobs:
                   echo "Skipping ${file_name}"
                   continue
               fi
-              # if number is in the list of [125, 246, 678, 1008] then continue
-              if [ $number -eq 235 ] || [ $number -eq 255 ] || [ $number -eq 1008 ]; then
+              if [ $number -eq 235 ] || [ $number -eq 255 ] || [ $number -eq 595 ] || [ $number -eq 1008 ]; then
                   echo "Skipping ${file_name} - too slow or otherwise misbehaved"
                   continue
               fi
diff --git a/vcell-cli/src/main/resources/test_cases.ndjson b/vcell-cli/src/main/resources/test_cases.ndjson
index 2ddccca06a..231d604b19 100644
--- a/vcell-cli/src/main/resources/test_cases.ndjson
+++ b/vcell-cli/src/main/resources/test_cases.ndjson
@@ -676,7 +676,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000572.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000573.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000574.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000575.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000575.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":"> 12 minutes - too long - skip it"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000576.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000577.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000578.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}

From e9fc76d19177122fa211c951765f2b99a0b02c85 Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Tue, 24 Dec 2024 11:33:34 -0500
Subject: [PATCH 35/45] test BMDB 600-800 (while recording faults for 400-600)

---
 .github/workflows/NightlyBMDB_CLI.yml | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 4fecd684b0..627cd3202c 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -161,17 +161,12 @@ jobs:
                 echo "Input is invalid. Expecting 'BIOMD##########.omex format, got '${file_name}'."
                 continue
               fi
-              # ...is it the infamous BioModel 1008?
-              if [[ $file_name == *1008* ]]; then
-                  echo -n 'skipping '
-                  echo $file_name
-                  continue
-              fi
+
               number=$(echo "$file_name" | sed -n 's/^BIOMD\([0-9]\+\)\.omex$/\1/p' | sed 's/^0*//')
 
               # TODO: remove later - if number is greater than MAX_TEST_NUMBER or less than MIN_TEST_NUMBER then continue
-              MIN_TEST_NUMBER=400
-              MAX_TEST_NUMBER=600
+              MIN_TEST_NUMBER=600
+              MAX_TEST_NUMBER=800
               if [ $number -lt $MIN_TEST_NUMBER ]; then
                   echo "Skipping ${file_name}"
                   continue

From b638bc5bfd2cb2c2ace0d3a92e9c7e06e2dcf36c Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Tue, 24 Dec 2024 12:14:46 -0500
Subject: [PATCH 36/45] recorded BMDB faults for 600-800, set tests for
 800-1080

---
 .github/workflows/NightlyBMDB_CLI.yml         |  4 +-
 .../src/main/resources/test_cases.ndjson      | 96 +++++++++----------
 2 files changed, 50 insertions(+), 50 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 627cd3202c..f9f13478f6 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -165,8 +165,8 @@ jobs:
               number=$(echo "$file_name" | sed -n 's/^BIOMD\([0-9]\+\)\.omex$/\1/p' | sed 's/^0*//')
 
               # TODO: remove later - if number is greater than MAX_TEST_NUMBER or less than MIN_TEST_NUMBER then continue
-              MIN_TEST_NUMBER=600
-              MAX_TEST_NUMBER=800
+              MIN_TEST_NUMBER=800
+              MAX_TEST_NUMBER=2000
               if [ $number -lt $MIN_TEST_NUMBER ]; then
                   echo "Skipping ${file_name}"
                   continue
diff --git a/vcell-cli/src/main/resources/test_cases.ndjson b/vcell-cli/src/main/resources/test_cases.ndjson
index 231d604b19..acbed10e27 100644
--- a/vcell-cli/src/main/resources/test_cases.ndjson
+++ b/vcell-cli/src/main/resources/test_cases.ndjson
@@ -516,7 +516,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000412.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000413.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000414.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000415.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000415.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('0.574' for product 'species_7' in reaction 'reaction_1') not handled in VCell at this time."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000416.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000417.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000418.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -525,12 +525,12 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000421.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000422.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000423.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000424.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000424.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error adding Feature to vcModel org.vcell.sbml.vcell.SBMLImportException: compartment 'Default' has constant attribute set to False, not currently supported."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000425.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000426.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000426.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('0.08' for reactant 'species_31' in reaction 'reaction_28') not handled in VCell at this time."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000427.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000428.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000429.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000429.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error adding Feature to vcModel org.vcell.sbml.vcell.SBMLImportException: compartment 'compartment_1' has constant attribute set to False, not currently supported."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000430.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000431.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000432.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -558,24 +558,24 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000454.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000455.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000456.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000457.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000457.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'parameter_1' to model: 'UNRESOLVED.Size' is either not found in your model or is not allowed to be used in the current context."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000458.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000459.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000460.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000461.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000459.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: couldn't find SBase with sid=null in SBMLSymbolMapping"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000460.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: couldn't find SBase with sid=null in SBMLSymbolMapping"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000461.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: couldn't find SBase with sid=null in SBMLSymbolMapping"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000462.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000463.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000463.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('8.5' for reactant 'species_14' in reaction 'reaction_11') not handled in VCell at this time."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000464.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000465.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000466.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000467.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000468.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000469.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000470.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000471.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000472.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000473.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000474.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000468.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"found more than one SBase match for sid=unitime, matched [org.vcell.sbml.vcell.SBMLSymbolMapping$SBaseWrapper@15c6027d, org.vcell.sbml.vcell.SBMLSymbolMapping$SBaseWrapper@11c88cca]"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000469.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('1.5' for reactant 's_1372' in reaction 'r_1230') not handled in VCell at this time."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000470.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('1.5' for reactant 's_1372' in reaction 'r_1230') not handled in VCell at this time."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000471.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('4.33333333333333' for reactant 's_0056' in reaction 'r_1052') not handled in VCell at this time."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000472.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('4.33333333333333' for reactant 's_0056' in reaction 'r_1052') not handled in VCell at this time."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000473.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('1.8' for reactant 's_0595' in reaction 'r_1014') not handled in VCell at this time."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000474.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"found more than one SBase match for sid=v, matched [org.vcell.sbml.vcell.SBMLSymbolMapping$SBaseWrapper@3ba97962, org.vcell.sbml.vcell.SBMLSymbolMapping$SBaseWrapper@5b5b53c6]"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000475.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000476.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000477.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -597,12 +597,12 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000493.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000494.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000495.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000496.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000497.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000498.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000499.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000500.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000501.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000496.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('1.1358' for reactant 's_0001' in reaction 'r_1812') not handled in VCell at this time."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000497.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('1.1358' for reactant 's_0001' in reaction 'r_1812') not handled in VCell at this time."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000498.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Unable to initialize bioModel for the given selection: Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000499.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Metabolite_9' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000500.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Unable to initialize bioModel for the given selection: Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000501.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Unable to initialize bioModel for the given selection: Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000502.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000503.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000504.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -632,54 +632,54 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000528.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000529.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000530.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000531.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000532.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000531.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000531_model.sedml_model' Task 'task1'.  java.lang.RuntimeException: Could not execute code: \n[CVDENSE ERROR]  CVDense\n  A memory request failed.\n\nCVODE solver failed : CV_CONV_FAILURE: convergence test failures occurred too many times during one internal step"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000532.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000532_Vazquez2014.sedml_model' Task 'task1'.  java.lang.RuntimeException: Could not execute code: \n[CVDENSE ERROR]  CVDense\n  A memory request failed.\n\nCVODE solver failed : CV_CONV_FAILURE: convergence test failures occurred too many times during one internal step"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000533.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000534.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000535.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000536.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000537.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000534.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Metabolite_6' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000535.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Metabolite_3' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000536.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Metabolite_80' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000537.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Metabolite_40' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000538.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000539.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000540.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000541.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000542.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000542.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Unable to create and add rate rule to VC model : 'r77.LumpedJ' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000543.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000544.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000545.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000546.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000547.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000547.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Compartment_3' to model: 'UNRESOLVED.Size' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000548.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000549.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000550.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000551.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000552.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000553.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000554.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000555.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000556.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000554.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000554_Cloutier2009(final).sedml_model' Task 'task1'.  java.lang.RuntimeException: Could not execute code: \n[CVODE ERROR]  CVode\n  The right-hand side routine failed at the first call.\n\nCVODE solver failed : CV_FIRST_RHSFUNC_ERR: The right-hand side routine failed at the first call : FunctionRangeException : FunctionRangeException : Evaluated to infinity in \"(0.01200000000000000025 + (0.005040000000000000209 / (1 + exp( - (4.5918599999999996086 * ( - 199 + t))))) - (0.005040000000000000209 / (1 + exp( - (4.5918599999999996086 * ( - 505 + t))))) - (0.01200000000000000025 * (((42.372881355932207725 * Vv) ^ 2) + (1483.050847457627242 * ((42.372881355932207725 * Vv) ^  - 0.5) * (0.01200000000000000025 + (0.005040000000000000209 / (1 + exp( - (4.5918599999999996086 * ( - 199 + t))))) - (0.005040000000000000209 / (1 + exp( - (4.5918599999999996086 * ( - 505 + t)))))))) / (1 + (17.796610169491525966 * ((42.372881355932207725 * Vv) ^  - 0.5)))))\"\n  where:\n\tt = 0.000000\n\tVv = 0.023700"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000555.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000555_Auer2010.sedml_model' Task 'task1'.  java.lang.RuntimeException: Could not execute code: \n[CVDENSE ERROR]  CVDense\n  A memory request failed.\n\nCVODE solver failed : CV_CONV_FAILURE: convergence test failures occurred too many times during one internal step"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000556.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Unable to create and add rate rule to VC model : 'r0.LumpedJ' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000557.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000558.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000559.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000560.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000561.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000562.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000561.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000561_Martins2013.sedml_model' Task 'task1'.  java.lang.RuntimeException: Could not execute code: \n[CVDENSE ERROR]  CVDense\n  A memory request failed.\n\nCVODE solver failed : CV_CONV_FAILURE: convergence test failures occurred too many times during one internal step"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000562.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SEDML_NO_SEDMLS_TO_EXECUTE","known_failure_desc":"writeErrorList(): BIOMD0000000562 java.lang.RuntimeException: There are no SED-MLs in the archive to execute"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000563.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000564.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000565.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000566.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000567.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000566.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000566_Morris2009.sedml_model' Task 'task1'.  java.lang.RuntimeException: Could not execute code: \n[CVDENSE ERROR]  CVDense\n  A memory request failed.\n\nCVODE solver failed : CV_CONV_FAILURE: convergence test failures occurred too many times during one internal step"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000567.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000567_Morris2008.sedml_model' Task 'task1'.  java.lang.RuntimeException: Could not execute code: \n[CVDENSE ERROR]  CVDense\n  A memory request failed.\n\nCVODE solver failed : CV_CONV_FAILURE: convergence test failures occurred too many times during one internal step"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000568.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000569.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000570.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000570.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error adding Feature to vcModel org.vcell.sbml.vcell.SBMLImportException: compartment 'compartment_4' has constant attribute set to False, not currently supported."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000571.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000572.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000573.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000574.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000575.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":"> 12 minutes - too long - skip it"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000574.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000575.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Unable to create and add rate rule to VC model : 'J21.LumpedJ' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000576.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000577.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000578.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000578.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Unable to create and add rate rule to VC model : 'v_r33.LumpedJ' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000579.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000580.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000581.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -690,17 +690,17 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000586.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000587.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000588.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000589.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000589.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Metabolite_17' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000590.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000591.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000592.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000593.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000591.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error adding Lambda functionParse Error while parsing expression '\"Boehm2014 - isoform-specific dimerization of pSTAT5A and pSTAT5B\"': Lexical error at line 1, column 1.  Encountered: \"\\\"\" (34), after : \"\""}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000592.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SEDML_NO_SEDMLS_TO_EXECUTE","known_failure_desc":"writeErrorList(): BIOMD0000000592 java.lang.RuntimeException: There are no SED-MLs in the archive to execute"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000593.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SEDML_NO_SEDMLS_TO_EXECUTE","known_failure_desc":"writeErrorList(): BIOMD0000000593 java.lang.RuntimeException: There are no SED-MLs in the archive to execute"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000594.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000595.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000596.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000595.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":"timeout > 12 minutes"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000596.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.ClassCastException: class org.sbml.jsbml.xml.XMLNode cannot be cast to class org.sbml.jsbml.Annotation (org.sbml.jsbml.xml.XMLNode and org.sbml.jsbml.Annotation are in unnamed module of loader 'app')"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000597.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000598.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000599.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000599.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Metabolite_1' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000600.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000601.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000602.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}

From d94e0d7573a869b92a5b5ab8306cc83d1507e807 Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Tue, 24 Dec 2024 12:53:22 -0500
Subject: [PATCH 37/45] updated BMDB known faults for 600-800, exclude 711

---
 .github/workflows/NightlyBMDB_CLI.yml         |  2 +-
 .../src/main/resources/test_cases.ndjson      | 52 +++++++++----------
 2 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index f9f13478f6..9514f724ec 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -175,7 +175,7 @@ jobs:
                   echo "Skipping ${file_name}"
                   continue
               fi
-              if [ $number -eq 235 ] || [ $number -eq 255 ] || [ $number -eq 595 ] || [ $number -eq 1008 ]; then
+              if [ $number -eq 235 ] || [ $number -eq 255 ] || [ $number -eq 595 ]  || [ $number -eq 711 ] || [ $number -eq 1008 ]; then
                   echo "Skipping ${file_name} - too slow or otherwise misbehaved"
                   continue
               fi
diff --git a/vcell-cli/src/main/resources/test_cases.ndjson b/vcell-cli/src/main/resources/test_cases.ndjson
index acbed10e27..401c39f518 100644
--- a/vcell-cli/src/main/resources/test_cases.ndjson
+++ b/vcell-cli/src/main/resources/test_cases.ndjson
@@ -709,44 +709,44 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000605.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000606.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000607.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000608.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000608.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: Non-integer stoichiometry ('0.5' for reactant 'x_18' in reaction 'R_18') not handled in VCell at this time."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000609.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000610.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000611.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000612.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000613.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000613.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Q0' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000614.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000615.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000616.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000617.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000618.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000618.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error processing model: model2 - couldn't find SBase with sid=null in SBMLSymbolMapping org.vcell.sbml.vcell.SBMLImportException: couldn't find SBase with sid=null in SBMLSymbolMapping"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000619.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000620.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000621.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000620.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000620_BIOMD0000000620_url.sedml_BIOMD0000000620_url' Task 'task1'.  java.lang.RuntimeException: Could not execute code: CVODE solver failed : at time 5.5, discontinuity (t == Anakinra_dose_counter) evaluated to TRUE, solver assumed FALSE"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000621.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000621_BIOMD0000000621_url.sedml_BIOMD0000000621_url' Task 'task1'.  java.lang.RuntimeException: Could not execute code: CVODE solver failed : at time 5.5, discontinuity (t == Anakinra_dose_counter) evaluated to TRUE, solver assumed FALSE"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000622.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000623.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000624.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000625.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000626.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000627.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000628.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000627.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error adding Feature to vcModel org.vcell.sbml.vcell.SBMLImportException: compartment 'venous_balloon' has constant attribute set to False, not currently supported."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000628.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Metabolite_8' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000629.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000630.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000631.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000632.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000632.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'k4b' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000633.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000634.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000635.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000636.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000637.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000638.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000637.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000638.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000639.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000640.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000641.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000642.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000643.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000644.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000645.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000643.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000644.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000645.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000646.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000647.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000648.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -773,7 +773,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000670.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000671.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000672.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000673.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000673.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000673_MODEL1006230054_edited.sedml_model' Task 'task1'.  java.lang.RuntimeException: Could not execute code: \n[CVDENSE ERROR]  CVDense\n  A memory request failed.\n\nCVODE solver failed : CV_CONV_FAILURE: convergence test failures occurred too many times during one internal step"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000674.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000675.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000676.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -795,7 +795,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000692.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000693.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000695.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000696.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000696.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Metabolite_16' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000697.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000698.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000699.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -804,13 +804,13 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000702.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000703.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000704.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000705.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000706.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000705.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Metabolite_21' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000706.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"found more than one SBase match for sid=v, matched [org.vcell.sbml.vcell.SBMLSymbolMapping$SBaseWrapper@4c5da4f9, org.vcell.sbml.vcell.SBMLSymbolMapping$SBaseWrapper@6987a133]"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000707.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000708.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000709.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000710.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000711.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000710.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Metabolite_0_0' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000711.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"> 5 minutes - Failed execution: Model 'BIOMD0000000711_Hancioglu2007 - Human Immune Response to Influenza A virus.sedml_model' Task 'task1'.  java.lang.RuntimeException: Could not execute code: Unexpected error: Process timed out"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000712.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000713.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000714.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -818,7 +818,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000716.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000717.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000718.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000719.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000719.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000719_tsai2014.sedml_model' Task 'task1'.  java.lang.RuntimeException: Could not execute code: \n[CVODE ERROR]  CVode\n  The right-hand side routine failed at the first call.\n\nCVODE solver failed : CV_FIRST_RHSFUNC_ERR: The right-hand side routine failed at the first call : DivideByZeroException : DivideByZeroException : divide by zero in \"( - (1.5 * (APC_C_active - APC_C_total) / (1 + ((0.5 / Plx1_active) ^ 4))) - (0.14999999999999999445 * APC_C_active))\"\n  where:\n\tAPC_C_active = 1.000000\n\tAPC_C_total = 1.000000\n\tPlx1_active = 0.000000"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000720.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000721.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000722.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -830,15 +830,15 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000728.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000729.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000730.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000731.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000732.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000731.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Treg_origin_fraction_CD4' to model: 'func_TRegs_Production_from_CD4' is either not found in your model or is not allowed to be used in the current context"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000732.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000732_Kirschner_1998.sedml_model' Task 'task1'.  java.lang.RuntimeException: Could not execute code: \n[CVODE WARNING]  CVode\n  Internal t = 737.903 and h = 3.5563e-14 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  Internal t = 737.903 and h = 3.5563e-14 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  Internal t = 737.903 and h = 3.5563e-14 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  Internal t = 737.903 and h = 3.5563e-14 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  Internal t = 737.903 and h = 2.21887e-14 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  Internal t = 737.903 and h = 2.21887e-14 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  Internal t = 737.903 and h = 2.21887e-14 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  Internal t = 737.903 and h = 2.21887e-14 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  Internal t = 737.903 and h = 1.36968e-14 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  Internal t = 737.903 and h = 1.36968e-14 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  The above warning has been issued mxhnil times and will not be issued again for this problem.\n\n\n[CVODE ERROR]  CVode\n  At t = 737.903, mxstep steps taken before reaching tout.\n\nCVODE solver failed : CV_TOO_MUCH_WORK: took mxstep internal steps but could not reach tout.\n\nTry reducing maximum time step."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000733.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000734.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000735.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000736.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000737.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000738.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000739.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000739.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Metabolite_19' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000740.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000741.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000742.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -863,7 +863,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000761.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000762.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000763.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000764.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000764.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Metabolite_3' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000765.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000766.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000767.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -885,7 +885,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000783.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000784.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000785.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000786.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000786.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"class org.sbml.jsbml.xml.XMLNode cannot be cast to class org.sbml.jsbml.Annotation (org.sbml.jsbml.xml.XMLNode and org.sbml.jsbml.Annotation are in unnamed module of loader 'app')"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000787.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000788.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000789.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -893,7 +893,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000791.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000792.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000793.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000794.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000794.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"class org.sbml.jsbml.xml.XMLNode cannot be cast to class org.sbml.jsbml.Annotation (org.sbml.jsbml.xml.XMLNode and org.sbml.jsbml.Annotation are in unnamed module of loader 'app')"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000795.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000796.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000797.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}

From 66d39183bc11106faed913fc4ac0e60bcf779a82 Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Wed, 25 Dec 2024 00:38:19 -0500
Subject: [PATCH 38/45] add known failures for BMDB 800-1080, enable tests
 0000-1080

---
 .github/workflows/NightlyBMDB_CLI.yml         |   8 +-
 .../src/main/resources/test_cases.ndjson      | 122 +++++++++---------
 2 files changed, 66 insertions(+), 64 deletions(-)

diff --git a/.github/workflows/NightlyBMDB_CLI.yml b/.github/workflows/NightlyBMDB_CLI.yml
index 9514f724ec..4487a13d4b 100644
--- a/.github/workflows/NightlyBMDB_CLI.yml
+++ b/.github/workflows/NightlyBMDB_CLI.yml
@@ -165,7 +165,7 @@ jobs:
               number=$(echo "$file_name" | sed -n 's/^BIOMD\([0-9]\+\)\.omex$/\1/p' | sed 's/^0*//')
 
               # TODO: remove later - if number is greater than MAX_TEST_NUMBER or less than MIN_TEST_NUMBER then continue
-              MIN_TEST_NUMBER=800
+              MIN_TEST_NUMBER=0
               MAX_TEST_NUMBER=2000
               if [ $number -lt $MIN_TEST_NUMBER ]; then
                   echo "Skipping ${file_name}"
@@ -175,10 +175,12 @@ jobs:
                   echo "Skipping ${file_name}"
                   continue
               fi
-              if [ $number -eq 235 ] || [ $number -eq 255 ] || [ $number -eq 595 ]  || [ $number -eq 711 ] || [ $number -eq 1008 ]; then
+              case $number in
+                235|255|595|711|1008|1013|1022|1025|1030|1035|1042)
                   echo "Skipping ${file_name} - too slow or otherwise misbehaved"
                   continue
-              fi
+                  ;;
+              esac
 
               if [ $((number % ${{ strategy.job-total }})) -eq $((MOD_VALUE + 0)) ]; then
                   echo "Accepting ${file_name}"
diff --git a/vcell-cli/src/main/resources/test_cases.ndjson b/vcell-cli/src/main/resources/test_cases.ndjson
index 401c39f518..cd676755e8 100644
--- a/vcell-cli/src/main/resources/test_cases.ndjson
+++ b/vcell-cli/src/main/resources/test_cases.ndjson
@@ -902,8 +902,8 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000800.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000801.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000802.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000803.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000804.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000803.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000804.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Metabolite_1' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000805.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000806.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000807.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -913,7 +913,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000811.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000812.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000813.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000814.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000814.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000814_Perez-Garcia19 Computational design of improved standardized chemotherapy protocols for grade 2 oligodendrogliomas.sedml_model' Task 'task1'.  java.lang.RuntimeException: Could not execute code: CVODE solver failed : at time 28.616, discontinuity ((((t - (28 * ceil((0.035714285714285712303 * t)))) * (((t < 0.0) && 1) || 0.0)) + ((t - (28 * floor((0.035714285714285712303 * t)))) * !((((t < 0.0) && 1) || 0.0)))) == 27) evaluated to FALSE, solver assumed TRUE"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000815.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000816.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000817.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -926,12 +926,12 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000824.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000825.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000826.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000827.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000827.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000827_Ito2019 - gefitnib resistance of lung adenocarcinoma caused by MET amplification.sedml_model' Task 'task1'.  java.lang.RuntimeException: Could not execute code: \n[CVODE WARNING]  CVode\n  Internal t = 78.2588 and h = 6.54889e-15 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  Internal t = 78.2588 and h = 6.54889e-15 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  Internal t = 78.2588 and h = 6.54889e-15 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  Internal t = 78.2588 and h = 6.54889e-15 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  Internal t = 78.2588 and h = 6.54889e-15 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  Internal t = 78.2588 and h = 6.54889e-15 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  Internal t = 78.2588 and h = 6.54889e-15 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  Internal t = 78.2588 and h = 6.54889e-15 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  Internal t = 78.2588 and h = 6.54889e-15 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  Internal t = 78.2588 and h = 4.51381e-15 are such that t + h = t on the next step. The solver will continue anyway.\n\n\n[CVODE WARNING]  CVode\n  The above warning has been issued mxhnil times and will not be issued again for this problem.\n\n\n[CVODE ERROR]  CVode\n  At t = 78.2588, mxstep steps taken before reaching tout.\n\nCVODE solver failed : CV_TOO_MUCH_WORK: took mxstep internal steps but could not reach tout.\n\nTry reducing maximum time step."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000828.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000829.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000830.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000830.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.ClassCastException: class org.sbml.jsbml.xml.XMLNode cannot be cast to class org.sbml.jsbml.Annotation (org.sbml.jsbml.xml.XMLNode and org.sbml.jsbml.Annotation are in unnamed module of loader 'app')"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000831.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000832.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000832.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Metabolite_4' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000833.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000834.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000835.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -940,22 +940,22 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000838.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000839.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000840.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000841.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000841.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"org.vcell.sbml.vcell.SBMLImportException: unsupported SBML element 'delay' in expression"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000842.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000843.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000844.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000845.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000846.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000847.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000847.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.NullPointerException: Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000848.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000849.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000849.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.ClassCastException: class org.sbml.jsbml.xml.XMLNode cannot be cast to class org.sbml.jsbml.Annotation (org.sbml.jsbml.xml.XMLNode and org.sbml.jsbml.Annotation are in unnamed module of loader 'app')"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000850.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000851.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000852.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000853.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000854.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000855.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000856.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000856.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error adding Feature to vcModel org.vcell.sbml.vcell.SBMLImportException: compartment 'tV' has constant attribute set to False, not currently supported."}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000857.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000858.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000859.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -964,14 +964,14 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000862.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000863.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000864.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000865.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000866.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000867.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000868.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000869.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000870.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000871.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000872.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000865.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.NullPointerException: Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000866.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.NullPointerException: Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000867.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Metabolite_9' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000868.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.NullPointerException: Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000869.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.NullPointerException: Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000870.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.NullPointerException: Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000871.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.NullPointerException: Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000872.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.ClassCastException: class org.sbml.jsbml.xml.XMLNode cannot be cast to class org.sbml.jsbml.Annotation (org.sbml.jsbml.xml.XMLNode and org.sbml.jsbml.Annotation are in unnamed module of loader 'app')"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000873.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000874.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000875.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -980,7 +980,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000878.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000879.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000880.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000881.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000881.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.NullPointerException: Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000882.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000883.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000884.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -998,7 +998,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000896.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000897.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000898.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000899.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000899.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.ClassCastException: class org.sbml.jsbml.xml.XMLNode cannot be cast to class org.sbml.jsbml.Annotation (org.sbml.jsbml.xml.XMLNode and org.sbml.jsbml.Annotation are in unnamed module of loader 'app')"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000900.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000901.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000902.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -1007,7 +1007,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000905.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000906.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000907.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000908.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000908.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Metabolite_2' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000909.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000910.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000911.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -1018,15 +1018,15 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000916.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000917.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000918.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000919.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000919.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.NullPointerException: Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000920.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000921.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000922.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000923.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000924.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000925.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000925.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.ClassCastException: class org.sbml.jsbml.xml.XMLNode cannot be cast to class org.sbml.jsbml.Annotation (org.sbml.jsbml.xml.XMLNode and org.sbml.jsbml.Annotation are in unnamed module of loader 'app')"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000926.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000927.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000927.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.NullPointerException: Cannot invoke \"cbit.vcell.solver.Simulation.getImportedTaskID()\" because \"simulation\" is null"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000928.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000929.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000930.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -1051,24 +1051,24 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000949.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000950.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000951.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000952.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000952.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000000952_Rodenfels2019_V1.sedml_model' Task 'task1'.  java.lang.IllegalArgumentException: X must contain more than one value"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000953.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000954.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000955.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000956.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000956.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Unable to initialize bioModel for the given selection: expecting vcell var 'Ro_CA' mapped to SBML target 'Ro_CA' to be constant valued org.vcell.sedml.SEDMLImportException: expecting vcell var 'Ro_CA' mapped to SBML target 'Ro_CA' to be constant valued"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000957.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000958.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000959.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000960.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000961.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000961.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.ClassCastException: class org.sbml.jsbml.xml.XMLNode cannot be cast to class org.sbml.jsbml.Annotation (org.sbml.jsbml.xml.XMLNode and org.sbml.jsbml.Annotation are in unnamed module of loader 'app')"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000962.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000963.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000964.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000965.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000965.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.ClassCastException: class org.sbml.jsbml.xml.XMLNode cannot be cast to class org.sbml.jsbml.Annotation (org.sbml.jsbml.xml.XMLNode and org.sbml.jsbml.Annotation are in unnamed module of loader 'app')"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000966.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000967.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000968.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000969.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000969.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Metabolite_11' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000970.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000971.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000972.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -1087,7 +1087,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000985.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000986.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000987.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000988.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000988.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.ClassCastException: class org.sbml.jsbml.xml.XMLNode cannot be cast to class org.sbml.jsbml.Annotation (org.sbml.jsbml.xml.XMLNode and org.sbml.jsbml.Annotation are in unnamed module of loader 'app')"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000989.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000990.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000991.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -1110,7 +1110,7 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001010.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001011.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001012.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001013.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001013.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000001013_Leon-Triana2021 - eqs 1 and 2 - fig 3b.sedml_model15' Task 'task1'.  java.lang.RuntimeException: Could not execute code: Unexpected error: Process timed out\n\n(/usr/local/app/vcell/installDir/localsolvers/linux64/SundialsSolverStandalone_x64 \"/tmp/VCell_CLI_193f9b57e146875651314089006626/BIOMD0000001013/Leon-Triana2021 - eqs 1 and 2 - fig 3b.sedml/SimID_1601148481_0_.cvodeInput\" \"/tmp/VCell_CLI_193f9b57e146875651314089006626/BIOMD0000001013/Leon-Triana2021 - eqs 1 and 2 - fig 3b.sedml/SimID_1601148481_0_.ida\") "}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001014.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001015.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001016.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -1118,32 +1118,32 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001018.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001019.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001020.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001021.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001022.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001021.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Metabolite_0' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001022.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000001022_Creemers2021 - Fig 2(B) inset 2.sedml_model6' Task 'task1'.  java.lang.RuntimeException: Could not execute code: Unexpected error: Process timed out\n\n(/usr/local/app/vcell/installDir/localsolvers/linux64/SundialsSolverStandalone_x64 \"/tmp/VCell_CLI_193f9b4214e7972318304477051167/BIOMD0000001022/Creemers2021 - Fig 2(B) inset 2.sedml/SimID_1038023516_0_.cvodeInput\" \"/tmp/VCell_CLI_193f9b4214e7972318304477051167/BIOMD0000001022/Creemers2021 - Fig 2(B) inset 2.sedml/SimID_1038023516_0_.ida\") "}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001023.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001024.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001025.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001026.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001027.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001028.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001029.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001030.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001031.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001025.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000001025_Chaudhury2020 Eq 7-9 - Fig 4(B) simulation.sedml_model5' Task 'task1'.  java.lang.RuntimeException: Could not execute code: Unexpected error: Process timed out\n\n(/usr/local/app/vcell/installDir/localsolvers/linux64/SundialsSolverStandalone_x64 \"/tmp/VCell_CLI_193f9b43fd52760581088069625327/BIOMD0000001025/Chaudhury2020 Eq 7-9 - Fig 4(B) simulation.sedml/SimID_1974748314_0_.cvodeInput\" \"/tmp/VCell_CLI_193f9b43fd52760581088069625327/BIOMD0000001025/Chaudhury2020 Eq 7-9 - Fig 4(B) simulation.sedml/SimID_1974748314_0_.ida\") "}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001026.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Summary_flux_to_RBC' to model: 'Vin' is either not found in your model or is not allowed to be used in the current context"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001027.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error adding Feature to vcModel org.vcell.sbml.vcell.SBMLImportException: compartment 'Liver' has constant attribute set to False, not currently supported."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001028.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error adding Feature to vcModel org.vcell.sbml.vcell.SBMLImportException: compartment 'Liver' has constant attribute set to False, not currently supported."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001029.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error adding Feature to vcModel org.vcell.sbml.vcell.SBMLImportException: compartment 'Liver' has constant attribute set to False, not currently supported."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001030.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000001030_Sontag2017 - Fig 1D shifted exponential simulation.sedml_model4' Task 'task1'.  java.lang.RuntimeException: Could not execute code: Unexpected error: Process timed out\n\n(/usr/local/app/vcell/installDir/localsolvers/linux64/SundialsSolverStandalone_x64 \"/tmp/VCell_CLI_193f9b42ace2198722070284996283/BIOMD0000001030/Sontag2017 - Fig 1D shifted exponential simulation.sedml/SimID_1085451342_0_.cvodeInput\" \"/tmp/VCell_CLI_193f9b42ace2198722070284996283/BIOMD0000001030/Sontag2017 - Fig 1D shifted exponential simulation.sedml/SimID_1085451342_0_.ida\") "}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001031.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000001031_Al-Tuwairqi2020 - Fig2b simulation.sedml_model6' Task 'task1'.  java.lang.RuntimeException: Could not execute code: Unexpected error: Process timed out\n\n(/usr/local/app/vcell/installDir/localsolvers/linux64/SundialsSolverStandalone_x64 \"/tmp/VCell_CLI_193f9b4239d16177731767565387322/BIOMD0000001031/Al-Tuwairqi2020 - Fig2b simulation.sedml/SimID_1472540734_0_.cvodeInput\" \"/tmp/VCell_CLI_193f9b4239d16177731767565387322/BIOMD0000001031/Al-Tuwairqi2020 - Fig2b simulation.sedml/SimID_1472540734_0_.ida\") "}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001032.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001033.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001034.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001035.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001035.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000001035_Al-Tuwairqi2020 - Fig7(a) simulation.sedml_model11' Task 'task1'.  java.lang.RuntimeException: Could not execute code: Unexpected error: Process timed out\n\n(/usr/local/app/vcell/installDir/localsolvers/linux64/SundialsSolverStandalone_x64 \"/tmp/VCell_CLI_193f9b48bb36616768759896770720/BIOMD0000001035/Al-Tuwairqi2020 - Fig7(a) simulation.sedml/SimID_724140336_0_.cvodeInput\" \"/tmp/VCell_CLI_193f9b48bb36616768759896770720/BIOMD0000001035/Al-Tuwairqi2020 - Fig7(a) simulation.sedml/SimID_724140336_0_.ida\") "}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001036.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001037.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001038.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001039.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001040.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001039.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error adding Feature to vcModel org.vcell.sbml.vcell.SBMLImportException: compartment 'Liver' has constant attribute set to False, not currently supported."}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001040.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SBML_IMPORT_FAILURE","known_failure_desc":"Error binding global parameter 'Summary_flux_to_RBC' to model: 'Vin' is either not found in your model or is not allowed to be used in the current context"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001041.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001042.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001043.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001044.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001042.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000001042_Makhlouf2020 - Fig1(b).sedml_model1' Task 'task1'.  java.lang.RuntimeException: Could not execute code: Unexpected error: Process timed out\n\n(/usr/local/app/vcell/installDir/localsolvers/linux64/SundialsSolverStandalone_x64 \"/tmp/VCell_CLI_193fa3dcefc8848860965362711991/BIOMD0000001042/Makhlouf2020 - Fig1(b).sedml/SimID_1236394790_0_.cvodeInput\" \"/tmp/VCell_CLI_193fa3dcefc8848860965362711991/BIOMD0000001042/Makhlouf2020 - Fig1(b).sedml/SimID_1236394790_0_.ida\") "}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001043.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000001043_Wodarz2001 - Fig3A non-cytotoxic.sedml_model' Task 'task1'.  java.lang.RuntimeException: Could not execute code: Unexpected error: Process timed out\n\n(/usr/local/app/vcell/installDir/localsolvers/linux64/SundialsSolverStandalone_x64 \"/tmp/VCell_CLI_193f9b4ce9c2304006454824128300/BIOMD0000001043/Wodarz2001 - Fig3A non-cytotoxic.sedml/SimID_1036516481_0_.cvodeInput\" \"/tmp/VCell_CLI_193f9b4ce9c2304006454824128300/BIOMD0000001043/Wodarz2001 - Fig3A non-cytotoxic.sedml/SimID_1036516481_0_.ida\") "}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001044.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"java.lang.ClassCastException: class org.sbml.jsbml.xml.XMLNode cannot be cast to class org.sbml.jsbml.Annotation (org.sbml.jsbml.xml.XMLNode and org.sbml.jsbml.Annotation are in unnamed module of loader 'app')"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001045.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001046.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001046.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SEDML_NO_SEDMLS_TO_EXECUTE","known_failure_desc":"writeErrorList(): BIOMD0000001046 java.lang.RuntimeException: There are no SED-MLs in the archive to execute"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001047.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001048.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001052.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
@@ -1155,21 +1155,21 @@
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001058.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001059.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001060.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001061.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001062.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001063.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001064.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001065.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001066.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001067.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001068.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001069.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001070.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001071.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001061.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SEDML_NO_SEDMLS_TO_EXECUTE","known_failure_desc":"writeErrorList(): BIOMD0000001061 java.lang.RuntimeException: There are no SED-MLs in the archive to execute"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001062.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SEDML_NO_SEDMLS_TO_EXECUTE","known_failure_desc":"writeErrorList(): BIOMD0000001062 java.lang.RuntimeException: There are no SED-MLs in the archive to execute"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001063.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SEDML_NO_SEDMLS_TO_EXECUTE","known_failure_desc":"writeErrorList(): BIOMD0000001063 java.lang.RuntimeException: There are no SED-MLs in the archive to execute"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001064.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SEDML_NO_SEDMLS_TO_EXECUTE","known_failure_desc":"writeErrorList(): BIOMD0000001064 java.lang.RuntimeException: There are no SED-MLs in the archive to execute"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001065.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"UNCATETORIZED_FAULT","known_failure_desc":"Failed execution: Model 'BIOMD0000001065_vonDassow2000_1x4.timecourse1.sedml_model' Task 'task1'.  java.lang.RuntimeException: Could not execute code: \n[CVODE ERROR]  CVode\n  At t = 660.5repeated recoverable right-hand side function errors.\n\nCVODE solver failed : CV_REPTD_RHSFUNC_ERR: repeated recoverable right-hand side function errors : FunctionDomainException : FunctionDomainException : pow(u,v) and u=-0.000000<0 and v=2.619640 not an integer in \"((0.052821806954783216692 * ((0.00030494421841729539872 / (0.00030494421841729539872 + (EN_0_0 ^ 2.6196399999999999686))) ^ 9.6309500000000003439) / (5.7617237886031359827e-23 + (0.4000000000000000222 * ((0.00030494421841729539872 / (0.00030494421841729539872 + (EN_0_0 ^ 2.6196399999999999686))) ^ 9.6309500000000003439)))) - (0.13205451738695803132 * ci_0_0))\"\n  where:\n\tEN_0_0 = -0.000000\n\tci_0_0 = 1.000000\n\n\n\n(/usr/local/app/vcell/installDir/localsolvers/linux64/SundialsSolverStandalone_x64 /tmp/VCell_CLI_193f9f93f2012192791967497390908/BIOMD0000001065/vonDassow2000_1x4.timecourse1.sedml/SimID_1545840175_0_.cvodeInput /tmp/VCell_CLI_193f9f93f2012192791967497390908/BIOMD0000001065/vonDassow2000_1x4.timecourse1.sedml/SimID_1545840175_0_.ida) "}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001066.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SEDML_NO_SEDMLS_TO_EXECUTE","known_failure_desc":"writeErrorList(): BIOMD0000001066 java.lang.RuntimeException: There are no SED-MLs in the archive to execute"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001067.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SEDML_NO_SEDMLS_TO_EXECUTE","known_failure_desc":"writeErrorList(): BIOMD0000001067 java.lang.RuntimeException: There are no SED-MLs in the archive to execute"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001068.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SEDML_NO_SEDMLS_TO_EXECUTE","known_failure_desc":"writeErrorList(): BIOMD0000001068 java.lang.RuntimeException: There are no SED-MLs in the archive to execute"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001069.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SEDML_NO_SEDMLS_TO_EXECUTE","known_failure_desc":"writeErrorList(): BIOMD0000001069 java.lang.RuntimeException: There are no SED-MLs in the archive to execute"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001070.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SEDML_NO_SEDMLS_TO_EXECUTE","known_failure_desc":"writeErrorList(): BIOMD0000001070 java.lang.RuntimeException: There are no SED-MLs in the archive to execute"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001071.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SEDML_NO_SEDMLS_TO_EXECUTE","known_failure_desc":"writeErrorList(): BIOMD0000001071 java.lang.RuntimeException: There are no SED-MLs in the archive to execute"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001072.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001073.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001074.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
-{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001076.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001073.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SEDML_NO_SEDMLS_TO_EXECUTE","known_failure_desc":"writeErrorList(): BIOMD0000001073 java.lang.RuntimeException: There are no SED-MLs in the archive to execute"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001074.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SEDML_NO_SEDMLS_TO_EXECUTE","known_failure_desc":"writeErrorList(): BIOMD0000001074 java.lang.RuntimeException: There are no SED-MLs in the archive to execute"}
+{"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001076.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"SEDML_NO_SEDMLS_TO_EXECUTE","known_failure_desc":"writeErrorList(): BIOMD0000001076 java.lang.RuntimeException: There are no SED-MLs in the archive to execute"}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001077.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001078.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000001079.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}

From f303528d1177c69e8c6d59aea3cb8b84c4da6dda Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Thu, 26 Dec 2024 15:53:25 -0500
Subject: [PATCH 39/45] omex testsupport classes from CLI to CORE

---
 vcell-cli/src/main/java/org/vcell/cli/CLIStandalone.java    | 2 +-
 .../src/main/java/org/vcell/cli/run/ExecuteOmexCommand.java | 4 ++--
 .../cli/{testsupport => testing}/TestReportCommand.java     | 6 +++++-
 .../test/java/org/vcell/cli/run/BSTSBasedOmexExecTest.java  | 6 +++---
 .../java/org/vcell/cli/run/BSTSBasedTestSuiteFiles.java     | 4 ++--
 .../org/vcell/cli/testsupport/TestOmexTestingDatabase.java  | 4 +---
 .../main/java/org/vcell/sedml}/testsupport/FailureType.java | 2 +-
 .../java/org/vcell/sedml}/testsupport/OmexExecSummary.java  | 2 +-
 .../java/org/vcell/sedml}/testsupport/OmexTestCase.java     | 2 +-
 .../java/org/vcell/sedml}/testsupport/OmexTestReport.java   | 2 +-
 .../org/vcell/sedml}/testsupport/OmexTestingDatabase.java   | 2 +-
 .../main/java/org/vcell/sedml}/testsupport/PathUtils.java   | 2 +-
 .../src/test/java/org/vcell/sbml/SEDMLExporterCommon.java   | 1 -
 13 files changed, 20 insertions(+), 19 deletions(-)
 rename vcell-cli/src/main/java/org/vcell/cli/{testsupport => testing}/TestReportCommand.java (93%)
 rename {vcell-cli/src/main/java/org/vcell/cli => vcell-core/src/main/java/org/vcell/sedml}/testsupport/FailureType.java (97%)
 rename {vcell-cli/src/main/java/org/vcell/cli => vcell-core/src/main/java/org/vcell/sedml}/testsupport/OmexExecSummary.java (94%)
 rename {vcell-cli/src/main/java/org/vcell/cli => vcell-core/src/main/java/org/vcell/sedml}/testsupport/OmexTestCase.java (97%)
 rename {vcell-cli/src/main/java/org/vcell/cli => vcell-core/src/main/java/org/vcell/sedml}/testsupport/OmexTestReport.java (99%)
 rename {vcell-cli/src/main/java/org/vcell/cli => vcell-core/src/main/java/org/vcell/sedml}/testsupport/OmexTestingDatabase.java (99%)
 rename {vcell-cli/src/main/java/org/vcell/cli => vcell-core/src/main/java/org/vcell/sedml}/testsupport/PathUtils.java (96%)

diff --git a/vcell-cli/src/main/java/org/vcell/cli/CLIStandalone.java b/vcell-cli/src/main/java/org/vcell/cli/CLIStandalone.java
index e4b79ba1e8..fa923bd19f 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/CLIStandalone.java
+++ b/vcell-cli/src/main/java/org/vcell/cli/CLIStandalone.java
@@ -4,7 +4,7 @@
 import org.vcell.cli.run.ExecuteCommand;
 import org.vcell.cli.run.ExecuteOmexCommand;
 import org.vcell.cli.sbml.ModelCommand;
-import org.vcell.cli.testsupport.TestReportCommand;
+import org.vcell.cli.testing.TestReportCommand;
 import org.vcell.cli.vcml.*;
 
 import org.apache.logging.log4j.LogManager;
diff --git a/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteOmexCommand.java b/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteOmexCommand.java
index 100836c48f..3b9a17716c 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteOmexCommand.java
+++ b/vcell-cli/src/main/java/org/vcell/cli/run/ExecuteOmexCommand.java
@@ -8,8 +8,8 @@
 import org.vcell.cli.CLIPythonManager;
 import org.vcell.cli.CLIRecordable;
 import org.vcell.cli.CliTracer;
-import org.vcell.cli.testsupport.OmexExecSummary;
-import org.vcell.cli.testsupport.OmexTestingDatabase;
+import org.vcell.sedml.testsupport.OmexExecSummary;
+import org.vcell.sedml.testsupport.OmexTestingDatabase;
 import org.vcell.trace.Tracer;
 import org.vcell.util.FileUtils;
 import org.vcell.util.exe.Executable;
diff --git a/vcell-cli/src/main/java/org/vcell/cli/testsupport/TestReportCommand.java b/vcell-cli/src/main/java/org/vcell/cli/testing/TestReportCommand.java
similarity index 93%
rename from vcell-cli/src/main/java/org/vcell/cli/testsupport/TestReportCommand.java
rename to vcell-cli/src/main/java/org/vcell/cli/testing/TestReportCommand.java
index 8a51169e8d..157cf2e6fa 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/testsupport/TestReportCommand.java
+++ b/vcell-cli/src/main/java/org/vcell/cli/testing/TestReportCommand.java
@@ -1,9 +1,13 @@
-package org.vcell.cli.testsupport;
+package org.vcell.cli.testing;
 
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.core.LoggerContext;
+import org.vcell.sedml.testsupport.OmexExecSummary;
+import org.vcell.sedml.testsupport.OmexTestCase;
+import org.vcell.sedml.testsupport.OmexTestReport;
+import org.vcell.sedml.testsupport.OmexTestingDatabase;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Option;
 
diff --git a/vcell-cli/src/test/java/org/vcell/cli/run/BSTSBasedOmexExecTest.java b/vcell-cli/src/test/java/org/vcell/cli/run/BSTSBasedOmexExecTest.java
index 80e9132304..69cb4f6212 100644
--- a/vcell-cli/src/test/java/org/vcell/cli/run/BSTSBasedOmexExecTest.java
+++ b/vcell-cli/src/test/java/org/vcell/cli/run/BSTSBasedOmexExecTest.java
@@ -11,9 +11,9 @@
 import org.vcell.cli.CLIPythonManager;
 import org.vcell.cli.CLIRecordable;
 import org.vcell.cli.PythonStreamException;
-import org.vcell.cli.testsupport.FailureType;
-import org.vcell.cli.testsupport.OmexTestCase;
-import org.vcell.cli.testsupport.OmexTestingDatabase;
+import org.vcell.sedml.testsupport.FailureType;
+import org.vcell.sedml.testsupport.OmexTestCase;
+import org.vcell.sedml.testsupport.OmexTestingDatabase;
 import org.vcell.trace.TraceEvent;
 import org.vcell.trace.Tracer;
 import org.vcell.util.VCellUtilityHub;
diff --git a/vcell-cli/src/test/java/org/vcell/cli/run/BSTSBasedTestSuiteFiles.java b/vcell-cli/src/test/java/org/vcell/cli/run/BSTSBasedTestSuiteFiles.java
index 57cb164e61..c566913afe 100644
--- a/vcell-cli/src/test/java/org/vcell/cli/run/BSTSBasedTestSuiteFiles.java
+++ b/vcell-cli/src/test/java/org/vcell/cli/run/BSTSBasedTestSuiteFiles.java
@@ -3,8 +3,8 @@
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
-import org.vcell.cli.testsupport.OmexTestCase;
-import org.vcell.cli.testsupport.OmexTestingDatabase;
+import org.vcell.sedml.testsupport.OmexTestCase;
+import org.vcell.sedml.testsupport.OmexTestingDatabase;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
diff --git a/vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestingDatabase.java b/vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestingDatabase.java
index 8b8a70e6a9..71a40dd182 100644
--- a/vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestingDatabase.java
+++ b/vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestingDatabase.java
@@ -1,15 +1,13 @@
 package org.vcell.cli.testsupport;
 
-import org.apache.commons.io.IOUtils;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
+import org.vcell.sedml.testsupport.*;
 
 import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
-import java.util.Objects;
 
 
 public class TestOmexTestingDatabase {
diff --git a/vcell-cli/src/main/java/org/vcell/cli/testsupport/FailureType.java b/vcell-core/src/main/java/org/vcell/sedml/testsupport/FailureType.java
similarity index 97%
rename from vcell-cli/src/main/java/org/vcell/cli/testsupport/FailureType.java
rename to vcell-core/src/main/java/org/vcell/sedml/testsupport/FailureType.java
index 4872d5b54e..55f8a5c120 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/testsupport/FailureType.java
+++ b/vcell-core/src/main/java/org/vcell/sedml/testsupport/FailureType.java
@@ -1,4 +1,4 @@
-package org.vcell.cli.testsupport;
+package org.vcell.sedml.testsupport;
 
 @SuppressWarnings("unused")
 public enum FailureType {
diff --git a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexExecSummary.java b/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexExecSummary.java
similarity index 94%
rename from vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexExecSummary.java
rename to vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexExecSummary.java
index d16f4820de..b82c3ef37e 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexExecSummary.java
+++ b/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexExecSummary.java
@@ -1,4 +1,4 @@
-package org.vcell.cli.testsupport;
+package org.vcell.sedml.testsupport;
 
 public class OmexExecSummary {
 
diff --git a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestCase.java b/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestCase.java
similarity index 97%
rename from vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestCase.java
rename to vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestCase.java
index c312dacec6..52591dd799 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestCase.java
+++ b/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestCase.java
@@ -1,4 +1,4 @@
-package org.vcell.cli.testsupport;
+package org.vcell.sedml.testsupport;
 
 public class OmexTestCase {
 
diff --git a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java b/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestReport.java
similarity index 99%
rename from vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java
rename to vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestReport.java
index ac6bd753ad..61b30beb9c 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestReport.java
+++ b/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestReport.java
@@ -1,4 +1,4 @@
-package org.vcell.cli.testsupport;
+package org.vcell.sedml.testsupport;
 
 
 import com.fasterxml.jackson.core.JsonProcessingException;
diff --git a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestingDatabase.java b/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestingDatabase.java
similarity index 99%
rename from vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestingDatabase.java
rename to vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestingDatabase.java
index 81ee2ec940..76e70c8021 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestingDatabase.java
+++ b/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestingDatabase.java
@@ -1,4 +1,4 @@
-package org.vcell.cli.testsupport;
+package org.vcell.sedml.testsupport;
 
 import cbit.vcell.mapping.MappingException;
 import com.fasterxml.jackson.databind.MappingIterator;
diff --git a/vcell-cli/src/main/java/org/vcell/cli/testsupport/PathUtils.java b/vcell-core/src/main/java/org/vcell/sedml/testsupport/PathUtils.java
similarity index 96%
rename from vcell-cli/src/main/java/org/vcell/cli/testsupport/PathUtils.java
rename to vcell-core/src/main/java/org/vcell/sedml/testsupport/PathUtils.java
index add1be3fad..cd3107ebbe 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/testsupport/PathUtils.java
+++ b/vcell-core/src/main/java/org/vcell/sedml/testsupport/PathUtils.java
@@ -1,4 +1,4 @@
-package org.vcell.cli.testsupport;
+package org.vcell.sedml.testsupport;
 
 import java.nio.file.Path;
 import java.util.List;
diff --git a/vcell-core/src/test/java/org/vcell/sbml/SEDMLExporterCommon.java b/vcell-core/src/test/java/org/vcell/sbml/SEDMLExporterCommon.java
index 78ad789ece..2641decf6e 100644
--- a/vcell-core/src/test/java/org/vcell/sbml/SEDMLExporterCommon.java
+++ b/vcell-core/src/test/java/org/vcell/sbml/SEDMLExporterCommon.java
@@ -6,7 +6,6 @@
 import cbit.vcell.mapping.SimulationContext;
 import cbit.vcell.math.MathCompareResults;
 import cbit.vcell.math.MathDescription;
-import cbit.vcell.resource.NativeLib;
 import cbit.vcell.resource.PropertyLoader;
 import cbit.vcell.solver.MathOverrides;
 import cbit.vcell.solver.Simulation;

From 58409f36efefcc439c45fa742fc312b70b04fd10 Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Thu, 26 Dec 2024 17:31:48 -0500
Subject: [PATCH 40/45] use common FailureType in BiosimulationsExecTest and
 SpatialExecTest

---
 .../vcell/cli/run/BiosimulationsExecTest.java | 101 +++++++++---------
 .../org/vcell/cli/run/SpatialExecTest.java    |  99 ++++++++---------
 2 files changed, 96 insertions(+), 104 deletions(-)

diff --git a/vcell-cli/src/test/java/org/vcell/cli/run/BiosimulationsExecTest.java b/vcell-cli/src/test/java/org/vcell/cli/run/BiosimulationsExecTest.java
index b51c6c164a..4b1802308a 100644
--- a/vcell-cli/src/test/java/org/vcell/cli/run/BiosimulationsExecTest.java
+++ b/vcell-cli/src/test/java/org/vcell/cli/run/BiosimulationsExecTest.java
@@ -12,6 +12,7 @@
 import org.vcell.cli.CLIPythonManager;
 import org.vcell.cli.CLIRecordable;
 import org.vcell.cli.PythonStreamException;
+import org.vcell.sedml.testsupport.FailureType;
 import org.vcell.trace.Tracer;
 import org.vcell.util.VCellUtilityHub;
 import org.vcell.util.exe.Executable;
@@ -50,45 +51,45 @@ public static void teardown() throws Exception {
 		VCellUtilityHub.shutdown();
 	}
 
-	@SuppressWarnings("unused")
-	public enum FAULT {
-		ARRAY_INDEX_OUT_OF_BOUNDS,
-		BAD_EULER_FORWARD,
-		DIVIDE_BY_ZERO,
-		EXPRESSIONS_DIFFERENT,
-		EXPRESSION_BINDING,
-		GEOMETRY_SPEC_DIFFERENT,
-		HDF5_FILE_ALREADY_EXISTS, // reports.h5 file already exists, so action is blocked. Fixed in branch to be merged in.
-		MATHOVERRIDES_SurfToVol,
-		MATH_GENERATION_FAILURE,
-		MATH_OVERRIDES_A_FUNCTION,
-		MATH_OVERRIDES_INVALID,
-		NULL_POINTER_EXCEPTION,
-		OPERATION_NOT_SUPPORTED, // VCell simply doesn't have the necessary features to run this archive.
-		SBML_IMPORT_FAILURE,
-		SEDML_DIFF_NUMBER_OF_BIOMODELS,
-		SEDML_ERRONEOUS_UNIT_SYSTEM,
-		SEDML_ERROR_CONSTRUCTING_SIMCONTEXT,
-		SEDML_MATH_OVERRIDE_NAMES_DIFFERENT,
-		SEDML_MATH_OVERRIDE_NOT_EQUIVALENT,
-		SEDML_NONSPATIAL_STOCH_HISTOGRAM,
-		SEDML_NO_MODELS_IN_OMEX,
-		SEDML_SIMCONTEXT_NOT_FOUND_BY_NAME,
-		SEDML_SIMULATION_NOT_FOUND_BY_NAME,
-		SEDML_UNSUPPORTED_ENTITY,
-		SEDML_UNSUPPORTED_MODEL_REFERENCE, // Model refers to either a non-existent model (invalid SED-ML) or to another model with changes (not supported yet)
-		TOO_SLOW,
-		UNCATETORIZED_FAULT,
-		UNITS_EXCEPTION,
-		UNKNOWN_IDENTIFIER,
-		UNSUPPORTED_NONSPATIAL_STOCH_HISTOGRAM
-
-	}
+//	@SuppressWarnings("unused")
+//	public enum FailureType {
+//		ARRAY_INDEX_OUT_OF_BOUNDS,
+//		BAD_EULER_FORWARD,
+//		DIVIDE_BY_ZERO,
+//		EXPRESSIONS_DIFFERENT,
+//		EXPRESSION_BINDING,
+//		GEOMETRY_SPEC_DIFFERENT,
+//		HDF5_FILE_ALREADY_EXISTS, // reports.h5 file already exists, so action is blocked. Fixed in branch to be merged in.
+//		MATHOVERRIDES_SurfToVol,
+//		MATH_GENERATION_FAILURE,
+//		MATH_OVERRIDES_A_FUNCTION,
+//		MATH_OVERRIDES_INVALID,
+//		NULL_POINTER_EXCEPTION,
+//		OPERATION_NOT_SUPPORTED, // VCell simply doesn't have the necessary features to run this archive.
+//		SBML_IMPORT_FAILURE,
+//		SEDML_DIFF_NUMBER_OF_BIOMODELS,
+//		SEDML_ERRONEOUS_UNIT_SYSTEM,
+//		SEDML_ERROR_CONSTRUCTING_SIMCONTEXT,
+//		SEDML_MATH_OVERRIDE_NAMES_DIFFERENT,
+//		SEDML_MATH_OVERRIDE_NOT_EQUIVALENT,
+//		SEDML_NONSPATIAL_STOCH_HISTOGRAM,
+//		SEDML_NO_MODELS_IN_OMEX,
+//		SEDML_SIMCONTEXT_NOT_FOUND_BY_NAME,
+//		SEDML_SIMULATION_NOT_FOUND_BY_NAME,
+//		SEDML_UNSUPPORTED_ENTITY,
+//		SEDML_UNSUPPORTED_MODEL_REFERENCE, // Model refers to either a non-existent model (invalid SED-ML) or to another model with changes (not supported yet)
+//		TOO_SLOW,
+//		UNCATETORIZED_FailureType,
+//		UNITS_EXCEPTION,
+//		UNKNOWN_IDENTIFIER,
+//		UNSUPPORTED_NONSPATIAL_STOCH_HISTOGRAM
+//
+//	}
 
 	@SuppressWarnings("unused")
-	static Map<String, FAULT> knownFaults() {
-		HashMap<String, FAULT> faults = new HashMap<>();
-		return faults;
+	static Map<String, FailureType> knownFailureTypes() {
+		HashMap<String, FailureType> FailureTypes = new HashMap<>();
+		return FailureTypes;
 	}
 
 	public static Collection<String> testCases() {
@@ -138,7 +139,7 @@ public void writeImportErrorList(Exception e, String message) {
 	@ParameterizedTest
 	@MethodSource("testCases")
 	public void testBiosimulationsProject(String testCaseProjectID) throws Exception {
-		FAULT knownFault = knownFaults().get(testCaseProjectID);
+		FailureType knownFailureType = knownFailureTypes().get(testCaseProjectID);
 		try {
 			System.out.println("running test " + testCaseProjectID);
 
@@ -153,9 +154,9 @@ public void testBiosimulationsProject(String testCaseProjectID) throws Exception
 
 			String errorMessage = (Tracer.hasErrors()) ? "failure: '" + Tracer.getErrors().get(0).message.replace("\n", " | ") : "";
 			assertFalse(Tracer.hasErrors(), errorMessage);
-			if (knownFault != null){
-				throw new RuntimeException("test case passed, but expected " + knownFault.name() + ", remove "
-						+ testCaseProjectID + " from known faults");
+			if (knownFailureType != null){
+				throw new RuntimeException("test case passed, but expected " + knownFailureType.name() + ", remove "
+						+ testCaseProjectID + " from known FailureTypes");
 			}
 
 //			// verify log file has status of 'SUCCEEDED'
@@ -179,18 +180,18 @@ public void testBiosimulationsProject(String testCaseProjectID) throws Exception
 					stdOutString.substring(0, Math.min(300, stdOutString.length())));
 
 		} catch (Exception | AssertionError e){
-			FAULT fault = this.determineFault(e);
-			if (knownFault == fault) {
+			FailureType FailureType = this.determineFailureType(e);
+			if (knownFailureType == FailureType) {
 				System.err.println("Expected error: " + e.getMessage());
 				return;
 			}
 
-			System.err.println("add FAULT." + fault.name() + " to " + testCaseProjectID);
+			System.err.println("add FailureType." + FailureType.name() + " to " + testCaseProjectID);
 			throw new Exception("Test error: " + testCaseProjectID + " failed improperly", e);
 		}
 	}
 
-	private FAULT determineFault(Throwable caughtException){ // Throwable because Assertion Error
+	private FailureType determineFailureType(Throwable caughtException){ // Throwable because Assertion Error
 		String errorMessage = caughtException.getMessage();
 		if (errorMessage == null) errorMessage = ""; // Prevent nullptr exception
 
@@ -198,24 +199,24 @@ private FAULT determineFault(Throwable caughtException){ // Throwable because As
 			errorMessage = caughtException.getCause().getMessage();
 
 		if (errorMessage.contains("refers to either a non-existent model")) { //"refers to either a non-existent model (invalid SED-ML) or to another model with changes (not supported yet)"
-			return FAULT.SEDML_UNSUPPORTED_MODEL_REFERENCE;
+			return FailureType.SEDML_UNSUPPORTED_MODEL_REFERENCE;
 		} else if (errorMessage.contains("System IO encountered a fatal error")){
 			Throwable subException = caughtException.getCause();
 			//String subMessage = (subException == null) ? "" : subException.getMessage();
 			if (subException instanceof FileAlreadyExistsException){
-				return FAULT.HDF5_FILE_ALREADY_EXISTS;
+				return FailureType.HDF5_FILE_ALREADY_EXISTS;
 			}
 		} else if (errorMessage.contains("error while processing outputs: null")){
 			Throwable subException = caughtException.getCause();
 			if (subException instanceof ArrayIndexOutOfBoundsException){
-				return FAULT.ARRAY_INDEX_OUT_OF_BOUNDS;
+				return FailureType.ARRAY_INDEX_OUT_OF_BOUNDS;
 			}
 		} else if (errorMessage.contains("nconsistent unit system in SBML model") ||
 				errorMessage.contains("ust be of type")){
-			return FAULT.SEDML_ERRONEOUS_UNIT_SYSTEM;
+			return FailureType.SEDML_ERRONEOUS_UNIT_SYSTEM;
 		}
 
-		return FAULT.UNCATETORIZED_FAULT;
+		return FailureType.UNCATETORIZED_FAULT;
 	}
 
 }
diff --git a/vcell-cli/src/test/java/org/vcell/cli/run/SpatialExecTest.java b/vcell-cli/src/test/java/org/vcell/cli/run/SpatialExecTest.java
index 400404b3bd..c8d91596e2 100644
--- a/vcell-cli/src/test/java/org/vcell/cli/run/SpatialExecTest.java
+++ b/vcell-cli/src/test/java/org/vcell/cli/run/SpatialExecTest.java
@@ -1,7 +1,6 @@
 package org.vcell.cli.run;
 
 import cbit.vcell.mongodb.VCMongoMessage;
-import cbit.vcell.resource.NativeLib;
 import cbit.vcell.resource.PropertyLoader;
 import org.apache.commons.io.FileUtils;
 import org.junit.jupiter.api.AfterAll;
@@ -13,7 +12,7 @@
 import org.vcell.cli.CLIPythonManager;
 import org.vcell.cli.CLIRecordable;
 import org.vcell.cli.PythonStreamException;
-import org.vcell.trace.TraceEvent;
+import org.vcell.sedml.testsupport.FailureType;
 import org.vcell.trace.Tracer;
 import org.vcell.util.VCellUtilityHub;
 
@@ -48,40 +47,39 @@ public static void teardown() throws Exception {
         VCellUtilityHub.shutdown();
     }
 
-    @SuppressWarnings("unused")
-    public enum FAULT {
-        ARRAY_INDEX_OUT_OF_BOUNDS,
-        BAD_EULER_FORWARD,
-        DIVIDE_BY_ZERO,
-        EXPRESSIONS_DIFFERENT,
-        EXPRESSION_BINDING,
-        GEOMETRY_SPEC_DIFFERENT,
-        HDF5_FILE_ALREADY_EXISTS, // reports.h5 file already exists, so action is blocked. Fixed in branch to be merged in.
-        MATHOVERRIDES_SurfToVol,
-        MATH_GENERATION_FAILURE,
-        MATH_OVERRIDES_A_FUNCTION,
-        MATH_OVERRIDES_INVALID,
-        NULL_POINTER_EXCEPTION,
-        OPERATION_NOT_SUPPORTED, // VCell simply doesn't have the necessary features to run this archive.
-        SBML_IMPORT_FAILURE,
-        SEDML_DIFF_NUMBER_OF_BIOMODELS,
-        SEDML_ERRONEOUS_UNIT_SYSTEM,
-        SEDML_ERROR_CONSTRUCTING_SIMCONTEXT,
-        SEDML_MATH_OVERRIDE_NAMES_DIFFERENT,
-        SEDML_MATH_OVERRIDE_NOT_EQUIVALENT,
-        SEDML_NONSPATIAL_STOCH_HISTOGRAM,
-        SEDML_NO_MODELS_IN_OMEX,
-        SEDML_SIMCONTEXT_NOT_FOUND_BY_NAME,
-        SEDML_SIMULATION_NOT_FOUND_BY_NAME,
-        SEDML_UNSUPPORTED_ENTITY,
-        SEDML_UNSUPPORTED_MODEL_REFERENCE, // Model refers to either a non-existent model (invalid SED-ML) or to another model with changes (not supported yet)
-        TOO_SLOW,
-        UNCATETORIZED_FAULT,
-        UNITS_EXCEPTION,
-        UNKNOWN_IDENTIFIER,
-        UNSUPPORTED_NONSPATIAL_STOCH_HISTOGRAM
-
-    }
+//    public enum FAULT {
+//        ARRAY_INDEX_OUT_OF_BOUNDS,
+//        BAD_EULER_FORWARD,
+//        DIVIDE_BY_ZERO,
+//        EXPRESSIONS_DIFFERENT,
+//        EXPRESSION_BINDING,
+//        GEOMETRY_SPEC_DIFFERENT,
+//        HDF5_FILE_ALREADY_EXISTS, // reports.h5 file already exists, so action is blocked. Fixed in branch to be merged in.
+//        MATHOVERRIDES_SurfToVol,
+//        MATH_GENERATION_FAILURE,
+//        MATH_OVERRIDES_A_FUNCTION,
+//        MATH_OVERRIDES_INVALID,
+//        NULL_POINTER_EXCEPTION,
+//        OPERATION_NOT_SUPPORTED, // VCell simply doesn't have the necessary features to run this archive.
+//        SBML_IMPORT_FAILURE,
+//        SEDML_DIFF_NUMBER_OF_BIOMODELS,
+//        SEDML_ERRONEOUS_UNIT_SYSTEM,
+//        SEDML_ERROR_CONSTRUCTING_SIMCONTEXT,
+//        SEDML_MATH_OVERRIDE_NAMES_DIFFERENT,
+//        SEDML_MATH_OVERRIDE_NOT_EQUIVALENT,
+//        SEDML_NONSPATIAL_STOCH_HISTOGRAM,
+//        SEDML_NO_MODELS_IN_OMEX,
+//        SEDML_SIMCONTEXT_NOT_FOUND_BY_NAME,
+//        SEDML_SIMULATION_NOT_FOUND_BY_NAME,
+//        SEDML_UNSUPPORTED_ENTITY,
+//        SEDML_UNSUPPORTED_MODEL_REFERENCE, // Model refers to either a non-existent model (invalid SED-ML) or to another model with changes (not supported yet)
+//        TOO_SLOW,
+//        UNCATETORIZED_FAULT,
+//        UNITS_EXCEPTION,
+//        UNKNOWN_IDENTIFIER,
+//        UNSUPPORTED_NONSPATIAL_STOCH_HISTOGRAM
+//
+//    }
 
     static Set<String> blacklistedModels(){
         HashSet<String> blacklistSet = new HashSet<>();
@@ -89,8 +87,8 @@ static Set<String> blacklistedModels(){
         return blacklistSet;
     }
 
-    static Map<String, SpatialExecTest.FAULT> knownFaults() {
-        HashMap<String, SpatialExecTest.FAULT> faults = new HashMap<>();
+    static Map<String, FailureType> knownFaults() {
+        HashMap<String, FailureType> faults = new HashMap<>();
         // Hooray! no known faults yet!
         return faults;
     }
@@ -105,14 +103,7 @@ public static Collection<String> testCases() {
     @ParameterizedTest
     @MethodSource("testCases")
     public void testSpatialOmex(String testCaseFilename) throws Exception {
-        String osName = System.getProperty("os.name").toLowerCase();
-        String osArch = System.getProperty("os.arch").toLowerCase();
-
-        // Skip test if running on macOS ARM64
-        Assumptions.assumeFalse(osName.contains("mac") && osArch.equals("aarch64"),
-                "Test skipped on macOS ARM64");
-
-        SpatialExecTest.FAULT knownFault = knownFaults().get(testCaseFilename);
+        FailureType knownFault = knownFaults().get(testCaseFilename);
         Tracer.clearTraceEvents();
         try {
             System.out.println("running test " + testCaseFilename);
@@ -155,14 +146,14 @@ public void writeImportErrorList(Exception e, String message) {
             FileUtils.copyInputStreamToFile(omexInputStream, omexFile.toFile());
             ExecuteImpl.singleMode(omexFile.toFile(), outdirPath.toFile(), cliRecorder);
 
-            assertFalse(Tracer.hasErrors(), "no exception, but trace errors found: error[0] = "+Tracer.getErrors().get(0).message.replace("\n"," | "));
+            assertFalse(Tracer.hasErrors(), "no exception, but trace errors found: error[0] = "+(Tracer.hasErrors()?(Tracer.getErrors().get(0).message.replace("\n"," | ")):null));
             if (knownFault != null){
                 throw new RuntimeException("test case passed, but expected " + knownFault.name() + ", remove "
                         + testCaseFilename + " from known faults");
             }
 
         } catch (Exception | AssertionError e){
-            SpatialExecTest.FAULT fault = this.determineFault(e);
+            FailureType fault = this.determineFault(e);
             if (knownFault == fault) {
                 System.err.println("Expected error: " + e.getMessage());
                 return;
@@ -173,7 +164,7 @@ public void writeImportErrorList(Exception e, String message) {
         }
     }
 
-    private SpatialExecTest.FAULT determineFault(Throwable caughtException){ // Throwable because Assertion Error
+    private FailureType determineFault(Throwable caughtException){ // Throwable because Assertion Error
         String errorMessage = caughtException.getMessage();
         if (errorMessage == null) errorMessage = ""; // Prevent nullptr exception
 
@@ -181,23 +172,23 @@ private SpatialExecTest.FAULT determineFault(Throwable caughtException){ // Thro
             errorMessage = caughtException.getCause().getMessage();
 
         if (errorMessage.contains("refers to either a non-existent model")) { //"refers to either a non-existent model (invalid SED-ML) or to another model with changes (not supported yet)"
-            return SpatialExecTest.FAULT.SEDML_UNSUPPORTED_MODEL_REFERENCE;
+            return FailureType.SEDML_UNSUPPORTED_MODEL_REFERENCE;
         } else if (errorMessage.contains("System IO encountered a fatal error")){
             Throwable subException = caughtException.getCause();
             //String subMessage = (subException == null) ? "" : subException.getMessage();
             if (subException instanceof FileAlreadyExistsException){
-                return SpatialExecTest.FAULT.HDF5_FILE_ALREADY_EXISTS;
+                return FailureType.HDF5_FILE_ALREADY_EXISTS;
             }
         } else if (errorMessage.contains("error while processing outputs: null")){
             Throwable subException = caughtException.getCause();
             if (subException instanceof ArrayIndexOutOfBoundsException){
-                return SpatialExecTest.FAULT.ARRAY_INDEX_OUT_OF_BOUNDS;
+                return FailureType.ARRAY_INDEX_OUT_OF_BOUNDS;
             }
         } else if (errorMessage.contains("nconsistent unit system in SBML model") ||
                 errorMessage.contains("ust be of type")){
-            return SpatialExecTest.FAULT.SEDML_ERRONEOUS_UNIT_SYSTEM;
+            return FailureType.SEDML_ERRONEOUS_UNIT_SYSTEM;
         }
 
-        return SpatialExecTest.FAULT.UNCATETORIZED_FAULT;
+        return FailureType.UNCATETORIZED_FAULT;
     }
 }

From 92e777c829f9df4a3b127e43e4292c23042fbb45 Mon Sep 17 00:00:00 2001
From: jcschaff <schaff.jim@gmail.com>
Date: Fri, 27 Dec 2024 11:30:51 -0500
Subject: [PATCH 41/45] renamed tests QuantOmexExecTest and integrated with
 unified test cases

---
 .../src/main/resources/test_cases.ndjson      |   4 +
 .../cli/run/BSTSBasedTestSuiteFiles.java      |   6 +-
 .../vcell/cli/run/BiosimulationsFiles.java    |  63 -------------
 .../org/vcell/cli/run/QuantOmexExecFiles.java |  58 ++++++++++++
 ...nsExecTest.java => QuantOmexExecTest.java} |  86 ++++++------------
 ...ellurium_A_minimal_cascade_model_for_th.h5 | Bin
 ...m_A_minimal_cascade_model_for_th.spec.omex | Bin
 ...ellurium_Modeling_the_cell_division_cyc.h5 | Bin
 ...m_Modeling_the_cell_division_cyc.spec.omex | Bin
 ...ellurium_A_simple_model_of_circadian_rh.h5 | Bin
 ...m_A_simple_model_of_circadian_rh.spec.omex | Bin
 .../BIOMD0000000300.h5                        | Bin
 .../BIOMD0000000300.spec.omex                 | Bin
 .../testsupport/OmexTestingDatabase.java      |  31 +++++++
 14 files changed, 124 insertions(+), 124 deletions(-)
 delete mode 100644 vcell-cli/src/test/java/org/vcell/cli/run/BiosimulationsFiles.java
 create mode 100644 vcell-cli/src/test/java/org/vcell/cli/run/QuantOmexExecFiles.java
 rename vcell-cli/src/test/java/org/vcell/cli/run/{BiosimulationsExecTest.java => QuantOmexExecTest.java} (66%)
 rename vcell-cli/src/test/resources/{BiosimulationsOmexWithResults => OmexWithThirdPartyResults}/BIOMD0000000003_tellurium_A_minimal_cascade_model_for_th.h5 (100%)
 rename vcell-cli/src/test/resources/{BiosimulationsOmexWithResults => OmexWithThirdPartyResults}/BIOMD0000000003_tellurium_A_minimal_cascade_model_for_th.spec.omex (100%)
 rename vcell-cli/src/test/resources/{BiosimulationsOmexWithResults => OmexWithThirdPartyResults}/BIOMD0000000006_tellurium_Modeling_the_cell_division_cyc.h5 (100%)
 rename vcell-cli/src/test/resources/{BiosimulationsOmexWithResults => OmexWithThirdPartyResults}/BIOMD0000000006_tellurium_Modeling_the_cell_division_cyc.spec.omex (100%)
 rename vcell-cli/src/test/resources/{BiosimulationsOmexWithResults => OmexWithThirdPartyResults}/BIOMD0000000036_tellurium_A_simple_model_of_circadian_rh.h5 (100%)
 rename vcell-cli/src/test/resources/{BiosimulationsOmexWithResults => OmexWithThirdPartyResults}/BIOMD0000000036_tellurium_A_simple_model_of_circadian_rh.spec.omex (100%)
 rename vcell-cli/src/test/resources/{BiosimulationsOmexWithResults => OmexWithThirdPartyResults}/BIOMD0000000300.h5 (100%)
 rename vcell-cli/src/test/resources/{BiosimulationsOmexWithResults => OmexWithThirdPartyResults}/BIOMD0000000300.spec.omex (100%)

diff --git a/vcell-cli/src/main/resources/test_cases.ndjson b/vcell-cli/src/main/resources/test_cases.ndjson
index cd676755e8..304ae9e5ab 100644
--- a/vcell-cli/src/main/resources/test_cases.ndjson
+++ b/vcell-cli/src/main/resources/test_cases.ndjson
@@ -1,3 +1,7 @@
+{"test_collection":"VCELL_QUANT_OMEX","file_path":"BIOMD0000000003_tellurium_A_minimal_cascade_model_for_th","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"VCELL_QUANT_OMEX","file_path":"BIOMD0000000006_tellurium_Modeling_the_cell_division_cyc","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"VCELL_QUANT_OMEX","file_path":"BIOMD0000000036_tellurium_A_simple_model_of_circadian_rh","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"VCELL_QUANT_OMEX","file_path":"BIOMD0000000300","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"VCELL_BIOMD","file_path":"BIOMD0000000005.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"VCELL_BIOMD","file_path":"BIOMD0000000175.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"MATH_GENERATION_FAILURE","known_failure_desc":"unknown identifier I_Net_E44PPI3K_binding"}
 {"test_collection":"VCELL_BIOMD","file_path":"BIOMD0000000302.omex","should_fail":false,"known_status":"FAIL","known_failure_type":"MATH_GENERATION_FAILURE","known_failure_desc":"Initial condition for variable 'h_post' references variable 'V_post'"}
diff --git a/vcell-cli/src/test/java/org/vcell/cli/run/BSTSBasedTestSuiteFiles.java b/vcell-cli/src/test/java/org/vcell/cli/run/BSTSBasedTestSuiteFiles.java
index c566913afe..d5c4d267d2 100644
--- a/vcell-cli/src/test/java/org/vcell/cli/run/BSTSBasedTestSuiteFiles.java
+++ b/vcell-cli/src/test/java/org/vcell/cli/run/BSTSBasedTestSuiteFiles.java
@@ -19,7 +19,11 @@ public class BSTSBasedTestSuiteFiles {
 
     public static OmexTestCase[] getBSTSTestCases() throws IOException {
         List<OmexTestCase> allTestCases = OmexTestingDatabase.loadOmexTestCases();
-        Predicate<OmexTestCase> testFilter = t -> t.test_collection.repo == OmexTestingDatabase.TestDataRepo.vcell;
+        Predicate<OmexTestCase> testFilter = t ->
+                        t.test_collection == OmexTestingDatabase.TestCollection.VCELL_BIOMD ||
+                        t.test_collection == OmexTestingDatabase.TestCollection.VCELL_BSTS_VCML ||
+                        t.test_collection == OmexTestingDatabase.TestCollection.VCELL_BSTS_SBML_CORE ||
+                        t.test_collection == OmexTestingDatabase.TestCollection.VCELL_BSTS_SYNTHS;
 
         return allTestCases.stream().filter(testFilter).toArray(OmexTestCase[]::new);
     }
diff --git a/vcell-cli/src/test/java/org/vcell/cli/run/BiosimulationsFiles.java b/vcell-cli/src/test/java/org/vcell/cli/run/BiosimulationsFiles.java
deleted file mode 100644
index cfddd80d85..0000000000
--- a/vcell-cli/src/test/java/org/vcell/cli/run/BiosimulationsFiles.java
+++ /dev/null
@@ -1,63 +0,0 @@
-package org.vcell.cli.run;
-
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Tag;
-
-import java.io.InputStream;
-import java.util.Arrays;
-import java.util.function.Predicate;
-
-
-@Tag("Fast")
-public class BiosimulationsFiles {
-
-    private final static String[] allProjectIDs = new String[]{
-            "BIOMD0000000003_tellurium_A_minimal_cascade_model_for_th",
-            "BIOMD0000000006_tellurium_Modeling_the_cell_division_cyc",
-            "BIOMD0000000036_tellurium_A_simple_model_of_circadian_rh",
-            "BIOMD0000000300"
-    };
-
-    public static String[] getProjectIDs() {
-        Predicate<String> testFilter = t -> true;
-
-        return Arrays.stream(allProjectIDs).filter(testFilter).toArray(String[]::new);
-    }
-
-    public static InputStream getOmex(String projectID) {
-        if (!Arrays.stream(allProjectIDs).anyMatch(pid -> pid.equals(projectID))) {
-            throw new RuntimeException("project "+projectID+" not in project list");
-        }
-        return getFileFromResourceAsStream(projectID + ".spec.omex");
-    }
-
-     private static InputStream getFileFromResourceAsStream(String fileName) {
-        InputStream inputStream = BiosimulationsFiles.class.getResourceAsStream("/BiosimulationsOmexWithResults/"+fileName);
-        if (inputStream == null) {
-            throw new RuntimeException("file not found! " + fileName);
-        } else {
-            return inputStream;
-        }
-    }
-
-    public static InputStream getH5(String projectID) {
-        if (!Arrays.stream(allProjectIDs).anyMatch(pid -> pid.equals(projectID))) {
-            throw new RuntimeException("project "+projectID+" not in project list");
-        }
-        return getFileFromResourceAsStream(projectID + ".h5");
-    }
-
-    @Test
-    public void test_read_omex_file() {
-        InputStream inputStream = getOmex(allProjectIDs[0]);
-        Assertions.assertTrue(inputStream != null);
-    }
-
-    @Test
-    public void test_read_H5_file() {
-        InputStream inputStream = getH5(allProjectIDs[0]);
-        Assertions.assertTrue(inputStream != null);
-    }
-
-}
diff --git a/vcell-cli/src/test/java/org/vcell/cli/run/QuantOmexExecFiles.java b/vcell-cli/src/test/java/org/vcell/cli/run/QuantOmexExecFiles.java
new file mode 100644
index 0000000000..7938705473
--- /dev/null
+++ b/vcell-cli/src/test/java/org/vcell/cli/run/QuantOmexExecFiles.java
@@ -0,0 +1,58 @@
+package org.vcell.cli.run;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.vcell.sedml.testsupport.OmexTestCase;
+import org.vcell.sedml.testsupport.OmexTestingDatabase;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.function.Predicate;
+
+
+@Tag("Fast")
+public class QuantOmexExecFiles {
+
+    public static OmexTestCase[] getTestCases() throws IOException {
+        List<OmexTestCase> allTestCases = OmexTestingDatabase.loadOmexTestCases();
+        Predicate<OmexTestCase> testFilter = t -> t.test_collection == OmexTestingDatabase.TestCollection.VCELL_QUANT_OMEX;
+
+        return allTestCases.stream().filter(testFilter).toArray(OmexTestCase[]::new);
+    }
+
+    public static InputStream getOmex(OmexTestCase testCase) {
+        String fullPath = testCase.test_collection.pathPrefix + "/" + testCase.file_path;
+        String path = fullPath.substring(fullPath.indexOf("/OmexWithThirdPartyResults"));
+        return getFileFromResourceAsStream(path + ".spec.omex");
+    }
+
+     private static InputStream getFileFromResourceAsStream(String path) {
+        InputStream inputStream = QuantOmexExecFiles.class.getResourceAsStream(path);
+        if (inputStream == null) {
+            throw new RuntimeException("file not found! " + path);
+        } else {
+            return inputStream;
+        }
+    }
+
+    public static InputStream getH5(OmexTestCase testCase) {
+        String fullPath = testCase.test_collection.pathPrefix + "/" + testCase.file_path;
+        String path = fullPath.substring(fullPath.indexOf("/OmexWithThirdPartyResults"));
+        return getFileFromResourceAsStream(path + ".h5");
+    }
+
+    @Test
+    public void test_read_omex_file() throws IOException {
+        InputStream inputStream = getOmex(getTestCases()[0]);
+        Assertions.assertTrue(inputStream != null);
+    }
+
+    @Test
+    public void test_read_H5_file() throws IOException {
+        InputStream inputStream = getH5(getTestCases()[0]);
+        Assertions.assertTrue(inputStream != null);
+    }
+
+}
diff --git a/vcell-cli/src/test/java/org/vcell/cli/run/BiosimulationsExecTest.java b/vcell-cli/src/test/java/org/vcell/cli/run/QuantOmexExecTest.java
similarity index 66%
rename from vcell-cli/src/test/java/org/vcell/cli/run/BiosimulationsExecTest.java
rename to vcell-cli/src/test/java/org/vcell/cli/run/QuantOmexExecTest.java
index 4b1802308a..7694fbfb78 100644
--- a/vcell-cli/src/test/java/org/vcell/cli/run/BiosimulationsExecTest.java
+++ b/vcell-cli/src/test/java/org/vcell/cli/run/QuantOmexExecTest.java
@@ -1,7 +1,6 @@
 package org.vcell.cli.run;
 
 import cbit.vcell.mongodb.VCMongoMessage;
-import cbit.vcell.resource.NativeLib;
 import cbit.vcell.resource.PropertyLoader;
 import org.apache.commons.io.FileUtils;
 import org.junit.jupiter.api.AfterAll;
@@ -13,6 +12,9 @@
 import org.vcell.cli.CLIRecordable;
 import org.vcell.cli.PythonStreamException;
 import org.vcell.sedml.testsupport.FailureType;
+import org.vcell.sedml.testsupport.OmexTestCase;
+import org.vcell.sedml.testsupport.OmexTestingDatabase;
+import org.vcell.trace.TraceEvent;
 import org.vcell.trace.Tracer;
 import org.vcell.util.VCellUtilityHub;
 import org.vcell.util.exe.Executable;
@@ -20,20 +22,16 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.*;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 import static org.junit.jupiter.api.Assertions.assertFalse;
 
 @Tag("BSTS_IT")
-public class BiosimulationsExecTest {
+public class QuantOmexExecTest {
 	@BeforeAll
 	public static void setup() throws PythonStreamException, IOException {
 		PropertyLoader.setProperty(PropertyLoader.installationRoot, new File("..").getAbsolutePath());
@@ -92,10 +90,10 @@ static Map<String, FailureType> knownFailureTypes() {
 		return FailureTypes;
 	}
 
-	public static Collection<String> testCases() {
-		Predicate<String> projectFilter;
+	public static Collection<OmexTestCase> testCases() throws IOException {
+		Predicate<OmexTestCase> projectFilter;
 		projectFilter = (t) -> true; // don't skip any for now.
-		return Arrays.stream(BiosimulationsFiles.getProjectIDs()).filter(projectFilter).collect(Collectors.toList());
+		return Arrays.stream(QuantOmexExecFiles.getTestCases()).filter(projectFilter).collect(Collectors.toList());
 	}
 
     static class TestRecorder implements CLIRecordable {
@@ -138,13 +136,13 @@ public void writeImportErrorList(Exception e, String message) {
 
 	@ParameterizedTest
 	@MethodSource("testCases")
-	public void testBiosimulationsProject(String testCaseProjectID) throws Exception {
-		FailureType knownFailureType = knownFailureTypes().get(testCaseProjectID);
+	public void testBiosimulationsProject(OmexTestCase testCase) throws Exception {
+		FailureType knownFailureType = testCase.known_failure_type;
 		try {
-			System.out.println("running test " + testCaseProjectID);
+			System.out.println("running testCase " + testCase.test_collection + " " + testCase.file_path);
 
 			Path outdirPath = Files.createTempDirectory("BiosimulationsExecTest");
-			InputStream omexInputStream = BiosimulationsFiles.getOmex(testCaseProjectID);
+			InputStream omexInputStream = QuantOmexExecFiles.getOmex(testCase);
             Path omexFile = Files.createTempFile("BiosimulationsExec_", "omex");
 			FileUtils.copyInputStreamToFile(omexInputStream, omexFile.toFile());
 
@@ -153,21 +151,17 @@ public void testBiosimulationsProject(String testCaseProjectID) throws Exception
 			Path computedH5File = outdirPath.resolve("report.h5");
 
 			String errorMessage = (Tracer.hasErrors()) ? "failure: '" + Tracer.getErrors().get(0).message.replace("\n", " | ") : "";
-			assertFalse(Tracer.hasErrors(), errorMessage);
+			if (Tracer.hasErrors()) {
+				throw new RuntimeException(errorMessage);
+			}
 			if (knownFailureType != null){
 				throw new RuntimeException("test case passed, but expected " + knownFailureType.name() + ", remove "
-						+ testCaseProjectID + " from known FailureTypes");
+						+ testCase.file_path + " from known FailureTypes");
 			}
 
-//			// verify log file has status of 'SUCCEEDED'
-//			Gson gson = new Gson();
-//			try (Reader jsonReader = new )
-//			}
-//			gson.newJsonReader(new )
-
 			// compare hdf5 files to within absolute tolerance of 1e-9
 			double absTolerance = 1e-9;
-			InputStream h5fileStream = BiosimulationsFiles.getH5(testCaseProjectID);
+			InputStream h5fileStream = QuantOmexExecFiles.getH5(testCase);
 			Path expectedH5File = Files.createTempFile("BiosimulationsExec_", "h5");
 			FileUtils.copyInputStreamToFile(h5fileStream, expectedH5File.toFile());
 			Executable command = new Executable(new String[]{ "sh", "-c", "h5diff", "-r", "--delta", Double.toString(absTolerance),
@@ -176,47 +170,19 @@ public void testBiosimulationsProject(String testCaseProjectID) throws Exception
 			});
 			command.start(new int[] { 0, 1 });
 			String stdOutString = command.getStdoutString();
-			assertFalse(stdOutString.contains("position"), "H5 files have significant differences: " +
-					stdOutString.substring(0, Math.min(300, stdOutString.length())));
-
-		} catch (Exception | AssertionError e){
-			FailureType FailureType = this.determineFailureType(e);
-			if (knownFailureType == FailureType) {
+			if (stdOutString.contains("position")){
+				throw new RuntimeException("H5 files have significant differences: " + stdOutString.substring(0, Math.min(300, stdOutString.length())));
+			}
+		} catch (Exception e){
+			List<TraceEvent> errorEvents = Tracer.getErrors();
+			FailureType observedFailure = OmexTestingDatabase.determineFault(e, errorEvents);
+			if (knownFailureType == observedFailure) {
 				System.err.println("Expected error: " + e.getMessage());
 				return;
 			}
 
-			System.err.println("add FailureType." + FailureType.name() + " to " + testCaseProjectID);
-			throw new Exception("Test error: " + testCaseProjectID + " failed improperly", e);
+			System.err.println("add FailureType." + observedFailure.name() + " to " + testCase.file_path);
+			throw new Exception("Test error: " + testCase.file_path + " failed improperly", e);
 		}
 	}
-
-	private FailureType determineFailureType(Throwable caughtException){ // Throwable because Assertion Error
-		String errorMessage = caughtException.getMessage();
-		if (errorMessage == null) errorMessage = ""; // Prevent nullptr exception
-
-		if (caughtException instanceof Error && caughtException.getCause() != null)
-			errorMessage = caughtException.getCause().getMessage();
-
-		if (errorMessage.contains("refers to either a non-existent model")) { //"refers to either a non-existent model (invalid SED-ML) or to another model with changes (not supported yet)"
-			return FailureType.SEDML_UNSUPPORTED_MODEL_REFERENCE;
-		} else if (errorMessage.contains("System IO encountered a fatal error")){
-			Throwable subException = caughtException.getCause();
-			//String subMessage = (subException == null) ? "" : subException.getMessage();
-			if (subException instanceof FileAlreadyExistsException){
-				return FailureType.HDF5_FILE_ALREADY_EXISTS;
-			}
-		} else if (errorMessage.contains("error while processing outputs: null")){
-			Throwable subException = caughtException.getCause();
-			if (subException instanceof ArrayIndexOutOfBoundsException){
-				return FailureType.ARRAY_INDEX_OUT_OF_BOUNDS;
-			}
-		} else if (errorMessage.contains("nconsistent unit system in SBML model") ||
-				errorMessage.contains("ust be of type")){
-			return FailureType.SEDML_ERRONEOUS_UNIT_SYSTEM;
-		}
-
-		return FailureType.UNCATETORIZED_FAULT;
-	}
-
 }
diff --git a/vcell-cli/src/test/resources/BiosimulationsOmexWithResults/BIOMD0000000003_tellurium_A_minimal_cascade_model_for_th.h5 b/vcell-cli/src/test/resources/OmexWithThirdPartyResults/BIOMD0000000003_tellurium_A_minimal_cascade_model_for_th.h5
similarity index 100%
rename from vcell-cli/src/test/resources/BiosimulationsOmexWithResults/BIOMD0000000003_tellurium_A_minimal_cascade_model_for_th.h5
rename to vcell-cli/src/test/resources/OmexWithThirdPartyResults/BIOMD0000000003_tellurium_A_minimal_cascade_model_for_th.h5
diff --git a/vcell-cli/src/test/resources/BiosimulationsOmexWithResults/BIOMD0000000003_tellurium_A_minimal_cascade_model_for_th.spec.omex b/vcell-cli/src/test/resources/OmexWithThirdPartyResults/BIOMD0000000003_tellurium_A_minimal_cascade_model_for_th.spec.omex
similarity index 100%
rename from vcell-cli/src/test/resources/BiosimulationsOmexWithResults/BIOMD0000000003_tellurium_A_minimal_cascade_model_for_th.spec.omex
rename to vcell-cli/src/test/resources/OmexWithThirdPartyResults/BIOMD0000000003_tellurium_A_minimal_cascade_model_for_th.spec.omex
diff --git a/vcell-cli/src/test/resources/BiosimulationsOmexWithResults/BIOMD0000000006_tellurium_Modeling_the_cell_division_cyc.h5 b/vcell-cli/src/test/resources/OmexWithThirdPartyResults/BIOMD0000000006_tellurium_Modeling_the_cell_division_cyc.h5
similarity index 100%
rename from vcell-cli/src/test/resources/BiosimulationsOmexWithResults/BIOMD0000000006_tellurium_Modeling_the_cell_division_cyc.h5
rename to vcell-cli/src/test/resources/OmexWithThirdPartyResults/BIOMD0000000006_tellurium_Modeling_the_cell_division_cyc.h5
diff --git a/vcell-cli/src/test/resources/BiosimulationsOmexWithResults/BIOMD0000000006_tellurium_Modeling_the_cell_division_cyc.spec.omex b/vcell-cli/src/test/resources/OmexWithThirdPartyResults/BIOMD0000000006_tellurium_Modeling_the_cell_division_cyc.spec.omex
similarity index 100%
rename from vcell-cli/src/test/resources/BiosimulationsOmexWithResults/BIOMD0000000006_tellurium_Modeling_the_cell_division_cyc.spec.omex
rename to vcell-cli/src/test/resources/OmexWithThirdPartyResults/BIOMD0000000006_tellurium_Modeling_the_cell_division_cyc.spec.omex
diff --git a/vcell-cli/src/test/resources/BiosimulationsOmexWithResults/BIOMD0000000036_tellurium_A_simple_model_of_circadian_rh.h5 b/vcell-cli/src/test/resources/OmexWithThirdPartyResults/BIOMD0000000036_tellurium_A_simple_model_of_circadian_rh.h5
similarity index 100%
rename from vcell-cli/src/test/resources/BiosimulationsOmexWithResults/BIOMD0000000036_tellurium_A_simple_model_of_circadian_rh.h5
rename to vcell-cli/src/test/resources/OmexWithThirdPartyResults/BIOMD0000000036_tellurium_A_simple_model_of_circadian_rh.h5
diff --git a/vcell-cli/src/test/resources/BiosimulationsOmexWithResults/BIOMD0000000036_tellurium_A_simple_model_of_circadian_rh.spec.omex b/vcell-cli/src/test/resources/OmexWithThirdPartyResults/BIOMD0000000036_tellurium_A_simple_model_of_circadian_rh.spec.omex
similarity index 100%
rename from vcell-cli/src/test/resources/BiosimulationsOmexWithResults/BIOMD0000000036_tellurium_A_simple_model_of_circadian_rh.spec.omex
rename to vcell-cli/src/test/resources/OmexWithThirdPartyResults/BIOMD0000000036_tellurium_A_simple_model_of_circadian_rh.spec.omex
diff --git a/vcell-cli/src/test/resources/BiosimulationsOmexWithResults/BIOMD0000000300.h5 b/vcell-cli/src/test/resources/OmexWithThirdPartyResults/BIOMD0000000300.h5
similarity index 100%
rename from vcell-cli/src/test/resources/BiosimulationsOmexWithResults/BIOMD0000000300.h5
rename to vcell-cli/src/test/resources/OmexWithThirdPartyResults/BIOMD0000000300.h5
diff --git a/vcell-cli/src/test/resources/BiosimulationsOmexWithResults/BIOMD0000000300.spec.omex b/vcell-cli/src/test/resources/OmexWithThirdPartyResults/BIOMD0000000300.spec.omex
similarity index 100%
rename from vcell-cli/src/test/resources/BiosimulationsOmexWithResults/BIOMD0000000300.spec.omex
rename to vcell-cli/src/test/resources/OmexWithThirdPartyResults/BIOMD0000000300.spec.omex
diff --git a/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestingDatabase.java b/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestingDatabase.java
index 76e70c8021..f6d5fab47e 100644
--- a/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestingDatabase.java
+++ b/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestingDatabase.java
@@ -21,6 +21,7 @@ public enum TestDataRepo {
         vcell, sysbio
     }
     public enum TestCollection {
+        VCELL_QUANT_OMEX(TestDataRepo.vcell, "vcell-cli/src/test/resources/OmexWithThirdPartyResults"),
         VCELL_BIOMD(TestDataRepo.vcell, "vcell-cli/src/test/resources/bsts-omex/misc-projects"),
         VCELL_BSTS_VCML(TestDataRepo.vcell, "vcell-cli/src/test/resources/bsts-omex/vcml"),
         VCELL_BSTS_SBML_CORE(TestDataRepo.vcell, "vcell-cli/src/test/resources/bsts-omex/sbml-core"),
@@ -146,4 +147,34 @@ public static FailureType determineFault(Exception caughtException, List<TraceEv
         return FailureType.UNCATETORIZED_FAULT;
     }
 
+
+    private FailureType determineFailureType_FROM_QUANT_OMEX_EXEC(Throwable caughtException){ // Throwable because Assertion Error
+        String errorMessage = caughtException.getMessage();
+        if (errorMessage == null) errorMessage = ""; // Prevent nullptr exception
+
+        if (caughtException instanceof Error && caughtException.getCause() != null)
+            errorMessage = caughtException.getCause().getMessage();
+
+        if (errorMessage.contains("refers to either a non-existent model")) { //"refers to either a non-existent model (invalid SED-ML) or to another model with changes (not supported yet)"
+            return FailureType.SEDML_UNSUPPORTED_MODEL_REFERENCE;
+        } else if (errorMessage.contains("System IO encountered a fatal error")){
+            Throwable subException = caughtException.getCause();
+            //String subMessage = (subException == null) ? "" : subException.getMessage();
+            if (subException instanceof FileAlreadyExistsException){
+                return FailureType.HDF5_FILE_ALREADY_EXISTS;
+            }
+        } else if (errorMessage.contains("error while processing outputs: null")){
+            Throwable subException = caughtException.getCause();
+            if (subException instanceof ArrayIndexOutOfBoundsException){
+                return FailureType.ARRAY_INDEX_OUT_OF_BOUNDS;
+            }
+        } else if (errorMessage.contains("nconsistent unit system in SBML model") ||
+                errorMessage.contains("ust be of type")){
+            return FailureType.SEDML_ERRONEOUS_UNIT_SYSTEM;
+        }
+
+        return FailureType.UNCATETORIZED_FAULT;
+    }
+
+
 }

From d37599aea58ad03b160101c0d2c18e87c6bf407a Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Mon, 30 Dec 2024 17:28:44 -0500
Subject: [PATCH 42/45] Made Markdown report "pretty"

---
 .../cli/testsupport/TestOmexTestReport.java   |  33 ++
 .../sedml/testsupport/OmexTestReport.java     | 372 +++++++++++++++---
 .../testsupport/OmexTestingDatabase.java      |  37 +-
 3 files changed, 357 insertions(+), 85 deletions(-)
 create mode 100644 vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestReport.java

diff --git a/vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestReport.java b/vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestReport.java
new file mode 100644
index 0000000000..57e9e9bc87
--- /dev/null
+++ b/vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestReport.java
@@ -0,0 +1,33 @@
+package org.vcell.cli.testsupport;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.vcell.sedml.testsupport.*;
+
+public class TestOmexTestReport {
+    @Test
+    void checkTableIsMade(){
+        OmexTestReport.MarkdownTable table = new OmexTestReport.MarkdownTable();
+        table.setNumRowsAndColums(4,3);
+        table.setRowTitle(0, "VCell Developer");
+        table.setRowTitle(1, "Favorite Byte");
+        table.setRowTitle(2, "Favorite Pizza Topping");
+        table.setRowTitle(3, "Favorite Number");
+        table.setColumnTitle(0, "Logan");
+        table.setColumnTitle(1, "Jim");
+        table.setColumnTitle(2, "Alex");
+        table.setTableValue(0, 0, true);
+        table.setTableValue(0, 1, true);
+        table.setTableValue(0, 2, false);
+        table.setTableValue(1, 0, (byte)136);
+        table.setTableValue(1, 1, (byte)255);
+        table.setTableValue(1, 2, (byte)0);
+        table.setTableValue(2, 0, "Pepperoni");
+        table.setTableValue(2, 1, "Feta Cheeze"); // Spelling is correct
+        table.setTableValue(2, 2, "Extra Cheese");
+        table.setTableValue(3, 0, 17);
+        table.setTableValue(3, 1, 42);
+        table.setTableValue(3, 2, 3.14159);
+        System.out.println(table.getMarkdownTable());
+    }
+}
diff --git a/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestReport.java b/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestReport.java
index 61b30beb9c..156538181a 100644
--- a/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestReport.java
+++ b/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestReport.java
@@ -4,15 +4,16 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.vcell.util.Pair;
 
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 public class OmexTestReport {
+    private final static Logger logger = LogManager.getLogger(OmexTestReport.class);
 
     public static class OmexTestCaseChange {
         public final OmexTestCase original;
@@ -41,7 +42,8 @@ public static class OmexTestStatistics {
     public final List<OmexExecSummary> unmatchedExecSummaries = new ArrayList<>();
     public final List<OmexTestCase> unmatchedTestCases = new ArrayList<>();
     public final OmexTestStatistics statistics = new OmexTestStatistics();
-    public final Map<FailureType,Integer> failureTypeCounts = new HashMap<>();
+    public final Map<FailureType, Integer> historicalFailureTypeCounts = new HashMap<>();
+    public final Map<FailureType, Integer> currentFailureTypeCounts = new HashMap<>();
 
     public OmexTestReport(List<OmexTestCase> testCases, List<OmexExecSummary> execSummaries) {
         // find common path prefix of list of Path objects
@@ -54,10 +56,17 @@ public OmexTestReport(List<OmexTestCase> testCases, List<OmexExecSummary> execSu
                 continue;
             }
             FailureType failureType = (testCase.known_failure_type != null) ? testCase.known_failure_type : FailureType.UNCATETORIZED_FAULT;
-            failureTypeCounts.put(failureType, failureTypeCounts.getOrDefault(failureType, 0) + 1);
+            this.historicalFailureTypeCounts.put(failureType, this.historicalFailureTypeCounts.getOrDefault(failureType, 0) + 1);
         }
 
         for (OmexExecSummary execSummary : execSummaries) {
+            // Collect Failure Types
+            if (execSummary.status == OmexExecSummary.ActualStatus.FAILED){
+                FailureType failureType = (execSummary.failure_type != null) ? execSummary.failure_type : FailureType.UNCATETORIZED_FAULT;
+                this.currentFailureTypeCounts.put(failureType, this.currentFailureTypeCounts.getOrDefault(failureType, 0) + 1);
+            }
+
+            // find matching test case
             List<OmexTestCase> matchingTestCases = OmexTestingDatabase.queryOmexTestCase(testCases, Paths.get(execSummary.file_path), commonPrefix);
             if (matchingTestCases.isEmpty()) {
                 System.out.println("No test case found for: " + execSummary.file_path);
@@ -69,6 +78,8 @@ public OmexTestReport(List<OmexTestCase> testCases, List<OmexExecSummary> execSu
             }
             OmexTestCase matchingTestCase = matchingTestCases.get(0);
             unmatchedTestCases.remove(matchingTestCase);
+
+            // Compare Historical to Current
             OmexTestCase updatedTestCase = null;
             if (execSummary.status == OmexExecSummary.ActualStatus.PASSED){
                 if (matchingTestCase.known_status == OmexTestCase.Status.FAIL || matchingTestCase.known_status == OmexTestCase.Status.SKIP || matchingTestCase.known_status == null) {
@@ -150,60 +161,321 @@ public String toYaml() {
     }
 
     public String toMarkdown() throws JsonProcessingException {
-        ObjectMapper objectMapper = new ObjectMapper();
+        MarkdownTable resultsStatistics = OmexTestReport.generateResultStatistics(this.statistics);
+        MarkdownTable errorTypeStatistics = OmexTestReport.generateErrorTypeStatistics(this.historicalFailureTypeCounts, this.currentFailureTypeCounts);
+        MarkdownTable incomparableStatistics = OmexTestReport.generateIncomparableStatistics(this.statistics, this.unmatchedTestCases, this.unmatchedExecSummaries);
+        MarkdownTable unmatchedResultStatistics = OmexTestReport.generateUnmatchedResultStatistics(this.testCaseChanges);
+
         StringBuilder sb = new StringBuilder();
         sb.append("# Test Report\n");
-        int testCasesFailed = statistics.testCaseFailCount;
-        int testCasesPassed = statistics.testCasePassCount;
-        int testCasesUnknown = statistics.testCaseCount - testCasesFailed - testCasesPassed;
-        sb.append("## Historical Test Case Records: "+statistics.testCaseCount+" (KNOWN PASSES = "+statistics.testCasePassCount+", KNOWN FAILURES = "+statistics.testCaseFailCount+", STATUS NOT RECORDED = "+testCasesUnknown).append(")\n");
-        // print failure type counts
-        for (Map.Entry<FailureType,Integer> entry : failureTypeCounts.entrySet()) {
-            sb.append(" - ").append(entry.getKey()).append(": ").append(entry.getValue()).append("\n");
-        }
-        sb.append("## New Test Executions: "+statistics.totalExecutions+" (PASSES = "+statistics.passedExecutionsCount+", FAILURES = "+statistics.failedExecutionsCount).append(")\n");
-
-        if (statistics.unmatchedTestCaseCount > 0) {
-            sb.append("## Historical Test Case Records without matching Test Executions: ").append(statistics.unmatchedTestCaseCount).append("\n");
-            int count = 0;
-            for (OmexTestCase testCase : unmatchedTestCases) {
-                sb.append(" - ").append(testCase.test_collection).append(" : ").append(testCase.file_path).append("\n");
-                count++;
-                if (count > 10) {
-                    sb.append(" - ...").append("\n");
-                    break;
+        sb.append("## Results Statistics:\n");
+        sb.append(resultsStatistics.getMarkdownTable()).append("\n");
+        sb.append("## Error Type Statistics:\n");
+        sb.append(errorTypeStatistics.getMarkdownTable()).append("\n");
+
+        if (this.statistics.unmatchedExecutionsCount > 0 || this.statistics.unmatchedTestCaseCount > 0 )
+            sb.append("## Incomparable Results Statistics:\n").append(incomparableStatistics.getMarkdownTable()).append("\n");
+        if (!this.testCaseChanges.isEmpty())
+            sb.append("## Unmatched Results Statistics:\n").append(unmatchedResultStatistics.getMarkdownTable()).append("\n");
+        return sb.toString();
+    }
+
+    private static MarkdownTable generateResultStatistics(OmexTestStatistics statistics){
+        MarkdownTable resultStatistics = new MarkdownTable();
+        resultStatistics.resizeRowAndColumnLabels(4, 2);
+        resultStatistics.setColumnTitle(0, "Historical");
+        resultStatistics.setColumnTitle(1, "Current");
+        resultStatistics.setRowTitle(0, "Passes");
+        resultStatistics.setRowTitle(1, "Failures");
+        resultStatistics.setRowTitle(2, "Unknown");
+        resultStatistics.setRowTitle(3, "Total");
+        resultStatistics.setTableValue(0,0, statistics.testCasePassCount);
+        resultStatistics.setTableValue(1,0, statistics.testCaseFailCount);
+        resultStatistics.setTableValue(2,0, statistics.testCaseCount - (statistics.testCaseFailCount + statistics.testCasePassCount));
+        resultStatistics.setTableValue(3,0, statistics.testCaseCount);
+        resultStatistics.setTableValue(0,1, statistics.passedExecutionsCount);
+        resultStatistics.setTableValue(1,1, statistics.failedExecutionsCount);
+        resultStatistics.setTableValue(2,1, "---");
+        resultStatistics.setTableValue(3,1, statistics.totalExecutions);
+        return resultStatistics;
+    }
+
+    private static MarkdownTable generateErrorTypeStatistics(Map<FailureType, Integer> historicalFailures,
+                                                             Map<FailureType, Integer> currentFailures){
+        MarkdownTable errorTypeStatistics = new MarkdownTable();
+        List<Pair<FailureType, Pair<Integer, Integer>>> failureTypeList = OmexTestReport.getFailureTypePairings(historicalFailures, currentFailures);
+
+        errorTypeStatistics.resizeRowAndColumnLabels(failureTypeList.size(), 2);
+        errorTypeStatistics.setColumnTitle(0, "Historical");
+        errorTypeStatistics.setColumnTitle(1, "Current");
+        for (int i = 0; i < failureTypeList.size(); i++){
+            Pair<FailureType, Pair<Integer, Integer>> pairing = failureTypeList.get(i);
+            errorTypeStatistics.setRowTitle(i, pairing.one.toString());
+            errorTypeStatistics.setTableValue(i, 0, pairing.two.one);
+            errorTypeStatistics.setTableValue(i, 1, pairing.two.two);
+        }
+        return errorTypeStatistics;
+    }
+
+    private static List<Pair<FailureType, Pair<Integer, Integer>>> getFailureTypePairings(Map<FailureType, Integer> historicalFailures,
+                                                                                          Map<FailureType, Integer> currentFailures){
+        Set<FailureType> encounteredErrorTypes = new HashSet<>(historicalFailures.keySet());
+        encounteredErrorTypes.addAll(currentFailures.keySet());
+        List<Pair<FailureType, Pair<Integer, Integer>>> failureTypeList = new ArrayList<>();
+        for (FailureType type : FailureType.values()){ // We want to
+            if (!encounteredErrorTypes.contains(type)) continue;
+            Pair<Integer, Integer> historicalAndCurrentCount = new Pair<>(
+                    historicalFailures.getOrDefault(type, 0),
+                    currentFailures.getOrDefault(type, 0)
+            );
+            failureTypeList.add(new Pair<>(type, historicalAndCurrentCount));
+        }
+        return failureTypeList;
+    }
+
+    private static MarkdownTable generateIncomparableStatistics(OmexTestStatistics statistics,
+                                                                List<OmexTestCase> unmatchedTestCases,
+                                                                List<OmexExecSummary> unmatchedExecSummaries){
+        MarkdownTable incomparableStatistics = new MarkdownTable();
+        int allUnmatched = statistics.unmatchedTestCaseCount + statistics.unmatchedExecutionsCount;
+        incomparableStatistics.resizeRowAndColumnLabels(allUnmatched, 2);
+        incomparableStatistics.setColumnTitle(0, "Historically Ran");
+        incomparableStatistics.setColumnTitle(1, "Currently Ran");
+        Map<String, Boolean> nameToIfHistoricalMap = new HashMap<>();
+        if (statistics.unmatchedTestCaseCount > 0) for (OmexTestCase testCase : unmatchedTestCases){
+            nameToIfHistoricalMap.put(testCase.file_path, true);
+        }
+        if (statistics.unmatchedExecutionsCount > 0) for (OmexExecSummary execSummary : unmatchedExecSummaries){
+            nameToIfHistoricalMap.put(execSummary.file_path, false);
+        }
+        List<String> unmatchedNames = new ArrayList<>(nameToIfHistoricalMap.keySet());
+        unmatchedNames.sort(Comparator.naturalOrder());
+        for (int i = 0; i < unmatchedNames.size(); i++){
+            String name = unmatchedNames.get(i);
+            incomparableStatistics.setRowTitle(i, name);
+            // We want to put a checkmark in the correct column, based on which actually ran the test
+            int matchedColumn = nameToIfHistoricalMap.get(name) ? 0 : 1;
+            incomparableStatistics.setTableValue(i, matchedColumn, "✔");
+            incomparableStatistics.setTableValue(i, 1 - matchedColumn, "❌");
+        }
+        return incomparableStatistics;
+    }
+
+    private static MarkdownTable generateUnmatchedResultStatistics(List<OmexTestCaseChange> testCaseChanges){
+        MarkdownTable unmatchedResultStatistics = new MarkdownTable();
+        unmatchedResultStatistics.resizeRowAndColumnLabels(testCaseChanges.size(), 2);
+        unmatchedResultStatistics.setColumnTitle(0, "Historical");
+        unmatchedResultStatistics.setColumnTitle(1, "Current");
+        for (int i = 0; i < testCaseChanges.size(); i++){
+            OmexTestCaseChange testCaseChange = testCaseChanges.get(i);
+            unmatchedResultStatistics.setRowTitle(i, testCaseChange.original.file_path);
+
+            String historicalResult = switch (testCaseChange.original.known_status){
+                case PASS:
+                    yield "PASSED";
+                case SKIP:
+                    yield "SKIPPED";
+                case FAIL:
+                    yield String.format("FAILED (%s)", testCaseChange.original.known_failure_type.name());
+            };
+            unmatchedResultStatistics.setTableValue(i, 0, historicalResult);
+
+            String currentResult = switch (testCaseChange.updated.known_status){
+                case PASS:
+                    yield "PASSED";
+                case SKIP:
+                    yield "SKIPPED";
+                case FAIL:
+                    yield String.format("FAILED (%s)", testCaseChange.original.known_failure_type.name());
+            };
+            unmatchedResultStatistics.setTableValue(i, 1, currentResult);
+        }
+        return unmatchedResultStatistics;
+    }
+
+    public static class MarkdownTable {
+        private String[] rowTitles, columnTitles;
+        private String[] values;
+
+        public MarkdownTable(){
+            this.rowTitles = new String[0]; this.columnTitles = new String[0];
+            this.values = new String[0];
+        }
+
+        /**
+         * Retrieves the value at the target coordinate.
+         * Throws IllegalArgumentException if request would cause an IndexOutOfBounds exception.
+         * @param row the row index to look-up
+         * @param column the column index to look-up
+         * @return the value in the table, as a String.
+         */
+        public String getTableValue(int row, int column){
+            if (row >= this.rowTitles.length) throw new IllegalArgumentException(String.format("Row index `%d` is out of bounds (size: %d)", row, this.rowTitles.length));
+            if (column >= this.columnTitles.length) throw new IllegalArgumentException(String.format("Column index `%d` is out of bounds (size: %d)", column, this.columnTitles.length));
+            return this.values[row * this.columnTitles.length + column];
+        }
+
+        /**
+         * Sets the value at the target coordinate. Converts from whatever type provided into a String.
+         * Throws IllegalArgumentException if request would cause an IndexOutOfBounds exception.
+         * @param row the row index to apply to
+         * @param column the column index to apply to
+         * @return the previous value stored at the target coordinate.
+         */
+        public String setTableValue(int row, int column, Object value){
+            // Validate row / column pair
+            if (row >= this.rowTitles.length) throw new IllegalArgumentException(String.format("Row index `%d` is out of bounds (size: %d)", row, this.rowTitles.length));
+            if (column >= this.columnTitles.length) throw new IllegalArgumentException(String.format("Column index `%d` is out of bounds (size: %d)", column, this.columnTitles.length));
+
+            // Maybe save some time.
+            if (Objects.equals(value, this.values[row * this.columnTitles.length + column])) return (String)value;
+            // Convert value to String.
+            String newValue;
+            // Organize by likely-hood of being passed for efficiency
+            if (value instanceof String stringValue) newValue = stringValue;
+            else if (value instanceof Double doubleValue) newValue = Double.toString(doubleValue);
+            else if (value instanceof Integer integerValue) newValue = Integer.toString(integerValue);
+            else if (value instanceof Long longValue) newValue = Long.toString(longValue);
+            else if (value instanceof Float floatValue) newValue = Float.toString(floatValue);
+            else if (value instanceof Boolean booleanValue) newValue = booleanValue ? "true" : "false";
+            else if (value instanceof Byte byteValue) newValue = String.format("0x%02X", byteValue);
+            else if (value instanceof Short shortValue) newValue = Short.toString(shortValue);
+            else if (value instanceof Character charValue) newValue = Character.toString(charValue);
+            else if (value == null) newValue = "<null>";
+            else newValue = value.toString();
+
+            // Do the swap!
+            String oldValue = this.values[row * this.columnTitles.length + column];
+            this.values[row * this.columnTitles.length + column] = newValue;
+            return oldValue;
+        }
+
+        public void setNumRows(int number){
+            if (number == this.rowTitles.length) return;
+            this.resizeRowLabels(number);
+        }
+
+        public void setNumColumns(int number){
+            if (number == this.columnTitles.length) return;
+            this.resizeColumnLabels(number);
+        }
+
+        public void setNumRowsAndColums(int rowNumber, int colNumber){
+            if (rowNumber == this.rowTitles.length && colNumber == this.columnTitles.length) return;
+            this.resizeRowAndColumnLabels(rowNumber, colNumber);
+        }
+
+        public void setRowTitle(int rowIndex, String title) {
+            if (rowIndex >= this.rowTitles.length) this.resizeRowLabels(rowIndex + 1);
+            this.rowTitles[rowIndex] = title;
+        }
+
+        public void setColumnTitle(int colIndex, String title) {
+            if (colIndex >= this.columnTitles.length) this.resizeColumnLabels(colIndex + 1);
+            this.columnTitles[colIndex] = title;
+        }
+
+        public String getMarkdownTable(){
+            Map<Integer, Integer> maxLengthInColumn = new HashMap<>();
+            for (int i = 0; i < this.columnTitles.length + 1; i ++) maxLengthInColumn.put(i, 0);
+            String[][] elementMapping = new String[this.rowTitles.length + 2][this.columnTitles.length + 1];
+            for (int rowIndex = -2; rowIndex < this.rowTitles.length; rowIndex++){
+                for (int colIndex = -1; colIndex < this.columnTitles.length; colIndex++){
+                    if (rowIndex == -2){
+                        if (colIndex == -1){
+                            elementMapping[0][0] = "   "; // We do three spaces to make sure the min-width is 3; needed for the header line!
+                            maxLengthInColumn.put(0, 3);
+                            continue; // Top left should be empty!
+                        }
+                        elementMapping[0][colIndex + 1] = this.columnTitles[colIndex];
+                        if (this.columnTitles[colIndex].length() > maxLengthInColumn.get(colIndex + 1))
+                            maxLengthInColumn.put(colIndex + 1, this.columnTitles[colIndex].length());
+                        continue;
+                    } else if (rowIndex == -1){
+                        elementMapping[1][colIndex + 1] = "-";
+                        // We know the maxLength in any column will be 3 or greater, so don't bother checking...
+                        continue;
+                    }
+                    // Now we process "normal" rows
+                    if (colIndex == -1){
+                        elementMapping[rowIndex + 2][0] = this.rowTitles[rowIndex];
+                        if (this.rowTitles[rowIndex].length() > maxLengthInColumn.get(0))
+                            maxLengthInColumn.put(0, this.rowTitles[rowIndex].length());
+                        continue;
+                    }
+                    // This should be an element in the table!
+                    String tableValue = this.getTableValue(rowIndex, colIndex);
+                    elementMapping[rowIndex + 2][colIndex + 1] = tableValue;
+                    if (tableValue.length() > maxLengthInColumn.get(colIndex + 1))
+                        maxLengthInColumn.put(colIndex + 1, tableValue.length());
                 }
             }
-        } else {
-            sb.append("## All Historical Test Case Records have matching Test Executions\n");
-        }
-
-        if (statistics.unmatchedExecutionsCount > 0) {
-            sb.append("## New Test Executions without matching Historical Test Case Records: ").append(statistics.unmatchedExecutionsCount).append("\n");
-            int count = 0;
-            for (OmexExecSummary execSummary : unmatchedExecSummaries) {
-                sb.append(" - ").append(execSummary.file_path).append("\n");
-                count++;
-                if (count > 10) {
-                    sb.append(" - ...").append("\n");
-                    break;
+            // We've completed our parsing, now let's build the table
+            StringBuilder tableBuilder = new StringBuilder();
+            for (String[] row : elementMapping){
+                tableBuilder.append("|");
+                for (int colIndex = 0; colIndex < row.length; colIndex++){
+                    String rawValue = row[colIndex];
+                    if ("-".equals(rawValue)){
+                        String headerLine = rawValue.repeat(maxLengthInColumn.get(colIndex));
+                        tableBuilder.append(":").append(headerLine).append(":|");
+                        continue;
+                    }
+                    int numRepeats = maxLengthInColumn.get(colIndex) - rawValue.length();
+                    tableBuilder.append(" ").append(rawValue).append(" ".repeat(numRepeats)).append(" |");
                 }
+                tableBuilder.append("\n");
             }
-        } else {
-            sb.append("## All New Test Executions have matching Historical Test Case Records\n");
+            return tableBuilder.toString();
         }
 
-        if (!testCaseChanges.isEmpty()) {
-            sb.append("## New Test Executions which differ from Historical Test Case Records: ").append(statistics.testCaseChangeCount).append("\n");
-            for (OmexTestCaseChange testCaseChange : testCaseChanges) {
-                String diff = "(" + testCaseChange.original.known_status + ":" + testCaseChange.original.known_failure_type + ") -> (" + testCaseChange.updated.known_status + ":" + testCaseChange.updated.known_failure_type + ")";
-                sb.append(" - ").append(testCaseChange.original.file_path).append(": " + diff + " ===New==> `").append(objectMapper.writeValueAsString(testCaseChange.updated)).append("`\n");
-            }
-        } else {
-            sb.append("## No New Test Executions differ from Historical Test Case Records\n");
+        private void resizeRowLabels(int newSize){
+            if (newSize == this.rowTitles.length) return;
+            int oldRowSize = this.rowTitles.length;
+            String[] newRowTitles = new String[newSize];
+            System.arraycopy(this.rowTitles, 0, newRowTitles, 0, java.lang.Math.min(this.rowTitles.length, newSize));
+            this.rowTitles = newRowTitles;
+            scaleTableValues(oldRowSize, this.columnTitles.length); // We've changed table dimensions; need to scale!
         }
 
-        return sb.toString();
+        private void resizeColumnLabels(int newSize){
+            if (newSize == this.columnTitles.length) return;
+            int oldColumnSize = this.columnTitles.length;
+            String[] newColumnTitles = new String[newSize];
+            System.arraycopy(this.columnTitles, 0, newColumnTitles, 0, java.lang.Math.min(this.columnTitles.length, newSize));
+            this.columnTitles = newColumnTitles;
+            scaleTableValues(this.rowTitles.length, oldColumnSize); // We've changed table dimensions; need to scale!
+        }
+
+        private void resizeRowAndColumnLabels(int newRowSize, int newColumnSize){
+            if (newRowSize == this.rowTitles.length && newColumnSize == this.columnTitles.length) return;
+            int oldRowSize = this.rowTitles.length, oldColumnSize = this.columnTitles.length;
+            String[] newRowTitles = new String[newRowSize];
+            String[] newColumnTitles = new String[newColumnSize];
+            System.arraycopy(this.rowTitles, 0, newRowTitles, 0, java.lang.Math.min(this.rowTitles.length, newRowSize));
+            System.arraycopy(this.columnTitles, 0, newColumnTitles, 0, java.lang.Math.min(this.columnTitles.length, newColumnSize));
+            this.rowTitles = newRowTitles;
+            this.columnTitles = newColumnTitles;
+            scaleTableValues(oldRowSize, oldColumnSize); // We've changed table dimensions; need to scale!
+        }
+
+
+        /**
+         * Since we track the table dimensions using the size of the label arrays, this method needs to be called after
+         * label arrays have been modified; which in practice is actually sensible and useful.
+         */
+        private void scaleTableValues(int oldRowLength, int oldColumnLength){
+            String[] newValues = new String[this.rowTitles.length * this.columnTitles.length];
+            if (oldRowLength > this.rowTitles.length || oldColumnLength > this.columnTitles.length)
+                if (logger.isDebugEnabled()) logger.warn("Downscaling of table; some data may be lost!");
+            int safeRowLength = java.lang.Math.min(this.rowTitles.length, oldRowLength);
+            int safeColumnLength = java.lang.Math.min(this.columnTitles.length, oldColumnLength);
+            for (int rowIter = 0; rowIter < safeRowLength; rowIter++){
+                System.arraycopy(this.values, rowIter * oldColumnLength, newValues, rowIter * this.columnTitles.length, safeColumnLength);
+            }
+            this.values = newValues;
+        }
     }
 
+
 }
diff --git a/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestingDatabase.java b/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestingDatabase.java
index f6d5fab47e..ced8e4a0af 100644
--- a/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestingDatabase.java
+++ b/vcell-core/src/main/java/org/vcell/sedml/testsupport/OmexTestingDatabase.java
@@ -26,6 +26,7 @@ public enum TestCollection {
         VCELL_BSTS_VCML(TestDataRepo.vcell, "vcell-cli/src/test/resources/bsts-omex/vcml"),
         VCELL_BSTS_SBML_CORE(TestDataRepo.vcell, "vcell-cli/src/test/resources/bsts-omex/sbml-core"),
         VCELL_BSTS_SYNTHS(TestDataRepo.vcell, "vcell-cli/src/test/resources/bsts-omex/synths"),
+        VCELL_SPATIAL(TestDataRepo.vcell, "vcell-cli/src/test/resources/spatial"),
         SYSBIO_BIOMD(TestDataRepo.sysbio, "omex_files");
 
         public final TestDataRepo repo;
@@ -104,10 +105,7 @@ public static OmexExecSummary summarize(File inputFilePath, Exception exception,
     }
 
     public static FailureType determineFault(Exception caughtException, List<TraceEvent> errorEvents){ // Throwable because Assertion Error
-        String errorMessage = "";
-        if (caughtException != null) {
-            errorMessage = caughtException.getMessage();
-        }
+        String errorMessage = caughtException == null ? "" : caughtException.getMessage();
 
         if (errorMessage.contains("refers to either a non-existent model")) { //"refers to either a non-existent model (invalid SED-ML) or to another model with changes (not supported yet)"
             return FailureType.SEDML_UNSUPPORTED_MODEL_REFERENCE;
@@ -146,35 +144,4 @@ public static FailureType determineFault(Exception caughtException, List<TraceEv
 
         return FailureType.UNCATETORIZED_FAULT;
     }
-
-
-    private FailureType determineFailureType_FROM_QUANT_OMEX_EXEC(Throwable caughtException){ // Throwable because Assertion Error
-        String errorMessage = caughtException.getMessage();
-        if (errorMessage == null) errorMessage = ""; // Prevent nullptr exception
-
-        if (caughtException instanceof Error && caughtException.getCause() != null)
-            errorMessage = caughtException.getCause().getMessage();
-
-        if (errorMessage.contains("refers to either a non-existent model")) { //"refers to either a non-existent model (invalid SED-ML) or to another model with changes (not supported yet)"
-            return FailureType.SEDML_UNSUPPORTED_MODEL_REFERENCE;
-        } else if (errorMessage.contains("System IO encountered a fatal error")){
-            Throwable subException = caughtException.getCause();
-            //String subMessage = (subException == null) ? "" : subException.getMessage();
-            if (subException instanceof FileAlreadyExistsException){
-                return FailureType.HDF5_FILE_ALREADY_EXISTS;
-            }
-        } else if (errorMessage.contains("error while processing outputs: null")){
-            Throwable subException = caughtException.getCause();
-            if (subException instanceof ArrayIndexOutOfBoundsException){
-                return FailureType.ARRAY_INDEX_OUT_OF_BOUNDS;
-            }
-        } else if (errorMessage.contains("nconsistent unit system in SBML model") ||
-                errorMessage.contains("ust be of type")){
-            return FailureType.SEDML_ERRONEOUS_UNIT_SYSTEM;
-        }
-
-        return FailureType.UNCATETORIZED_FAULT;
-    }
-
-
 }

From f12ecc3e1464f5e5ecc6190fa11cbecf24328357 Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Mon, 30 Dec 2024 17:29:30 -0500
Subject: [PATCH 43/45] Syntax updating for clarity

---
 .../java/org/vcell/cli/run/SolverHandler.java | 10 +++++----
 .../run/hdf5/BiosimulationsHdf5Writer.java    |  2 +-
 .../cli/run/hdf5/SpatialResultsConverter.java | 19 ++++++++++++----
 .../org/vcell/cli/run/QuantOmexExecTest.java  |  4 ++--
 .../java/org/vcell/sedml/SEDMLImporter.java   | 22 ++++++++-----------
 5 files changed, 33 insertions(+), 24 deletions(-)

diff --git a/vcell-cli/src/main/java/org/vcell/cli/run/SolverHandler.java b/vcell-cli/src/main/java/org/vcell/cli/run/SolverHandler.java
index 2ee172159d..ba9fb83167 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/run/SolverHandler.java
+++ b/vcell-cli/src/main/java/org/vcell/cli/run/SolverHandler.java
@@ -99,7 +99,6 @@ private static void sanityCheck(VCDocument doc) {
     }
 
     public void initialize(List<BioModel> bioModelList, SedML sedml) throws ExpressionException {
-
 		Set <AbstractTask> topmostTasks = new LinkedHashSet<> ();
         for(BioModel bioModel : bioModelList) {
 			Simulation[] sims = bioModel.getSimulations();
@@ -391,12 +390,15 @@ public void simulateAllTasks(ExternalDocInfo externalDocInfo, SedML sedml, CLIRe
 				}
 
 				for (TempSimulationJob tempSimulationJob : simJobsList) {
-					logger.debug("Initializing simulation job... ");
-					String logTaskMessage = "Initializing simulation job " + tempSimulationJob.getJobIndex() + " ... ";
+					AbstractTask task = tempSimulationToTaskMap.get(tempSimulationJob.getTempSimulation());
+					String paramScanIndex = task instanceof RepeatedTask ? ":" + tempSimulationJob.getJobIndex() : "";
+					String tempSimJobLabel = tempSimulationJob.getSimulationJobID() + tempSimulationJob.getJobIndex();
+					String logTaskMessage = String.format("Initializing simulation job %s (%s%s)...", tempSimJobLabel, task.getId(), paramScanIndex);
+					logger.debug(logTaskMessage);
 					String logTaskError = "";
 					long startTimeTask_ms = System.currentTimeMillis();
 
-					AbstractTask task = tempSimulationToTaskMap.get(tempSimulationJob.getTempSimulation());
+
 
 					SimulationTask simTask;
 					String kisao = "null";
diff --git a/vcell-cli/src/main/java/org/vcell/cli/run/hdf5/BiosimulationsHdf5Writer.java b/vcell-cli/src/main/java/org/vcell/cli/run/hdf5/BiosimulationsHdf5Writer.java
index 7640c9ce7f..9bf5c0135f 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/run/hdf5/BiosimulationsHdf5Writer.java
+++ b/vcell-cli/src/main/java/org/vcell/cli/run/hdf5/BiosimulationsHdf5Writer.java
@@ -36,7 +36,7 @@ public static void writeHdf5(HDF5ExecutionResults hdf5ExecutionResults, File out
         boolean didFail = false;
 
         // Create and open the Hdf5 file
-        logger.info("Creating hdf5 file `reports.h5` in" + outDirForCurrentSedml.getAbsolutePath());
+        logger.info("Creating hdf5 file `reports.h5` in {}", outDirForCurrentSedml.getAbsolutePath());
         File tempFile = new File(outDirForCurrentSedml, "reports.h5");
         try {
             try (WritableHdfFile hdf5File = HdfFile.write(tempFile.toPath())){
diff --git a/vcell-cli/src/main/java/org/vcell/cli/run/hdf5/SpatialResultsConverter.java b/vcell-cli/src/main/java/org/vcell/cli/run/hdf5/SpatialResultsConverter.java
index 1624627afa..aab2a102e8 100644
--- a/vcell-cli/src/main/java/org/vcell/cli/run/hdf5/SpatialResultsConverter.java
+++ b/vcell-cli/src/main/java/org/vcell/cli/run/hdf5/SpatialResultsConverter.java
@@ -43,6 +43,11 @@ public static Map<Report, List<Hdf5SedmlResults>> convertSpatialResultsToSedmlFo
                 BiosimulationLog.instance().updateDatasetStatusYml(sedmlLocation, report.getId(), dataSet.getId(), BiosimulationLog.Status.SUCCEEDED);
             } // end of dataset
 
+            if (dataGenToDataSets.isEmpty()){
+                lg.warn(String.format("Report `%s` does not have any valid spatial component!", report.getId()));
+                continue;
+            }
+
             // Fill out DatasetWrapper Values
             Hdf5SedmlResultsSpatial.SpatialComponents reportMappings = convertedData.dataMapping.get(report);
             Hdf5SedmlResults hdf5DatasetWrapper = new Hdf5SedmlResults();
@@ -89,18 +94,24 @@ private static boolean processDataGenerator(SedML sedml, Report report, DataGene
         Map<AbstractTask, TempSimulation> sedmlTaskToVCellSim = new HashMap<>();
         boolean allVarsValid = true;
         for (Variable variable : dataGenVarList) { // Since we're only doing single variable, this should run once!
-            // Check if it's asking for time (we don't include time with the rest of spatial data).
-            if (variable.isSymbol()) if (VariableSymbol.TIME.equals(variable.getSymbol())) continue;
             // for each variable we recover the task
             AbstractTask completeTask = sedml.getTaskWithId(variable.getReference());
             if (completeTask == null) throw new RuntimeException("Null SedML task encountered");
-            AbstractTask fundamentalTask = SpatialResultsConverter.getBaseTask(completeTask, sedml);
             // from the task we get the sbml model
+            if (!sourceOfTruth.getTaskGroupSet().contains(completeTask.getId())){
+                lg.info(String.format("`%s` is not a spatial task!", completeTask.getId()));
+                allVarsValid = false;
+                continue;
+            }
+            AbstractTask fundamentalTask = SpatialResultsConverter.getBaseTask(completeTask, sedml);
             if (!(sedml.getSimulation(fundamentalTask.getSimulationReference()) instanceof UniformTimeCourse utcSim)){
                 lg.error("only uniform time course simulations are supported");
                 allVarsValid = false;
                 break;
             }
+            // Check if it's asking for time (we don't include time with the rest of spatial data).
+            if (variable.isSymbol()) if (VariableSymbol.TIME.equals(variable.getSymbol())) continue;
+
             sedmlTaskToVCellSim.put(completeTask, completeSedmlTaskToVCellSim.get(completeTask));
         }
         if (allVarsValid){
@@ -159,7 +170,7 @@ private static boolean processDataGenerator(SedML sedml, Report report, DataGene
             if (comps.varsToData == null) comps.varsToData = newVars;
             else comps.varsToData.integrateSimilarLocations(taskVars);
         }
-        return true;
+        return allVarsValid;
     }
 
     private static List<Report> getReports(List<Output> outputs){
diff --git a/vcell-cli/src/test/java/org/vcell/cli/run/QuantOmexExecTest.java b/vcell-cli/src/test/java/org/vcell/cli/run/QuantOmexExecTest.java
index 7694fbfb78..470e921dad 100644
--- a/vcell-cli/src/test/java/org/vcell/cli/run/QuantOmexExecTest.java
+++ b/vcell-cli/src/test/java/org/vcell/cli/run/QuantOmexExecTest.java
@@ -86,8 +86,8 @@ public static void teardown() throws Exception {
 
 	@SuppressWarnings("unused")
 	static Map<String, FailureType> knownFailureTypes() {
-		HashMap<String, FailureType> FailureTypes = new HashMap<>();
-		return FailureTypes;
+		HashMap<String, FailureType> failureTypes = new HashMap<>();
+		return failureTypes;
 	}
 
 	public static Collection<OmexTestCase> testCases() throws IOException {
diff --git a/vcell-core/src/main/java/org/vcell/sedml/SEDMLImporter.java b/vcell-core/src/main/java/org/vcell/sedml/SEDMLImporter.java
index ad2582cd1d..2a27d5a876 100644
--- a/vcell-core/src/main/java/org/vcell/sedml/SEDMLImporter.java
+++ b/vcell-core/src/main/java/org/vcell/sedml/SEDMLImporter.java
@@ -611,10 +611,9 @@ private Variable resolveMathVariable(SimulationContext importedSimContext, Simul
 
 	private void addRepeatedTasks(List<AbstractTask> listOfTasks, Map<String, Simulation> vcSimulations) throws ExpressionException, PropertyVetoException, SEDMLImportException {
 		for (AbstractTask abstractedRepeatedTask : listOfTasks) {
-			if (!(abstractedRepeatedTask instanceof RepeatedTask)) continue;
+			if (!(abstractedRepeatedTask instanceof RepeatedTask repeatedTask)) continue;
 
-			RepeatedTask repeatedTask = (RepeatedTask)abstractedRepeatedTask;
-			if (!repeatedTask.getResetModel() || repeatedTask.getSubTasks().size() != 1) { // if removed, see RunUtils.prepareNonspatialHdf5()
+            if (!repeatedTask.getResetModel() || repeatedTask.getSubTasks().size() != 1) { // if removed, see RunUtils.prepareNonspatialHdf5()
 				logger.error("sequential RepeatedTask not yet supported, task "+SEDMLUtil.getName(abstractedRepeatedTask)+" is being skipped");
 				continue;
 			}
@@ -1097,27 +1096,24 @@ private boolean simulationIsNeededAsOutput(Simulation sim){
 
 		List<Report> reportList = new LinkedList<>();
 		for (Output output : this.sedml.getOutputs()){
-			if (!(output instanceof Report)) continue;
-			reportList.add((Report)output);
+			if (!(output instanceof Report report)) continue;
+			reportList.add(report);
 		}
 
 		for (Report report : reportList){
 			Set<String> neededTaskReferences = new HashSet<>();
 			for (DataSet ds : report.getListOfDataSets()){
 				for (DataGenerator dataGenerator : this.sedml.getDataGenerators()){
-					if (ds.getDataReference().equals(dataGenerator.getId())){
-						for (org.jlibsedml.Variable var : dataGenerator.getListOfVariables()){
-							neededTaskReferences.add(var.getReference());
-						}
+					if (!ds.getDataReference().equals(dataGenerator.getId())) continue;
+					for (org.jlibsedml.Variable var : dataGenerator.getListOfVariables()){
+						neededTaskReferences.add(var.getReference());
 					}
 				}
 			}
 
 			for (AbstractTask task : this.sedml.getTasks()){
-				if (
-						neededTaskReferences.contains(task.getId()) &&
-						task.getId().equals(sim.getImportedTaskID())
-				) return true;
+				if (neededTaskReferences.contains(task.getId()) && task.getId().equals(sim.getImportedTaskID()))
+					return true;
 			}
 
 		}

From 3b47c7e753c2eb6e666bd7979828f3231ef960ce Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Mon, 30 Dec 2024 17:30:47 -0500
Subject: [PATCH 44/45] Converted Spatial Unit test to use new reporting
 framework

---
 .../src/main/resources/test_cases.ndjson      |   2 +
 .../vcell/cli/run/SpatialArchiveFiles.java    |  45 +++--
 .../org/vcell/cli/run/SpatialExecTest.java    | 160 ++++++++----------
 3 files changed, 96 insertions(+), 111 deletions(-)

diff --git a/vcell-cli/src/main/resources/test_cases.ndjson b/vcell-cli/src/main/resources/test_cases.ndjson
index 304ae9e5ab..0849bd437c 100644
--- a/vcell-cli/src/main/resources/test_cases.ndjson
+++ b/vcell-cli/src/main/resources/test_cases.ndjson
@@ -106,6 +106,8 @@
 {"test_collection":"VCELL_BSTS_SYNTHS","file_path":"sedml/SimulatorSupportsSubstitutingAlgorithms/4.execution-should-succeed.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"VCELL_BSTS_SYNTHS","file_path":"sedml/SimulatorSupportsSubstitutingAlgorithms/1.execute-should-fail.omex","should_fail":true,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"VCELL_BSTS_SYNTHS","file_path":"sedml/SimulatorSupportsSubstitutingAlgorithms/2.execution-should-succeed.omex","should_fail":false,"known_status":null,"known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"VCELL_SPATIAL","file_path":"TinySpatialProject","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
+{"test_collection":"VCELL_SPATIAL","file_path":"SimpleSpatialModel","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000001.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000002.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
 {"test_collection":"SYSBIO_BIOMD","file_path":"BIOMD0000000003.omex","should_fail":false,"known_status":"PASS","known_failure_type":null,"known_failure_desc":null}
diff --git a/vcell-cli/src/test/java/org/vcell/cli/run/SpatialArchiveFiles.java b/vcell-cli/src/test/java/org/vcell/cli/run/SpatialArchiveFiles.java
index d0ce3d6f6a..0aec8ea2ba 100644
--- a/vcell-cli/src/test/java/org/vcell/cli/run/SpatialArchiveFiles.java
+++ b/vcell-cli/src/test/java/org/vcell/cli/run/SpatialArchiveFiles.java
@@ -3,51 +3,46 @@
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
+import org.vcell.sedml.testsupport.OmexTestCase;
+import org.vcell.sedml.testsupport.OmexTestingDatabase;
 
 import java.io.FileNotFoundException;
+import java.io.IOException;
 import java.io.InputStream;
 import java.util.Arrays;
+import java.util.List;
 import java.util.function.Predicate;
 
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 
 @Tag("Fast")
 public class SpatialArchiveFiles {
-    private final static String[] allTestFiles = new String[]{
-            "TinySpatialProject.omex",
-            "SimpleSpatialModel.omex"
-    };
 
-    public static String[] getSpatialTestCases() {
-        Predicate<String> testFilter = t -> true;
+    public static OmexTestCase[] getSpatialTestCases() throws IOException {
+        List<OmexTestCase> allTestCases = OmexTestingDatabase.loadOmexTestCases();
+        Predicate<OmexTestCase> testFilter = t -> t.test_collection == OmexTestingDatabase.TestCollection.VCELL_SPATIAL;
 
-        return Arrays.stream(allTestFiles).filter(testFilter).toArray(String[]::new);
+        return allTestCases.stream().filter(testFilter).toArray(OmexTestCase[]::new);
     }
 
-    public static InputStream getSpatialTestCase(String testFile) {
-        if (Arrays.stream(allTestFiles).noneMatch(file -> file.equals(testFile))) {
-            throw new RuntimeException("file not found for VCell Published Test Suite test "+testFile);
-        }
-        try {
-            return getFileFromResourceAsStream(testFile);
-        }catch (FileNotFoundException e){
-            throw new RuntimeException("failed to find test case file '"+testFile+"': " + e.getMessage(), e);
-        }
+    public static InputStream getOmex(OmexTestCase testCase) {
+        String fullPath = testCase.test_collection.pathPrefix + "/" + testCase.file_path;
+        String path = fullPath.substring(fullPath.indexOf("/spatial"));
+        return getFileFromResourceAsStream(path + ".omex");
     }
 
-    private static InputStream getFileFromResourceAsStream(String fileName) throws FileNotFoundException {
-        Class<org.vcell.cli.run.SpatialArchiveFiles> spatialFilesClass = org.vcell.cli.run.SpatialArchiveFiles.class;
-        InputStream nextTestFile = spatialFilesClass.getResourceAsStream("/spatial/" + fileName);
-        if (nextTestFile == null) {
-            throw new FileNotFoundException("file not found! " + fileName);
+    private static InputStream getFileFromResourceAsStream(String path) {
+        InputStream inputStream = SpatialArchiveFiles.class.getResourceAsStream(path);
+        if (inputStream == null) {
+            throw new RuntimeException("file not found! " + path);
         } else {
-            return nextTestFile;
+            return inputStream;
         }
     }
 
     @Test
-    public void test_read_Spatial_omex_file() {
-        InputStream inputStream = getSpatialTestCase(allTestFiles[0]);
-        assertNotNull(inputStream);
+    public void test_read_omex_file() throws IOException {
+        InputStream inputStream = getOmex(getSpatialTestCases()[0]);
+        Assertions.assertTrue(inputStream != null);
     }
 }
diff --git a/vcell-cli/src/test/java/org/vcell/cli/run/SpatialExecTest.java b/vcell-cli/src/test/java/org/vcell/cli/run/SpatialExecTest.java
index c8d91596e2..303e7c166b 100644
--- a/vcell-cli/src/test/java/org/vcell/cli/run/SpatialExecTest.java
+++ b/vcell-cli/src/test/java/org/vcell/cli/run/SpatialExecTest.java
@@ -3,8 +3,10 @@
 import cbit.vcell.mongodb.VCMongoMessage;
 import cbit.vcell.resource.PropertyLoader;
 import org.apache.commons.io.FileUtils;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.LoggerContext;
 import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.Assumptions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -13,6 +15,9 @@
 import org.vcell.cli.CLIRecordable;
 import org.vcell.cli.PythonStreamException;
 import org.vcell.sedml.testsupport.FailureType;
+import org.vcell.sedml.testsupport.OmexTestCase;
+import org.vcell.sedml.testsupport.OmexTestingDatabase;
+import org.vcell.trace.TraceEvent;
 import org.vcell.trace.Tracer;
 import org.vcell.util.VCellUtilityHub;
 
@@ -38,6 +43,13 @@ public static void setup() throws PythonStreamException, IOException {
         PropertyLoader.setProperty(PropertyLoader.cliWorkingDir, new File("../vcell-cli-utils").getAbsolutePath());
         VCMongoMessage.enabled = false;
 
+        LoggerContext config = (LoggerContext)(LogManager.getContext(false));
+        config.getConfiguration().getLoggerConfig(LogManager.getLogger("org.vcell").getName()).setLevel(Level.DEBUG);
+        config.getConfiguration().getLoggerConfig(LogManager.getLogger("cbit").getName()).setLevel(Level.DEBUG);
+        config.updateLoggers();
+        config.getConfiguration().getLoggerConfig(LogManager.getLogger("io.jhdf").getName()).setLevel(Level.WARN);
+        config.updateLoggers();
+
         CLIPythonManager.getInstance().instantiatePythonProcess();
     }
 
@@ -81,114 +93,90 @@ public static void teardown() throws Exception {
 //
 //    }
 
-    static Set<String> blacklistedModels(){
-        HashSet<String> blacklistSet = new HashSet<>();
-        // Hooray! Nothing unsupported yet!
-        return blacklistSet;
+    static Map<String, FailureType> knownFailureTypes() {
+        HashMap<String, FailureType> failureTypes = new HashMap<>();
+        return failureTypes; // Hooray! no accepted failures yet!
     }
 
-    static Map<String, FailureType> knownFaults() {
-        HashMap<String, FailureType> faults = new HashMap<>();
-        // Hooray! no known faults yet!
-        return faults;
+    public static Collection<OmexTestCase> testCases() throws IOException {
+        Predicate<OmexTestCase> projectFilter;
+        projectFilter = (t) -> true; // don't skip any for now.
+
+        return Arrays.stream(SpatialArchiveFiles.getSpatialTestCases()).filter(projectFilter).collect(Collectors.toList());
     }
 
-    public static Collection<String> testCases() {
-        Set<String> modelsToFilter = new HashSet<>(blacklistedModels());
-        Predicate<String> filter = (t) -> !modelsToFilter.contains(t);
+    static class TestRecorder implements CLIRecordable {
 
-        return Arrays.stream(SpatialArchiveFiles.getSpatialTestCases()).filter(filter).collect(Collectors.toList());
+        public TestRecorder() {
+            Tracer.clearTraceEvents();
+        }
+
+        @Override
+        public void writeDetailedErrorList(Exception e, String message) {
+            System.err.println("writeDetailedErrorList(): " + message);
+            Tracer.failure(e, "writeDetailedErrorList(): "+message);
+        }
+        @Override
+        public void writeFullSuccessList(String message) {
+            System.out.println("writeFullSuccessList(): " + message);
+            Tracer.success("writeFullSuccessList(): " + message);
+        }
+        @Override
+        public void writeErrorList(Exception e, String message) {
+            System.err.println("writeErrorList(): " + message);
+            Tracer.failure(e, "writeErrorList(): " + message);
+        }
+        @Override
+        public void writeDetailedResultList(String message) {
+            System.out.println("writeDetailedResultList(): " + message);
+            Tracer.log("writeDetailedResultList(): "+message);
+        }
+        @Override
+        public void writeSpatialList(String message) {
+            System.out.println("writeSpatialList(): " + message);
+            Tracer.log("writeSpatialList(): "+message);
+        }
+        @Override
+        public void writeImportErrorList(Exception e, String message) {
+            System.err.println("writeImportErrorList(): " + message);
+            Tracer.failure(e, "writeImportErrorList(): " + message);
+        }
     }
 
     @ParameterizedTest
     @MethodSource("testCases")
-    public void testSpatialOmex(String testCaseFilename) throws Exception {
-        FailureType knownFault = knownFaults().get(testCaseFilename);
+    public void testSpatialOmex(OmexTestCase testCase) throws Exception {
+        FailureType knownFailureType = testCase.known_failure_type;
         Tracer.clearTraceEvents();
         try {
-            System.out.println("running test " + testCaseFilename);
+            System.out.println("running testCase " + testCase.test_collection + " " + testCase.file_path);
 
             Path outdirPath = Files.createTempDirectory("Spatial_OmexExecTest");
-            CLIRecordable cliRecorder = new CLIRecordable() {
-                @Override
-                public void writeDetailedErrorList(Exception e, String message) {
-                    System.err.println("writeDetailedErrorList(): " + message);
-                    Tracer.failure(e, "detailedErrorList: " + message);
-                }
-                @Override
-                public void writeFullSuccessList(String message) {
-                    System.out.println("writeFullSuccessList(): " + message);
-                    Tracer.success("fullSuccessList: " + message);
-                }
-                @Override
-                public void writeErrorList(Exception e, String message) {
-                    System.err.println("writeErrorList(): " + message);
-                    Tracer.failure(e, "errorList: " + message);
-                }
-                @Override
-                public void writeDetailedResultList(String message) {
-                    System.out.println("writeDetailedResultList(): " + message);
-                    Tracer.log("detailedResultList: " + message);
-                }
-                @Override
-                public void writeSpatialList(String message) {
-                    System.out.println("writeSpatialList(): " + message);
-                    Tracer.log("spatialList: " + message);
-                }
-                @Override
-                public void writeImportErrorList(Exception e, String message) {
-                    System.err.println("writeImportErrorList(): " + message);
-                    Tracer.failure(e, "importErrorList: " + message);
-                }
-            };
-            InputStream omexInputStream = SpatialArchiveFiles.getSpatialTestCase(testCaseFilename);
+            InputStream omexInputStream = SpatialArchiveFiles.getOmex(testCase);
             Path omexFile = Files.createTempFile("Spatial_OmexFile_", "omex");
             FileUtils.copyInputStreamToFile(omexInputStream, omexFile.toFile());
+
+            CLIRecordable cliRecorder = new TestRecorder();
             ExecuteImpl.singleMode(omexFile.toFile(), outdirPath.toFile(), cliRecorder);
 
-            assertFalse(Tracer.hasErrors(), "no exception, but trace errors found: error[0] = "+(Tracer.hasErrors()?(Tracer.getErrors().get(0).message.replace("\n"," | ")):null));
-            if (knownFault != null){
-                throw new RuntimeException("test case passed, but expected " + knownFault.name() + ", remove "
-                        + testCaseFilename + " from known faults");
+            String errorMessage = (Tracer.hasErrors()) ? "failure: '" + Tracer.getErrors().get(0).message.replace("\n", " | ") : "";
+            if (Tracer.hasErrors()) {
+                throw new RuntimeException(errorMessage);
+            }
+            if (knownFailureType != null){
+                throw new RuntimeException("test case passed, but expected " + knownFailureType.name() + ", remove " + testCase.file_path + " from known FailureTypes");
             }
 
-        } catch (Exception | AssertionError e){
-            FailureType fault = this.determineFault(e);
-            if (knownFault == fault) {
+        } catch (Exception e){
+            List<TraceEvent> errorEvents = Tracer.getErrors();
+            FailureType observedFailure = OmexTestingDatabase.determineFault(e, errorEvents);
+            if (knownFailureType == observedFailure) {
                 System.err.println("Expected error: " + e.getMessage());
                 return;
             }
 
-            System.err.println("add FAULT." + fault.name() + " to " + testCaseFilename);
-            throw new Exception("Test error: " + testCaseFilename + " failed improperly", e);
-        }
-    }
-
-    private FailureType determineFault(Throwable caughtException){ // Throwable because Assertion Error
-        String errorMessage = caughtException.getMessage();
-        if (errorMessage == null) errorMessage = ""; // Prevent nullptr exception
-
-        if (caughtException instanceof Error && caughtException.getCause() != null)
-            errorMessage = caughtException.getCause().getMessage();
-
-        if (errorMessage.contains("refers to either a non-existent model")) { //"refers to either a non-existent model (invalid SED-ML) or to another model with changes (not supported yet)"
-            return FailureType.SEDML_UNSUPPORTED_MODEL_REFERENCE;
-        } else if (errorMessage.contains("System IO encountered a fatal error")){
-            Throwable subException = caughtException.getCause();
-            //String subMessage = (subException == null) ? "" : subException.getMessage();
-            if (subException instanceof FileAlreadyExistsException){
-                return FailureType.HDF5_FILE_ALREADY_EXISTS;
-            }
-        } else if (errorMessage.contains("error while processing outputs: null")){
-            Throwable subException = caughtException.getCause();
-            if (subException instanceof ArrayIndexOutOfBoundsException){
-                return FailureType.ARRAY_INDEX_OUT_OF_BOUNDS;
-            }
-        } else if (errorMessage.contains("nconsistent unit system in SBML model") ||
-                errorMessage.contains("ust be of type")){
-            return FailureType.SEDML_ERRONEOUS_UNIT_SYSTEM;
+            System.err.println("add FailureType." + observedFailure.name() + " to " + testCase.file_path);
+            throw new Exception("Test error: " + testCase.file_path + " failed improperly", e);
         }
-
-        return FailureType.UNCATETORIZED_FAULT;
     }
 }

From a12905bd937a8dd39f4bcddcc1403b1a914e29f1 Mon Sep 17 00:00:00 2001
From: Logan Drescher <drescher@uchc.edu>
Date: Tue, 31 Dec 2024 09:32:43 -0500
Subject: [PATCH 45/45] Updated Test to do more than just run.

---
 .../vcell/cli/testsupport/TestOmexTestReport.java   | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestReport.java b/vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestReport.java
index 57e9e9bc87..b0a9e19292 100644
--- a/vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestReport.java
+++ b/vcell-cli/src/test/java/org/vcell/cli/testsupport/TestOmexTestReport.java
@@ -3,7 +3,9 @@
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.vcell.sedml.testsupport.*;
+import org.junit.jupiter.api.Tag;
 
+@Tag("Fast")
 public class TestOmexTestReport {
     @Test
     void checkTableIsMade(){
@@ -28,6 +30,15 @@ void checkTableIsMade(){
         table.setTableValue(3, 0, 17);
         table.setTableValue(3, 1, 42);
         table.setTableValue(3, 2, 3.14159);
-        System.out.println(table.getMarkdownTable());
+
+        String expectedResult = """
+                |                        | Logan     | Jim         | Alex         |
+                |:----------------------:|:---------:|:-----------:|:------------:|
+                | VCell Developer        | true      | true        | false        |
+                | Favorite Byte          | 0x88      | 0xFF        | 0x00         |
+                | Favorite Pizza Topping | Pepperoni | Feta Cheeze | Extra Cheese |
+                | Favorite Number        | 17        | 42          | 3.14159      |
+                """;
+        Assertions.assertEquals(expectedResult, table.getMarkdownTable());
     }
 }