Merge pull request #1402 from virtualcell/omex-error-patterns

unify fault reporting across unit and nightly tests for Omex execution

vcell-cli/src/main/java/org/vcell/cli/CLIStandalone.java

@@ -2,6 +2,7 @@
 
 import org.vcell.cli.biosimulation.BiosimulationsCommand;
 import org.vcell.cli.run.ExecuteCommand;
+import org.vcell.cli.run.ExecuteOmexCommand;
 import org.vcell.cli.sbml.ModelCommand;
 import org.vcell.cli.vcml.*;
 
@@ -22,6 +23,7 @@
         ExportOmexBatchCommand.class,
         ImportOmexCommand.class,
         ImportOmexBatchCommand.class,
+        ExecuteOmexCommand.class,
         ExecuteCommand.class,
         VersionCommand.class,
         ModelCommand.class,

vcell-cli/src/main/java/org/vcell/cli/CliTracer.java

@@ -0,0 +1,31 @@
+package org.vcell.cli;
+
+import org.vcell.trace.Tracer;
+
+public class CliTracer implements CLIRecordable {
+
+    @Override
+    public void writeDetailedErrorList(Exception e, String message) {
+        Tracer.failure(e, "writeDetailedErrorList(): "+message);
+    }
+    @Override
+    public void writeFullSuccessList(String message) {
+        Tracer.success("writeFullSuccessList(): " + message);
+    }
+    @Override
+    public void writeErrorList(Exception e, String message) {
+        Tracer.failure(e, "writeErrorList(): " + message);
+    }
+    @Override
+    public void writeDetailedResultList(String message) {
+        Tracer.log("writeDetailedResultList(): "+message);
+    }
+    @Override
+    public void writeSpatialList(String message) {
+        Tracer.log("writeSpatialList(): "+message);
+    }
+    @Override
+    public void writeImportErrorList(Exception e, String message) {
+        Tracer.failure(e, "writeImportErrorList(): " + message);
+    }
+}

vcell-cli/src/main/java/org/vcell/cli/run/ExecuteImpl.java

@@ -4,7 +4,6 @@
 import cbit.vcell.solver.ode.ODESolverResultSet;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.jlibsedml.XMLException;
 import org.vcell.cli.CLIRecordable;
 import org.vcell.cli.PythonStreamException;
 import org.vcell.cli.exceptions.ExecutionException;

vcell-cli/src/main/java/org/vcell/cli/run/ExecuteOmexCommand.java

@@ -0,0 +1,132 @@
+package org.vcell.cli.run;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.vcell.cli.CLIPythonManager;
+import org.vcell.cli.CLIRecordable;
+import org.vcell.cli.CliTracer;
+import org.vcell.cli.testsupport.OmexExecSummary;
+import org.vcell.cli.testsupport.OmexTestingDatabase;
+import org.vcell.trace.Tracer;
+import org.vcell.util.FileUtils;
+import org.vcell.util.exe.Executable;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.Date;
+import java.util.concurrent.Callable;
+
+@Command(name = "execute-omex", description = "run a single .omex file and log into exec_summary.json and tracer.json")
+public class ExecuteOmexCommand implements Callable<Integer> {
+
+    private final static Logger logger = LogManager.getLogger(ExecuteOmexCommand.class);
+
+    @Option(names = { "-i", "--inputFilePath" }, required = true, description = "Path to a COMBINE/OMEX archive file")
+    private File inputFilePath;
+
+    @Option(names = { "-o", "--outputFilePath"}, required = true, description = "Directory to save outputs")
+    private File outputFilePath;
+
+    @Option(names = {"--keepTempFiles"}, defaultValue = "false")
+    private boolean bKeepTempFiles = false;
+
+    @Option(names = {"--exactMatchOnly"}, defaultValue = "false")
+    private boolean bExactMatchOnly = false;
+
+    @Option(names = "--small-mesh", defaultValue = "false", description = "force spatial simulations to have a very small mesh to make execution faster")
+    private boolean bSmallMeshOverride = false;
+
+    @Option(names = {"--encapsulateOutput"}, defaultValue = "true", description =
+        "VCell will encapsulate output results in a sub directory when executing with a single input archive; has no effect when providing an input directory")
+    private boolean bEncapsulateOutput = true;
+
+    @Option(names = {"--timeout_ms"}, defaultValue = "600000", description = "executable wall clock timeout in milliseconds")
+    // timeout for compiled solver running long jobs; default 10 minutes
+    private long EXECUTABLE_MAX_WALLCLOCK_MILLIS = 10 * 60 * 1000;
+
+    @Option(names = {"-h", "--help"}, description = "show this help message and exit", usageHelp = true)
+    private boolean help = false;
+
+    @Option(names = {"-d", "--debug"}, description = "full application debug mode")
+    private boolean bDebug = false;
+
+    @Option(names = {"-q", "--quiet"}, description = "suppress all console output")
+    private boolean bQuiet = false;
+
+
+    public Integer call() {
+
+        CLIRecordable cliTracer = new CliTracer();
+        try {
+            if (bDebug && bQuiet) {
+                System.err.println("cannot specify both debug and quiet, try --help for usage");
+                return 1;
+            }
+            Level logLevel;
+            if (!bQuiet && bDebug) {
+                logLevel = Level.DEBUG;
+            } else if (bQuiet) {
+                logLevel = Level.OFF;
+            } else {
+                logLevel = logger.getLevel();
+            }
+
+            if (this.inputFilePath.exists() && !this.inputFilePath.isFile()) {
+                System.err.println("Input path must be a file");
+                return 1;
+            }
+            if (this.outputFilePath.exists() && !this.outputFilePath.isDirectory()) {
+                System.err.println("Output path must be a directory");
+                return 1;
+            }
+
+            LoggerContext config = (LoggerContext)(LogManager.getContext(false));
+            config.getConfiguration().getLoggerConfig(LogManager.getLogger("org.vcell").getName()).setLevel(logLevel);
+            config.getConfiguration().getLoggerConfig(LogManager.getLogger("cbit").getName()).setLevel(logLevel);
+            config.updateLoggers();
+
+
+            CLIPythonManager.getInstance().instantiatePythonProcess();
+
+            Executable.setGlobalTimeoutMS(EXECUTABLE_MAX_WALLCLOCK_MILLIS);
+            logger.info("Beginning execution");
+            File tmpDir = Files.createTempDirectory("VCell_CLI_" + Long.toHexString(new Date().getTime())).toFile();
+
+            Tracer.clearTraceEvents();
+            ExecuteImpl.singleMode(inputFilePath, tmpDir, cliTracer, bKeepTempFiles, bExactMatchOnly,
+                            bEncapsulateOutput, bSmallMeshOverride);
+            CLIPythonManager.getInstance().closePythonProcess();
+            // WARNING: Python needs re-instantiation once the above line is called!
+            FileUtils.copyDirectoryContents(tmpDir, outputFilePath, true, null);
+            final OmexExecSummary omexExecSummary;
+            if (Tracer.hasErrors()){
+                omexExecSummary = OmexTestingDatabase.summarize(inputFilePath,(Exception)null,Tracer.getErrors());
+            } else {
+                omexExecSummary = new OmexExecSummary();
+                omexExecSummary.file_path = String.valueOf(inputFilePath);
+                omexExecSummary.status = OmexExecSummary.ActualStatus.PASSED;
+            }
+            new ObjectMapper().writeValue(new File(outputFilePath, "exec_summary.json"), omexExecSummary);
+            new ObjectMapper().writeValue(new File(outputFilePath, "tracer.json"), Tracer.getTraceEvents());
+            return 0;
+        } catch (Exception e) { ///TODO: Break apart into specific exceptions to maximize logging.
+            LogManager.getLogger(this.getClass()).error(e.getMessage(), e);
+            OmexExecSummary omexExecSummary = OmexTestingDatabase.summarize(inputFilePath,e,Tracer.getErrors());
+            try {
+                new ObjectMapper().writeValue(new File(outputFilePath, "exec_summary.json"), omexExecSummary);
+                new ObjectMapper().writeValue(new File(outputFilePath, "tracer.json"), Tracer.getTraceEvents());
+            } catch (IOException ex) {
+                logger.error("Failed to write exec summary and structured logs", ex);
+            }
+            return 1;
+        } finally {
+            logger.debug("Completed all execution");
+        }
+    }
+}

vcell-cli/src/main/java/org/vcell/cli/run/ExecutionJob.java

@@ -6,19 +6,17 @@
 import org.vcell.cli.PythonStreamException;
 import org.vcell.cli.exceptions.ExecutionException;
 import org.vcell.cli.run.hdf5.BiosimulationsHdf5Writer;
+import org.vcell.cli.run.hdf5.BiosimulationsHdfWriterException;
 import org.vcell.cli.run.hdf5.HDF5ExecutionResults;
 import org.vcell.sedml.log.BiosimulationLog;
 import org.vcell.util.FileUtils;
 
 import java.io.File;
 import java.io.IOException;
-import java.math.BigDecimal;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
 
-import org.vcell.cli.run.hdf5.BiosimulationsHdfWriterException;
-
 /**
  * Contains the code necessary to execute an Omex archive in VCell
  */

vcell-cli/src/main/java/org/vcell/cli/testsupport/FailureType.java

@@ -0,0 +1,37 @@
+package org.vcell.cli.testsupport;
+
+@SuppressWarnings("unused")
+public enum FailureType {
+    ARRAY_INDEX_OUT_OF_BOUNDS,
+    BAD_EULER_FORWARD,
+    DIVIDE_BY_ZERO,
+    EXPRESSIONS_DIFFERENT,
+    EXPRESSION_BINDING,
+    GEOMETRY_SPEC_DIFFERENT,
+    HDF5_FILE_ALREADY_EXISTS, // reports.h5 file already exists, so action is blocked. Fixed in branch to be merged in.
+    MATHOVERRIDES_SurfToVol,
+    MATH_GENERATION_FAILURE,
+    MATH_OVERRIDES_A_FUNCTION,
+    MATH_OVERRIDES_INVALID,
+    NULL_POINTER_EXCEPTION,
+    OPERATION_NOT_SUPPORTED, // VCell simply doesn't have the necessary features to run this archive.
+    SBML_IMPORT_FAILURE,
+    SEDML_DIFF_NUMBER_OF_BIOMODELS,
+    SEDML_ERRONEOUS_UNIT_SYSTEM,
+    SEDML_ERROR_CONSTRUCTING_SIMCONTEXT,
+    SEDML_MATH_OVERRIDE_NAMES_DIFFERENT,
+    SEDML_MATH_OVERRIDE_NOT_EQUIVALENT,
+    SEDML_NONSPATIAL_STOCH_HISTOGRAM,
+    SEDML_NO_MODELS_IN_OMEX,
+    SEDML_SIMCONTEXT_NOT_FOUND_BY_NAME,
+    SEDML_SIMULATION_NOT_FOUND_BY_NAME,
+    SEDML_UNSUPPORTED_ENTITY,
+    SEDML_UNSUPPORTED_MODEL_REFERENCE, // Model refers to either a non-existent model (invalid SED-ML) or to another model with changes (not supported yet)
+    SEDML_SBML_LEVEL_CHANGE, // unsupported SBML Level change within the SED-ML
+    TOO_SLOW,
+    UNCATETORIZED_FAULT,
+    UNITS_EXCEPTION,
+    UNKNOWN_IDENTIFIER,
+    SEDML_NO_SEDMLS_TO_EXECUTE, SEDML_PREPROCESS_FAILURE, UNSUPPORTED_NONSPATIAL_STOCH_HISTOGRAM
+
+}

vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexExecSummary.java

@@ -0,0 +1,24 @@
+package org.vcell.cli.testsupport;
+
+public class OmexExecSummary {
+
+    public enum ActualStatus {
+        PASSED,
+        FAILED
+    }
+
+    public String file_path;
+    public ActualStatus status;
+    public FailureType failure_type;
+    public String failure_desc;
+
+    @Override
+    public String toString() {
+        return "OmexExecSummary{" +
+                ", file_path='" + file_path + '\'' +
+                ", status=" + status +
+                ", failure_type=" + failure_type +
+                ", failure_desc="+((failure_desc!=null)?('\'' + failure_desc + '\''):null) +
+                '}';
+    }
+}

vcell-cli/src/main/java/org/vcell/cli/testsupport/OmexTestCase.java

@@ -0,0 +1,34 @@
+package org.vcell.cli.testsupport;
+
+public class OmexTestCase {
+
+    public enum Status {
+        PASS,
+        FAIL,
+        SKIP
+    }
+
+    public OmexTestingDatabase.TestCollection test_collection;
+    public String file_path;
+    public Boolean should_fail;
+    public Status known_status;
+    public FailureType known_failure_type;
+    public String known_failure_desc;
+
+    @Override
+    public String toString() {
+        return "OmexTestCase{" +
+                "test_collection=" + test_collection +
+                ", file_path='" + file_path + '\'' +
+                ", should_fail=" + should_fail +
+                ", known_status=" + known_status +
+                ", known_failure_type=" + known_failure_type +
+                ", known_failure_desc="+((known_failure_desc!=null)?('\'' + known_failure_desc + '\''):null) +
+                '}';
+    }
+
+    public boolean matchFileSuffix(String path) {
+        String fullPath = test_collection.pathPrefix + "/" + file_path;
+        return fullPath.endsWith(path);
+    }
+}