Skip to content

Commit

Permalink
Merge pull request #910 from virtualcell/error-out-on-failed-files
Browse files Browse the repository at this point in the history
  • Loading branch information
@CodeByDrescher
CodeByDrescher authored Jun 15, 2023
2 parents 899f888 + 6cc04dc commit 19e0092
Show file tree
Hide file tree
Showing 4 changed files with 39 additions and 27 deletions.
8 changes: 6 additions & 2 deletions vcell-cli/src/main/java/org/vcell/cli/run/ExecuteCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,14 +67,17 @@ public class ExecuteCommand implements Callable<Integer> {
public Integer call() {
CLIRecorder cliLogger = null;
try {
cliLogger = new CLIRecorder(outputFilePath); // CLILogger will throw an execption if our output dir isn't valid.

Level logLevel = logger.getLevel();
if (!bQuiet && bDebug) {
logLevel = Level.DEBUG;
} else if (bQuiet) {
logLevel = Level.OFF;
}

// CLILogger will throw an exception if our output dir isn't valid.
boolean shouldFlush = this.bKeepFlushingLogs || (this.bForceLogFiles && this.bDebug);
cliLogger = new CLIRecorder(this.outputFilePath, this.bForceLogFiles, shouldFlush);

LoggerContext config = (LoggerContext)(LogManager.getContext(false));
config.getConfiguration().getLoggerConfig(LogManager.getLogger("org.vcell").getName()).setLevel(logLevel);
Expand Down Expand Up @@ -122,7 +125,8 @@ public Integer call() {
}


CLIPythonManager.getInstance().closePythonProcess(); // WARNING: Python will need reinstantiation after this is called
CLIPythonManager.getInstance().closePythonProcess();
// WARNING: Python needs re-instantiation once the above line is called!
return 0;
} catch (Exception e) { ///TODO: Break apart into specific exceptions to maximize logging.
org.apache.logging.log4j.LogManager.getLogger(this.getClass()).error(e.getMessage(), e);
Expand Down
22 changes: 11 additions & 11 deletions vcell-cli/src/main/java/org/vcell/cli/run/ExecuteImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,15 +76,6 @@ public static void batchMode(File dirOfArchivesToProcess, File outputDir, CLIRec
logger.info("Execution finished with no failures");
return;
}
// We had failures.
StringBuilder failedFileString = new StringBuilder();
for (String f : failedFiles){
failedFileString.append(String.format("\t- %s\n", f));
}
String errString = "Execution finished, but the following file(s) failed:\n" + failedFileString;
logger.error(errString);
throw new ExecutionException(errString);

} catch (Exception e) {
StringBuilder failedFileString = new StringBuilder();
logger.fatal("Fatal error caught executing batch mode (ending execution)", e);
Expand All @@ -95,6 +86,14 @@ public static void batchMode(File dirOfArchivesToProcess, File outputDir, CLIRec

throw new RuntimeException("Fatal error caught executing batch mode", e);
}

// We had failures.
StringBuilder failedFileString = new StringBuilder();
for (String f : failedFiles){
failedFileString.append(String.format("\t- %s\n", f));
}
String errString = "Execution finished, but the following file(s) failed:\n" + failedFileString;
logger.error(errString);
}

private static void runSingleExecOmex(File inputFile, File outputDir, CLIRecordable cliLogger, boolean bKeepTempFiles,
Expand Down Expand Up @@ -199,10 +198,11 @@ private static void singleExecOmex(File inputFile, File rootOutputDir, CLIRecord
boolean bKeepTempFiles, boolean bExactMatchOnly, boolean bEncapsulateOutput, boolean bSmallMeshOverride,
boolean bCoerceToDistributed)
throws ExecutionException, PythonStreamException, IOException, InterruptedException, HDF5Exception {
ExecutionJob requestedExecution = new ExecutionJob(inputFile, rootOutputDir, cliRecorder,

ExecutionJob requestedExecution = new ExecutionJob(inputFile, rootOutputDir, cliRecorder, bCoerceToDistributed,
bKeepTempFiles, bExactMatchOnly, bEncapsulateOutput, bSmallMeshOverride);
requestedExecution.preprocessArchive();
requestedExecution.executeArchive(bCoerceToDistributed);
requestedExecution.executeArchive();
requestedExecution.postProcessessArchive();
}

Expand Down
14 changes: 8 additions & 6 deletions vcell-cli/src/main/java/org/vcell/cli/run/ExecutionJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public class ExecutionJob {
private final static Logger logger = LogManager.getLogger(ExecutionJob.class);

private long startTime, endTime;
private boolean bExactMatchOnly, bSmallMeshOverride, bKeepTempFiles;
private boolean bCoerceToDistributed, bExactMatchOnly, bSmallMeshOverride, bKeepTempFiles;
private StringBuilder logOmexMessage;
private String inputFilePath, bioModelBaseName, outputBaseDir, outputDir;
private boolean anySedmlDocumentHasSucceeded = false; // set to true if at least one sedml document run is successful
Expand All @@ -51,7 +51,7 @@ public class ExecutionJob {
* @param bEncapsulateOutput whether to provide a sub-folder for outputs (needed for batch jobs)
* @param bSmallMeshOverride whether to use small meshes or standard meshes.
*/
public ExecutionJob(File inputFile, File rootOutputDir, CLIRecordable cliRecorder,
public ExecutionJob(File inputFile, File rootOutputDir, CLIRecordable cliRecorder, boolean bCoerceToDistributed,
boolean bKeepTempFiles, boolean bExactMatchOnly, boolean bEncapsulateOutput, boolean bSmallMeshOverride){
this();
this.inputFile = inputFile;
Expand All @@ -61,6 +61,7 @@ public ExecutionJob(File inputFile, File rootOutputDir, CLIRecordable cliRecorde
this.bioModelBaseName = FileUtils.getBaseName(inputFile.getName()); // input file without the path
this.outputBaseDir = rootOutputDir.getAbsolutePath();
this.outputDir = bEncapsulateOutput ? Paths.get(outputBaseDir, bioModelBaseName).toString() : outputBaseDir;
this.bCoerceToDistributed = bCoerceToDistributed;
this.bKeepTempFiles = bKeepTempFiles;
this.bExactMatchOnly = bExactMatchOnly;
this.bSmallMeshOverride = bSmallMeshOverride;
Expand Down Expand Up @@ -122,21 +123,22 @@ public void preprocessArchive() throws PythonStreamException, IOException {
* @throws IOException if there are system I/O issues
* @throws ExecutionException if an execution specfic error occurs
*/
public void executeArchive(boolean bCoerceToDistributed) throws HDF5Exception, PythonStreamException, ExecutionException {
public void executeArchive() throws HDF5Exception, PythonStreamException, ExecutionException {
try {
Hdf5DataContainer masterHdf5File = new Hdf5DataContainer();
this.queueAllSedml();

for (String sedmlLocation : this.sedmlLocations){
SedmlJob job = new SedmlJob(sedmlLocation, this.omexHandler, this.inputFile, new File(this.outputBaseDir), this.outputDir, this.sedmlPath2d3d.toString(),
this.cliRecorder, this.bKeepTempFiles, this.bExactMatchOnly, this.bSmallMeshOverride, this.logOmexMessage);
SedmlJob job = new SedmlJob(sedmlLocation, this.omexHandler, this.inputFile, new File(this.outputBaseDir),
this.outputDir, this.sedmlPath2d3d.toString(), this.cliRecorder, this.bCoerceToDistributed,
this.bKeepTempFiles, this.bExactMatchOnly, this.bSmallMeshOverride, this.logOmexMessage);
if (!job.preProcessDoc()){
SedmlStatistics stats = job.getDocStatistics(); // Must process document first
logger.error("Statistics of failed SedML:\n" + stats.toString());
this.anySedmlDocumentHasFailed = true;
}
SedmlStatistics stats = job.getDocStatistics();
boolean hasSucceeded = job.simulateSedml(masterHdf5File, bCoerceToDistributed);
boolean hasSucceeded = job.simulateSedml(masterHdf5File);
this.anySedmlDocumentHasSucceeded |= hasSucceeded;
this.anySedmlDocumentHasFailed &= hasSucceeded;
if (hasSucceeded) logger.info("Processing of SedML succeeded.\n" + stats.toString());
Expand Down
22 changes: 14 additions & 8 deletions vcell-cli/src/main/java/org/vcell/cli/run/SedmlJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@
*/
public class SedmlJob {

private final boolean SHOULD_KEEP_TEMP_FILES, ACCEPT_EXACT_MATCH_ONLY, SHOULD_OVERRIDE_FOR_SMALL_MESH;
private final boolean SHOULD_COERCE_TO_DISTRUBTED, SHOULD_KEEP_TEMP_FILES,
ACCEPT_EXACT_MATCH_ONLY, SHOULD_OVERRIDE_FOR_SMALL_MESH;
private final String SEDML_LOCATION, BIOMODEL_BASE_NAME, RESULTS_DIRECTORY_PATH;
private final StringBuilder LOG_OMEX_MESSAGE;
private final SedmlStatistics DOC_STATISTICS;
Expand Down Expand Up @@ -59,8 +60,10 @@ public class SedmlJob {
* @param bSmallMeshOverride whether to use small meshes or standard meshes.
* @param logOmexMessage a string-builder to contain progress updates of omex execution
*/
public SedmlJob(String sedmlLocation, OmexHandler omexHandler, File masterOmexArchive, File rootOutputDir, String resultsDirPath, String sedmlPath2d3dString,
CLIRecordable cliRecorder, boolean bKeepTempFiles, boolean bExactMatchOnly, boolean bSmallMeshOverride, StringBuilder logOmexMessage){
public SedmlJob(String sedmlLocation, OmexHandler omexHandler, File masterOmexArchive, File rootOutputDir,
String resultsDirPath, String sedmlPath2d3dString, CLIRecordable cliRecorder,
boolean bCoerceToDistributed, boolean bKeepTempFiles, boolean bExactMatchOnly, boolean bSmallMeshOverride,
StringBuilder logOmexMessage){
this.MASTER_OMEX_ARCHIVE = masterOmexArchive;
this.SEDML_LOCATION = sedmlLocation;
this.OUTPUT_DIRECTORY_FOR_CURRENT_SEDML = new File(omexHandler.getOutputPathFromSedml(sedmlLocation));
Expand All @@ -71,6 +74,7 @@ public SedmlJob(String sedmlLocation, OmexHandler omexHandler, File masterOmexAr
this.LOG_OMEX_MESSAGE = logOmexMessage;
this.PLOTS_DIRECTORY = new File(sedmlPath2d3dString);
this.CLI_RECORDER = cliRecorder;
this.SHOULD_COERCE_TO_DISTRUBTED = bCoerceToDistributed;
this.SHOULD_KEEP_TEMP_FILES = bKeepTempFiles;
this.ACCEPT_EXACT_MATCH_ONLY = bExactMatchOnly;
this.SHOULD_OVERRIDE_FOR_SMALL_MESH = bSmallMeshOverride;
Expand Down Expand Up @@ -200,7 +204,7 @@ public boolean preProcessDoc() throws PythonStreamException, InterruptedExceptio
* @throws PythonStreamException if calls to the python-shell instance are not working correctly
* @throws IOException if there are system I/O issues
*/
public boolean simulateSedml(Hdf5DataContainer masterHdf5File, boolean bCoerceToDistributed) throws InterruptedException, PythonStreamException, IOException {
public boolean simulateSedml(Hdf5DataContainer masterHdf5File) throws InterruptedException, PythonStreamException, IOException {
/* temp code to test plot name correctness
String idNamePlotsMap = utils.generateIdNamePlotsMap(sedml, outDirForCurrentSedml);
utils.execPlotOutputSedDoc(inputFile, idNamePlotsMap, this.resultsDirPath);
Expand All @@ -214,7 +218,7 @@ public boolean simulateSedml(Hdf5DataContainer masterHdf5File, boolean bCoerceTo
SolverHandler solverHandler = new SolverHandler();
ExternalDocInfo externalDocInfo = new ExternalDocInfo(this.MASTER_OMEX_ARCHIVE, true);

this.runSimulations(solverHandler, externalDocInfo, bCoerceToDistributed);
this.runSimulations(solverHandler, externalDocInfo);
this.recordRunDetails(solverHandler);
try {
this.processOutputs(solverHandler, masterHdf5File);
Expand All @@ -224,7 +228,7 @@ public boolean simulateSedml(Hdf5DataContainer masterHdf5File, boolean bCoerceTo
return this.evaluateResults();
}

private void runSimulations(SolverHandler solverHandler, ExternalDocInfo externalDocInfo, boolean bCoerceToDistributed) throws IOException {
private void runSimulations(SolverHandler solverHandler, ExternalDocInfo externalDocInfo) throws IOException {
/*
* - Run solvers and make reports; all failures/exceptions are being caught
* - we send both the whole OMEX file and the extracted SEDML file path
Expand All @@ -234,8 +238,10 @@ private void runSimulations(SolverHandler solverHandler, ExternalDocInfo externa
String str = "Building solvers and starting simulation of all tasks... ";
logger.info(str);
this.logDocumentMessage += str;
solverHandler.simulateAllTasks(externalDocInfo, this.sedml, this.CLI_RECORDER, this.OUTPUT_DIRECTORY_FOR_CURRENT_SEDML, this.RESULTS_DIRECTORY_PATH,
this.ROOT_OUTPUT_DIR.getAbsolutePath(), this.SEDML_LOCATION, this.SHOULD_KEEP_TEMP_FILES, this.ACCEPT_EXACT_MATCH_ONLY, this.SHOULD_OVERRIDE_FOR_SMALL_MESH, bCoerceToDistributed);
solverHandler.simulateAllTasks(externalDocInfo, this.sedml, this.CLI_RECORDER,
this.OUTPUT_DIRECTORY_FOR_CURRENT_SEDML, this.RESULTS_DIRECTORY_PATH,
this.ROOT_OUTPUT_DIR.getAbsolutePath(), this.SEDML_LOCATION, this.SHOULD_KEEP_TEMP_FILES,
this.ACCEPT_EXACT_MATCH_ONLY, this.SHOULD_OVERRIDE_FOR_SMALL_MESH, this.SHOULD_COERCE_TO_DISTRUBTED);
} catch (Exception e) {
Throwable currentTierOfException = e;
StringBuilder errorMessage = new StringBuilder();
Expand Down

0 comments on commit 19e0092

Please sign in to comment.