Skip to content

Commit

Permalink
Merged in master changes
Browse files Browse the repository at this point in the history
  • Loading branch information
@CodeByDrescher
CodeByDrescher committed Feb 1, 2024
2 parents 08d3eea + 790866f commit 5447d89
Show file tree
Hide file tree
Showing 8 changed files with 526 additions and 213 deletions.
16 changes: 14 additions & 2 deletions vcell-cli/src/main/java/org/vcell/cli/biosimulation/BiosimulationsCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import org.vcell.cli.CLIPythonManager;
import org.vcell.cli.CLIRecorder;
import org.vcell.cli.run.ExecuteImpl;
import org.vcell.cli.trace.Tracer;
import org.vcell.util.exe.Executable;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
Expand Down Expand Up @@ -94,7 +95,15 @@ public Integer call() {
try {
CLIPythonManager.getInstance().instantiatePythonProcess();
ExecuteImpl.singleMode(ARCHIVE, OUT_DIR, cliRecorder);
return 0; // Does this prevent finally?
if (Tracer.hasErrors()){
if (!bQuiet) {
logger.error("Errors occurred during execution");
Tracer.reportErrors(bDebug);
}
return 1;
}else {
return 0;
}
} finally {
try {
CLIPythonManager.getInstance().closePythonProcess(); // WARNING: Python will need reinstantiation after this is called
Expand All @@ -104,7 +113,10 @@ public Integer call() {
logger.debug("Finished all execution.");
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
if (!bQuiet) {
Tracer.reportErrors(bDebug);
logger.error(e.getMessage(), e);
}
return 1;
}
}
Expand Down
32 changes: 30 additions & 2 deletions vcell-cli/src/main/java/org/vcell/cli/run/ExecuteImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import org.vcell.cli.CLIRecordable;
import org.vcell.cli.PythonStreamException;
import org.vcell.cli.exceptions.ExecutionException;
import org.vcell.cli.trace.Span;
import org.vcell.cli.trace.Tracer;
import org.vcell.util.FileUtils;

import java.io.File;
Expand All @@ -18,6 +20,7 @@
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

public class ExecuteImpl {

Expand Down Expand Up @@ -54,18 +57,26 @@ public static void batchMode(File dirOfArchivesToProcess, File outputDir, CLIRec
String inputFileName = inputFile.getName();
System.out.println("\n\n");
logger.info("Processing " + inputFileName + "(" + inputFile + ")");
Span span = null;
try {
span = Tracer.startSpan(Span.ContextType.OMEX_EXECUTE, inputFileName, Map.of("filename", inputFileName));
if (inputFileName.endsWith("vcml"))
singleExecVcml(inputFile, outputDir, cliLogger);
if (inputFileName.endsWith("omex"))
runSingleExecOmex(inputFile, outputDir, cliLogger,
bKeepTempFiles, bExactMatchOnly, bSmallMeshOverride);
} catch (ExecutionException | RuntimeException | HDF5Exception e){
logger.error("Error caught executing batch mode", e);
Tracer.failure(e, "Error caught executing batch mode");
failedFiles.add(inputFileName);
} catch (Exception e){
Tracer.failure(e, "Error caught executing batch mode");
failedFiles.add(inputFileName);
throw e;
} finally {
if (span != null) {
span.close();
}
}
}
if (failedFiles.isEmpty()){
Expand Down Expand Up @@ -130,8 +141,20 @@ public static void singleMode(File inputFile, File outputDir, CLIRecordable cliL
final boolean bEncapsulateOutput = false;
final boolean bSmallMeshOverride = false;

ExecuteImpl.singleMode(inputFile, outputDir, cliLogger, bKeepTempFiles, bExactMatchOnly,
bEncapsulateOutput, bSmallMeshOverride);
Span span = null;
try {
span = Tracer.startSpan(Span.ContextType.OMEX_EXECUTE, inputFile.getName(), Map.of("filename", inputFile.getName()));
ExecuteImpl.singleMode(inputFile, outputDir, cliLogger, bKeepTempFiles, bExactMatchOnly,
bEncapsulateOutput, bSmallMeshOverride);
} catch (Exception e) {
Tracer.failure(e, "error message");
throw e;
} finally {
if (span != null) {
span.close();
}
}

}

@Deprecated
Expand All @@ -152,6 +175,7 @@ public static void singleExecVcml(File vcmlFile, File outputDir, CLIRecordable c
if (!vcmlFile.getParentFile().getCanonicalPath().contains(outDirForCurrentVcml.getCanonicalPath()))
RunUtils.removeAndMakeDirs(outDirForCurrentVcml);
} catch (Exception e) {
Tracer.failure(e, "Error caught while preparing output directory");
logger.error("Error in creating required directories: " + e.getMessage(), e);
somethingFailed = somethingDidFail();
}
Expand All @@ -167,15 +191,19 @@ public static void singleExecVcml(File vcmlFile, File outputDir, CLIRecordable c
PythonCalls.transposeVcmlCsv(CSVFilePath);
}
} catch (IOException e) {
Tracer.failure(e, "IOException while processing VCML " + vcmlFile.getName());
logger.error("IOException while processing VCML " + vcmlFile.getName(), e);
somethingFailed = somethingDidFail();
} catch (ExpressionException e) {
Tracer.failure(e, "ExpressionException while processing VCML " + vcmlFile.getName());
logger.error("InterruptedException while creating results CSV from VCML " + vcmlFile.getName(), e);
somethingFailed = somethingDidFail();
} catch (InterruptedException e) {
Tracer.failure(e, "InterruptedException while processing VCML " + vcmlFile.getName());
logger.error("InterruptedException while transposing CSV from VCML " + vcmlFile.getName(), e);
somethingFailed = somethingDidFail();
} catch (Exception e) {
Tracer.failure(e, "Unexpected exception while processing VCML " + vcmlFile.getName());
String errorMessage = String.format("Unexpected exception while transposing CSV from VCML <%s>\n%s", vcmlFile.getName(), e.toString());
logger.error(errorMessage, e);
somethingFailed = somethingDidFail();
Expand Down
26 changes: 26 additions & 0 deletions vcell-cli/src/main/java/org/vcell/cli/run/SedmlJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
import org.vcell.cli.exceptions.ExecutionException;
import org.vcell.cli.run.hdf5.Hdf5DataContainer;
import org.vcell.cli.run.hdf5.Hdf5DataExtractor;
import org.vcell.cli.trace.Span;
import org.vcell.cli.trace.Tracer;
import org.vcell.util.DataAccessException;
import org.vcell.util.FileUtils;

Expand Down Expand Up @@ -106,7 +108,9 @@ public boolean preProcessDoc() throws PythonStreamException, InterruptedExceptio
final String SAFE_UNIX_FILE_SEPARATOR = "/";
logger.info("Initializing SED-ML document...");

Span span = null;
try {
span = Tracer.startSpan(Span.ContextType.PROCESSING_SEDML, "preProcessDoc", null);
SedML sedmlFromOmex, sedmlFromPython;
String[] sedmlNameSplit;

Expand Down Expand Up @@ -187,10 +191,15 @@ public boolean preProcessDoc() throws PythonStreamException, InterruptedExceptio
} catch (Exception e) {
String prefix = "SED-ML processing for " + this.SEDML_LOCATION + " failed with error: ";
this.logDocumentError = prefix + e.getMessage();
Tracer.failure(e, prefix);
this.reportProblem(e);
this.somethingFailed = somethingDidFail();
PythonCalls.updateSedmlDocStatusYml(this.SEDML_LOCATION, Status.FAILED, this.RESULTS_DIRECTORY_PATH);
return false;
} finally {
if (span != null) {
span.close();
}
}
return true;
}
Expand Down Expand Up @@ -219,10 +228,17 @@ public boolean simulateSedml(Hdf5DataContainer masterHdf5File) throws Interrupte

this.runSimulations(solverHandler, externalDocInfo);
this.recordRunDetails(solverHandler);
Span span = null;
try {
span = Tracer.startSpan(Span.ContextType.PROCESSING_SIMULATION_OUTPUTS, "processOutputs", null);
this.processOutputs(solverHandler, masterHdf5File);
} catch (Exception e){ // TODO: Make more Fine grain
Tracer.failure(e, "Error processing outputs");
logger.warn("Outputs could not be processed.", e);
} finally {
if (span != null) {
span.close();
}
}
return this.evaluateResults();
}
Expand All @@ -233,7 +249,9 @@ private void runSimulations(SolverHandler solverHandler, ExternalDocInfo externa
* - we send both the whole OMEX file and the extracted SEDML file path
* - XmlHelper code uses two types of resolvers to handle absolute or relative paths
*/
Span span = null;
try {
span = Tracer.startSpan(Span.ContextType.SIMULATIONS_RUN, "runSimulations", null);
String str = "Building solvers and starting simulation of all tasks... ";
logger.info(str);
this.logDocumentMessage += str;
Expand All @@ -244,6 +262,7 @@ private void runSimulations(SolverHandler solverHandler, ExternalDocInfo externa
} catch (Exception e) {
Throwable currentTierOfException = e;
StringBuilder errorMessage = new StringBuilder();
Tracer.failure(e, errorMessage.toString());
this.somethingFailed = somethingDidFail();
while (currentTierOfException != null && !currentTierOfException.equals(currentTierOfException.getCause())){
errorMessage.append(currentTierOfException.getMessage());
Expand All @@ -252,6 +271,10 @@ private void runSimulations(SolverHandler solverHandler, ExternalDocInfo externa
this.logDocumentError = errorMessage.toString(); // probably the hash is empty
logger.error(errorMessage.toString(), e);
// still possible to have some data in the hash, from some task that was successful - that would be partial success
} finally {
if (span != null) {
span.close();
}
}

this.recordRunDetails(solverHandler);
Expand All @@ -266,6 +289,7 @@ private void processOutputs(SolverHandler solverHandler, Hdf5DataContainer maste
if (solverHandler.nonSpatialResults.containsValue(null) || solverHandler.spatialResults.containsValue(null)) { // some tasks failed, but not all
this.somethingFailed = somethingDidFail();
this.logDocumentMessage += "Failed to execute one or more tasks. ";
Tracer.failure(new Exception("Failed to execute one or more tasks in " + this.sedmlName), "Failed to execute one or more tasks in " + this.sedmlName);
logger.info("Failed to execute one or more tasks in " + this.sedmlName);
}

Expand Down Expand Up @@ -301,6 +325,7 @@ private boolean evaluateResults() throws PythonStreamException, InterruptedExcep
this.reportProblem(e);
org.apache.commons.io.FileUtils.deleteDirectory(this.PLOTS_DIRECTORY); // removing temp path generated from python
} catch (IOException ioe){
Tracer.failure(ioe, "Deletion of " + this.PLOTS_DIRECTORY.getName() + " failed");
logger.warn("Deletion of " + this.PLOTS_DIRECTORY.getName() + " failed", ioe);
}
logger.warn(this.logDocumentError);
Expand Down Expand Up @@ -338,6 +363,7 @@ private void generateCSV(SolverHandler solverHandler) throws DataAccessException
}
if (csvReports.isEmpty() || csvReports.containsValue(null)) {
this.somethingFailed = somethingDidFail();
Tracer.failure(new Exception("Failed to generate one or more reports"), "Failed to generate one or more reports");
String msg = "Failed to generate one or more reports. ";
this.logDocumentMessage += msg;
} else {
Expand Down
Loading

0 comments on commit 5447d89

Please sign in to comment.