From 9e183a4debc325a2812d1578b24cd712e4a10993 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Wed, 26 Jun 2024 11:08:30 -0400 Subject: [PATCH 01/24] Organize Simulation Classes Into Different Packages --- .../SimulationDispatcherEngine.java | 39 +++-- .../{ => Control}/SimulationStateMachine.java | 148 +++++++++--------- .../SimulationExecutionStatus.java} | 9 +- .../SimulationJobStatus.java} | 22 ++- .../SimulationQueueEntryStatus.java} | 9 +- .../SimulationStatus.java} | 10 +- .../restq/Simulations/DTO/StatusMessage.java | 5 + .../SimulationResource.java | 11 +- .../{db => Simulations}/SimulationRestDB.java | 16 +- .../restq/Simulations/StatusMessage.java | 7 - .../org/vcell/restq/db/BioModelRestDB.java | 2 +- 11 files changed, 134 insertions(+), 144 deletions(-) rename vcell-rest/src/main/java/org/vcell/restq/Simulations/{ => Control}/SimulationDispatcherEngine.java (71%) rename vcell-rest/src/main/java/org/vcell/restq/Simulations/{ => Control}/SimulationStateMachine.java (66%) rename vcell-rest/src/main/java/org/vcell/restq/Simulations/{SimulationExecutionStatusRecord.java => DTO/SimulationExecutionStatus.java} (70%) rename vcell-rest/src/main/java/org/vcell/restq/Simulations/{SimulationJobStatusRecord.java => DTO/SimulationJobStatus.java} (50%) rename vcell-rest/src/main/java/org/vcell/restq/Simulations/{SimulationQueueEntryStatusRecord.java => DTO/SimulationQueueEntryStatus.java} (60%) rename vcell-rest/src/main/java/org/vcell/restq/Simulations/{SimulationStatusPersistentRecord.java => DTO/SimulationStatus.java} (83%) create mode 100644 vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/StatusMessage.java rename vcell-rest/src/main/java/org/vcell/restq/{handlers => Simulations}/SimulationResource.java (85%) rename vcell-rest/src/main/java/org/vcell/restq/{db => Simulations}/SimulationRestDB.java (90%) delete mode 100644 vcell-rest/src/main/java/org/vcell/restq/Simulations/StatusMessage.java diff --git a/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationDispatcherEngine.java b/vcell-rest/src/main/java/org/vcell/restq/Simulations/Control/SimulationDispatcherEngine.java similarity index 71% rename from vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationDispatcherEngine.java rename to vcell-rest/src/main/java/org/vcell/restq/Simulations/Control/SimulationDispatcherEngine.java index 1a1c9288db..576d52e680 100644 --- a/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationDispatcherEngine.java +++ b/vcell-rest/src/main/java/org/vcell/restq/Simulations/Control/SimulationDispatcherEngine.java @@ -1,12 +1,10 @@ -package org.vcell.restq.Simulations; +package org.vcell.restq.Simulations.Control; import cbit.rmi.event.WorkerEvent; import cbit.vcell.message.VCMessageSession; import cbit.vcell.message.VCMessagingException; -import org.vcell.restq.Simulations.StatusMessage; +import org.vcell.restq.Simulations.DTO.SimulationJobStatus; import cbit.vcell.message.server.dispatcher.SimulationDatabase; -import org.vcell.restq.Simulations.SimulationStateMachine; -import cbit.vcell.server.SimulationJobStatus; import cbit.vcell.server.UpdateSynchronizationException; import cbit.vcell.solver.Simulation; import cbit.vcell.solver.SimulationInfo; @@ -14,6 +12,7 @@ import cbit.vcell.solver.server.SimulationMessage; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.vcell.restq.Simulations.DTO.StatusMessage; import org.vcell.util.DataAccessException; import org.vcell.util.document.KeyValue; import org.vcell.util.document.User; @@ -50,7 +49,7 @@ public SimulationStateMachine getSimulationStateMachine(KeyValue simulationKey, return newStateMachine; } - public void onDispatch(Simulation simulation, SimulationJobStatus simJobStatus, SimulationDatabase simulationDatabase, VCMessageSession dispatcherQueueSession) throws VCMessagingException, DataAccessException, SQLException { + public void onDispatch(Simulation simulation, cbit.vcell.server.SimulationJobStatus simJobStatus, SimulationDatabase simulationDatabase, VCMessageSession dispatcherQueueSession) throws VCMessagingException, DataAccessException, SQLException { KeyValue simulationKey = simJobStatus.getVCSimulationIdentifier().getSimulationKey(); SimulationStateMachine simStateMachine = getSimulationStateMachine(simulationKey, simJobStatus.getJobIndex()); @@ -64,30 +63,30 @@ public ArrayList onStartRequest(VCSimulationIdentifier vcSimID, U boolean isAdmin = Arrays.asList(myUser.getMySpecials()).contains(User.SPECIAL_CLAIM.admins); SimulationInfo simulationInfo = null; - SimulationJobStatus simJobStatus = null; + cbit.vcell.server.SimulationJobStatus simJobStatus = null; ArrayList status = new ArrayList<>(); try { simulationInfo = simulationDatabase.getSimulationInfo(user, simKey); } catch (DataAccessException ex) { if (lg.isWarnEnabled()) lg.warn("Bad simulation " + vcSimID); - simJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, - SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Failed to dispatch simulation: "+ ex.getMessage()), null, null); - status.add(new StatusMessage(SimulationJobStatusRecord.fromSimulationJobStatus(simJobStatus), user.getName(), null, null)); + simJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, + cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Failed to dispatch simulation: "+ ex.getMessage()), null, null); + status.add(new StatusMessage(SimulationJobStatus.fromSimulationJobStatus(simJobStatus), user.getName(), null, null)); return status; } if (simulationInfo == null) { if (lg.isWarnEnabled()) lg.warn("Can't start, simulation [" + vcSimID + "] doesn't exist in database"); - simJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, - SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Can't start, simulation [" + vcSimID + "] doesn't exist"), null, null); - status.add(new StatusMessage(SimulationJobStatusRecord.fromSimulationJobStatus(simJobStatus), user.getName(), null, null)); + simJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, + cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Can't start, simulation [" + vcSimID + "] doesn't exist"), null, null); + status.add(new StatusMessage(SimulationJobStatus.fromSimulationJobStatus(simJobStatus), user.getName(), null, null)); return status; } if (!isAdmin && simulationScanCount > Integer.parseInt(cbit.vcell.resource.PropertyLoader.getRequiredProperty(cbit.vcell.resource.PropertyLoader.maxJobsPerScan))) { if (lg.isWarnEnabled()) lg.warn("Too many simulations (" + simulationScanCount + ") for parameter scan." + vcSimID); - simJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, - SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Too many simulations (" + simulationScanCount + ") for parameter scan."), null, null); - status.add(new StatusMessage(SimulationJobStatusRecord.fromSimulationJobStatus(simJobStatus), user.getName(), null, null)); + simJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, + cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Too many simulations (" + simulationScanCount + ") for parameter scan."), null, null); + status.add(new StatusMessage(SimulationJobStatus.fromSimulationJobStatus(simJobStatus), user.getName(), null, null)); return status; } @@ -106,15 +105,15 @@ public ArrayList onStartRequest(VCSimulationIdentifier vcSimID, U public ArrayList onStopRequest(VCSimulationIdentifier vcSimID, User user, SimulationDatabase simulationDatabase, VCMessageSession session) throws DataAccessException, VCMessagingException, SQLException { KeyValue simKey = vcSimID.getSimulationKey(); - SimulationJobStatus[] allActiveSimJobStatusArray = simulationDatabase.getActiveJobs(VCellServerID.getSystemServerID()); - ArrayList simJobStatusArray = new ArrayList(); - for (SimulationJobStatus activeSimJobStatus : allActiveSimJobStatusArray){ + cbit.vcell.server.SimulationJobStatus[] allActiveSimJobStatusArray = simulationDatabase.getActiveJobs(VCellServerID.getSystemServerID()); + ArrayList simJobStatusArray = new ArrayList(); + for (cbit.vcell.server.SimulationJobStatus activeSimJobStatus : allActiveSimJobStatusArray){ if (activeSimJobStatus.getVCSimulationIdentifier().getSimulationKey().equals(vcSimID.getSimulationKey())){ simJobStatusArray.add(activeSimJobStatus); } } ArrayList stoppedSimulations = new ArrayList<>(); - for (SimulationJobStatus simJobStatus : simJobStatusArray){ + for (cbit.vcell.server.SimulationJobStatus simJobStatus : simJobStatusArray){ SimulationStateMachine simStateMachine = getSimulationStateMachine(simKey, simJobStatus.getJobIndex()); try { stoppedSimulations.add(simStateMachine.onStopRequest(user, simJobStatus, simulationDatabase, session)); @@ -138,7 +137,7 @@ public void onWorkerEvent(WorkerEvent workerEvent, SimulationDatabase simulation } - public void onSystemAbort(SimulationJobStatus jobStatus, String failureMessage, SimulationDatabase simulationDatabase, VCMessageSession session) { + public void onSystemAbort(cbit.vcell.server.SimulationJobStatus jobStatus, String failureMessage, SimulationDatabase simulationDatabase, VCMessageSession session) { try { KeyValue simKey = jobStatus.getVCSimulationIdentifier().getSimulationKey(); int jobIndex = jobStatus.getJobIndex(); diff --git a/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationStateMachine.java b/vcell-rest/src/main/java/org/vcell/restq/Simulations/Control/SimulationStateMachine.java similarity index 66% rename from vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationStateMachine.java rename to vcell-rest/src/main/java/org/vcell/restq/Simulations/Control/SimulationStateMachine.java index 1608b025c9..262b03ac35 100644 --- a/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationStateMachine.java +++ b/vcell-rest/src/main/java/org/vcell/restq/Simulations/Control/SimulationStateMachine.java @@ -1,11 +1,11 @@ -package org.vcell.restq.Simulations; +package org.vcell.restq.Simulations.Control; import cbit.rmi.event.WorkerEvent; import cbit.vcell.field.FieldDataIdentifierSpec; import cbit.vcell.message.*; import cbit.vcell.message.messages.MessageConstants; import cbit.vcell.message.messages.SimulationTaskMessage; -import org.vcell.restq.Simulations.StatusMessage; +import org.vcell.restq.Simulations.DTO.SimulationExecutionStatus; import cbit.vcell.message.messages.WorkerEventMessage; import cbit.vcell.message.server.dispatcher.SimulationDatabase; import cbit.vcell.message.server.htc.HtcProxy; @@ -16,13 +16,15 @@ import cbit.vcell.solver.server.SimulationMessage; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.vcell.restq.Simulations.DTO.SimulationJobStatus; +import org.vcell.restq.Simulations.DTO.SimulationQueueEntryStatus; +import org.vcell.restq.Simulations.DTO.StatusMessage; import org.vcell.util.DataAccessException; import org.vcell.util.document.KeyValue; import org.vcell.util.document.User; import org.vcell.util.document.VCellServerID; import java.sql.SQLException; -import java.time.ZoneId; import java.util.Arrays; import java.util.Date; @@ -97,7 +99,7 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba return; } KeyValue simKey = vcSimDataID.getSimulationKey(); - SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(simKey, jobIndex); + cbit.vcell.server.SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(simKey, jobIndex); if (oldSimulationJobStatus == null){ VCMongoMessage.sendInfo("onWorkerEvent() ignoring WorkerEvent, no current SimulationJobStatus: "+workerEvent.show()); @@ -108,7 +110,7 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba return; } int taskID = oldSimulationJobStatus.getTaskID(); - SimulationJobStatus.SchedulerStatus oldSchedulerStatus = oldSimulationJobStatus.getSchedulerStatus(); + cbit.vcell.server.SimulationJobStatus.SchedulerStatus oldSchedulerStatus = oldSimulationJobStatus.getSchedulerStatus(); // // status information (initialized as if new record) @@ -123,13 +125,13 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba Date submitDate = null; Date queueDate = null; int queuePriority = PRIORITY_DEFAULT; - SimulationJobStatus.SimulationQueueID simQueueID = SimulationJobStatus.SimulationQueueID.QUEUE_ID_WAITING; + cbit.vcell.server.SimulationJobStatus.SimulationQueueID simQueueID = cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_WAITING; // // update using previously stored status (if available). // - SimulationExecutionStatus oldSimExeStatus = oldSimulationJobStatus.getSimulationExecutionStatus(); + cbit.vcell.server.SimulationExecutionStatus oldSimExeStatus = oldSimulationJobStatus.getSimulationExecutionStatus(); if (oldSimExeStatus!=null && oldSimExeStatus.getStartDate()!=null){ startDate = oldSimExeStatus.getStartDate(); } @@ -150,7 +152,7 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba } vcServerID = oldSimulationJobStatus.getServerID(); submitDate = oldSimulationJobStatus.getSubmitDate(); - SimulationQueueEntryStatus oldQueueStatus = oldSimulationJobStatus.getSimulationQueueEntryStatus(); + cbit.vcell.server.SimulationQueueEntryStatus oldQueueStatus = oldSimulationJobStatus.getSimulationQueueEntryStatus(); if (oldQueueStatus!=null && oldQueueStatus.getQueueDate()!=null){ queueDate = oldQueueStatus.getQueueDate(); } @@ -176,7 +178,7 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba } - SimulationJobStatus newJobStatus = null; + cbit.vcell.server.SimulationJobStatus newJobStatus = null; if (workerEvent.isAcceptedEvent()) { // @@ -184,15 +186,15 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba // if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued()) { // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); // new exe status lastUpdateDate = new Date(); startDate = lastUpdateDate; endDate = null; - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.DISPATCHED, + newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.DISPATCHED, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); } @@ -200,35 +202,35 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba // only update database when the job event changes from started to running. The later progress event will not be recorded. if ( oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()) { // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); // new exe status lastUpdateDate = new Date(); if (startDate == null){ startDate = lastUpdateDate; } - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, + newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.RUNNING, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); } } else if (workerEvent.isNewDataEvent()) { if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ - if (!oldSchedulerStatus.isRunning() || simQueueID != SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL || hasData==false){ + if (!oldSchedulerStatus.isRunning() || simQueueID != cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL || hasData==false){ // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); // new exe status if (startDate == null){ startDate = lastUpdateDate; } hasData = true; - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, + newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.RUNNING, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); } } @@ -237,17 +239,17 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ - if (!oldSchedulerStatus.isRunning() || simQueueID != SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL){ + if (!oldSchedulerStatus.isRunning() || simQueueID != cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL){ // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); // new exe status if (startDate == null){ startDate = lastUpdateDate; } - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, + newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.RUNNING, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); }else if (oldSchedulerStatus.isRunning()){ @@ -255,10 +257,10 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba // Date latestUpdate = oldSimExeStatus.getLatestUpdateDate(); // if (System.currentTimeMillis() - latestUpdate.getTime() >= MessageConstants.INTERVAL_PING_SERVER_MS * 3 / 5) { // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, + newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.RUNNING, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); } // } @@ -268,15 +270,15 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba } else if (workerEvent.isCompletedEvent()) { if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); // new exe status endDate = new Date(); hasData = true; - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.COMPLETED, + newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.COMPLETED, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); } @@ -284,30 +286,30 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba } else if (workerEvent.isFailedEvent()) { if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); // new exe status endDate = new Date(); - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.FAILED, + newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); } } else if (workerEvent.isWorkerExitErrorEvent()) { if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); // new exe status endDate = new Date(); - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); SimulationMessage simulationMessage = SimulationMessage.workerFailure("solver stopped unexpectedly, "+workerEventSimulationMessage.getDisplayMessage()); - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.FAILED, + newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, taskID, simulationMessage, newQueueStatus, newExeStatus); } @@ -352,23 +354,23 @@ public synchronized StatusMessage onStartRequest(User user, VCSimulationIdentifi if (!user.equals(vcSimID.getOwner())) { lg.error(user + " is not authorized to start simulation (key=" + simKey + ")"); - SimulationJobStatus simulationJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, 0, null, - SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to start this simulation!"), null, null); - StatusMessage message = new StatusMessage(SimulationJobStatusRecord.fromSimulationJobStatus(simulationJobStatus), user.getName(), null, null); + cbit.vcell.server.SimulationJobStatus simulationJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, 0, null, + cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to start this simulation!"), null, null); + StatusMessage message = new StatusMessage(SimulationJobStatus.fromSimulationJobStatus(simulationJobStatus), user.getName(), null, null); VCMongoMessage.sendInfo("onStartRequest("+vcSimID.getID()+") ignoring start simulation request - wrong user): simID="+vcSimID); return message; } - SimulationJobStatus newJobStatus = saveSimulationStartRequest(vcSimID, jobIndex, simulationDatabase); + cbit.vcell.server.SimulationJobStatus newJobStatus = saveSimulationStartRequest(vcSimID, jobIndex, simulationDatabase); - return new StatusMessage(SimulationJobStatusRecord.fromSimulationJobStatus(newJobStatus), user.getName(), null, null); + return new StatusMessage(SimulationJobStatus.fromSimulationJobStatus(newJobStatus), user.getName(), null, null); } - public static SimulationJobStatus saveSimulationStartRequest(VCSimulationIdentifier vcSimID, int jobIndex, SimulationDatabase simulationDatabase) throws DataAccessException, SQLException { + public static cbit.vcell.server.SimulationJobStatus saveSimulationStartRequest(VCSimulationIdentifier vcSimID, int jobIndex, SimulationDatabase simulationDatabase) throws DataAccessException, SQLException { // // get latest simulation job task (if any). // - SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(vcSimID.getSimulationKey(), jobIndex); + cbit.vcell.server.SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(vcSimID.getSimulationKey(), jobIndex); int oldTaskID = -1; if (oldSimulationJobStatus != null){ oldTaskID = oldSimulationJobStatus.getTaskID(); @@ -391,7 +393,7 @@ public static SimulationJobStatus saveSimulationStartRequest(VCSimulationIdentif Date currentDate = new Date(); // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationJobStatus.SimulationQueueID.QUEUE_ID_WAITING); + cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_WAITING); // new exe status Date lastUpdateDate = new Date(); @@ -401,12 +403,12 @@ public static SimulationJobStatus saveSimulationStartRequest(VCSimulationIdentif HtcJobID htcJobID = null; boolean hasData = false; - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); VCellServerID vcServerID = VCellServerID.getSystemServerID(); Date submitDate = currentDate; - SimulationJobStatus newJobStatus = new SimulationJobStatus(vcServerID, vcSimID, jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.WAITING, + cbit.vcell.server.SimulationJobStatus newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimID, jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.WAITING, newTaskID, SimulationMessage.MESSAGE_JOB_WAITING, newQueueStatus, newExeStatus); simulationDatabase.insertSimulationJobStatus(newJobStatus); @@ -414,7 +416,7 @@ public static SimulationJobStatus saveSimulationStartRequest(VCSimulationIdentif } - public synchronized void onDispatch(Simulation simulation, SimulationJobStatus oldSimulationJobStatus, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, DataAccessException, SQLException { + public synchronized void onDispatch(Simulation simulation, cbit.vcell.server.SimulationJobStatus oldSimulationJobStatus, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, DataAccessException, SQLException { updateSolverProcessTimestamp(); VCSimulationIdentifier vcSimID = oldSimulationJobStatus.getVCSimulationIdentifier(); int taskID = oldSimulationJobStatus.getTaskID(); @@ -444,17 +446,17 @@ public synchronized void onDispatch(Simulation simulation, SimulationJobStatus o HtcProxy.MemLimitResults allowableMemMB = HtcProxy.getMemoryLimit(vcellUserid,simID,solverDescription, requiredMemMB, isPowerUser); - final SimulationJobStatus newSimJobStatus; + final cbit.vcell.server.SimulationJobStatus newSimJobStatus; if (requiredMemMB > allowableMemMB.getMemLimit()) { // // fail the simulation // Date currentDate = new Date(); // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - SimulationExecutionStatus newSimExeStatus = new SimulationExecutionStatus(null, null, new Date(), null, false, null); - newSimJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(),vcSimID,jobIndex, - oldSimulationJobStatus.getSubmitDate(), SimulationJobStatus.SchedulerStatus.FAILED,taskID, + cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + cbit.vcell.server.SimulationExecutionStatus newSimExeStatus = new cbit.vcell.server.SimulationExecutionStatus(null, null, new Date(), null, false, null); + newSimJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(),vcSimID,jobIndex, + oldSimulationJobStatus.getSubmitDate(), cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED,taskID, SimulationMessage.jobFailed("simulation required "+requiredMemMB+"MB of memory, only "+allowableMemMB.getMemLimit()+"MB allowed from "+allowableMemMB.getMemLimitSource()), newQueueStatus,newSimExeStatus); @@ -468,10 +470,10 @@ public synchronized void onDispatch(Simulation simulation, SimulationJobStatus o // dispatch the simulation, new queue status // Date currentDate = new Date(); - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationJobStatus.SimulationQueueID.QUEUE_ID_SIMULATIONJOB); - SimulationExecutionStatus newSimExeStatus = new SimulationExecutionStatus(null, null, new Date(), null, false, null); - newSimJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(),vcSimID,jobIndex, - oldSimulationJobStatus.getSubmitDate(), SimulationJobStatus.SchedulerStatus.DISPATCHED,taskID, + cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_SIMULATIONJOB); + cbit.vcell.server.SimulationExecutionStatus newSimExeStatus = new cbit.vcell.server.SimulationExecutionStatus(null, null, new Date(), null, false, null); + newSimJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(),vcSimID,jobIndex, + oldSimulationJobStatus.getSubmitDate(), cbit.vcell.server.SimulationJobStatus.SchedulerStatus.DISPATCHED,taskID, SimulationMessage.MESSAGE_JOB_DISPATCHED, newQueueStatus,newSimExeStatus); @@ -488,27 +490,27 @@ public synchronized void onDispatch(Simulation simulation, SimulationJobStatus o } - public synchronized StatusMessage onStopRequest(User user, SimulationJobStatus simJobStatus, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, DataAccessException, SQLException { + public synchronized StatusMessage onStopRequest(User user, cbit.vcell.server.SimulationJobStatus simJobStatus, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, DataAccessException, SQLException { updateSolverProcessTimestamp(); if (!user.equals(simJobStatus.getVCSimulationIdentifier().getOwner())) { lg.error(user + " is not authorized to stop simulation (key=" + simKey + ")"); - SimulationJobStatus simulationJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(), simJobStatus.getVCSimulationIdentifier(), 0, null, - SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to stop this simulation!"), null, null); + cbit.vcell.server.SimulationJobStatus simulationJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(), simJobStatus.getVCSimulationIdentifier(), 0, null, + cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to stop this simulation!"), null, null); VCMongoMessage.sendInfo("onStopRequest("+simJobStatus.getVCSimulationIdentifier()+") ignoring stop simulation request - wrong user)"); - return new StatusMessage(SimulationJobStatusRecord.fromSimulationJobStatus(simulationJobStatus), user.getName(), null, null); + return new StatusMessage(SimulationJobStatus.fromSimulationJobStatus(simulationJobStatus), user.getName(), null, null); } // stop latest task if active - SimulationJobStatus.SchedulerStatus schedulerStatus = simJobStatus.getSchedulerStatus(); + cbit.vcell.server.SimulationJobStatus.SchedulerStatus schedulerStatus = simJobStatus.getSchedulerStatus(); int taskID = simJobStatus.getTaskID(); if (schedulerStatus.isActive()){ - SimulationQueueEntryStatus simQueueEntryStatus = simJobStatus.getSimulationQueueEntryStatus(); - SimulationExecutionStatus simExeStatus = simJobStatus.getSimulationExecutionStatus(); - SimulationJobStatus newJobStatus = new SimulationJobStatus(simJobStatus.getServerID(),simJobStatus.getVCSimulationIdentifier(),jobIndex,simJobStatus.getSubmitDate(), - SimulationJobStatus.SchedulerStatus.STOPPED,taskID,SimulationMessage.solverStopped("simulation stopped by user"),simQueueEntryStatus,simExeStatus); + cbit.vcell.server.SimulationQueueEntryStatus simQueueEntryStatus = simJobStatus.getSimulationQueueEntryStatus(); + cbit.vcell.server.SimulationExecutionStatus simExeStatus = simJobStatus.getSimulationExecutionStatus(); + cbit.vcell.server.SimulationJobStatus newJobStatus = new cbit.vcell.server.SimulationJobStatus(simJobStatus.getServerID(),simJobStatus.getVCSimulationIdentifier(),jobIndex,simJobStatus.getSubmitDate(), + cbit.vcell.server.SimulationJobStatus.SchedulerStatus.STOPPED,taskID,SimulationMessage.solverStopped("simulation stopped by user"),simQueueEntryStatus,simExeStatus); // // send stopSimulation to serviceControl topic @@ -525,10 +527,10 @@ public synchronized StatusMessage onStopRequest(User user, SimulationJobStatus s // session.sendTopicMessage(VCellTopic.ServiceControlTopic, msg); if (lg.isTraceEnabled()) lg.trace("send " + MessageConstants.MESSAGE_TYPE_STOPSIMULATION_VALUE + " to " + VCellTopic.ServiceControlTopic.getName() + " topic"); - SimulationJobStatusRecord simulationJobStatusRecord = new SimulationJobStatusRecord( + SimulationJobStatus simulationJobStatusRecord = new SimulationJobStatus( null, new VCSimulationIdentifier(simKey, user), simJobStatus.getSubmitDate().toInstant(), - SimulationJobStatus.SchedulerStatus.STOPPED, simJobStatus.getSimulationMessage(), taskID, - VCellServerID.getSystemServerID().toString(), jobIndex, SimulationExecutionStatusRecord.fromSimulationExecutionStatus(simExeStatus), SimulationQueueEntryStatusRecord.fromStatusRecord(simQueueEntryStatus) + cbit.vcell.server.SimulationJobStatus.SchedulerStatus.STOPPED, simJobStatus.getSimulationMessage(), taskID, + VCellServerID.getSystemServerID().toString(), jobIndex, SimulationExecutionStatus.fromSimulationExecutionStatus(simExeStatus), SimulationQueueEntryStatus.fromStatusRecord(simQueueEntryStatus) ); simulationDatabase.updateSimulationJobStatus(newJobStatus); @@ -538,7 +540,7 @@ null, new VCSimulationIdentifier(simKey, user), simJobStatus.getSubmitDate().toI return null; } - public synchronized void onSystemAbort(SimulationJobStatus oldJobStatus, String failureMessage, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, UpdateSynchronizationException, DataAccessException, SQLException { + public synchronized void onSystemAbort(cbit.vcell.server.SimulationJobStatus oldJobStatus, String failureMessage, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, UpdateSynchronizationException, DataAccessException, SQLException { updateSolverProcessTimestamp(); int taskID = oldJobStatus.getTaskID(); @@ -559,7 +561,7 @@ public synchronized void onSystemAbort(SimulationJobStatus oldJobStatus, String // // update using previously stored status (if available). // - SimulationExecutionStatus oldSimExeStatus = oldJobStatus.getSimulationExecutionStatus(); + cbit.vcell.server.SimulationExecutionStatus oldSimExeStatus = oldJobStatus.getSimulationExecutionStatus(); if (oldSimExeStatus!=null && oldSimExeStatus.getStartDate()!=null){ startDate = oldSimExeStatus.getStartDate(); } @@ -574,7 +576,7 @@ public synchronized void onSystemAbort(SimulationJobStatus oldJobStatus, String } vcServerID = oldJobStatus.getServerID(); submitDate = oldJobStatus.getSubmitDate(); - SimulationQueueEntryStatus oldQueueStatus = oldJobStatus.getSimulationQueueEntryStatus(); + cbit.vcell.server.SimulationQueueEntryStatus oldQueueStatus = oldJobStatus.getSimulationQueueEntryStatus(); if (oldQueueStatus!=null && oldQueueStatus.getQueueDate()!=null){ queueDate = oldQueueStatus.getQueueDate(); } @@ -582,14 +584,14 @@ public synchronized void onSystemAbort(SimulationJobStatus oldJobStatus, String queuePriority = oldQueueStatus.getQueuePriority(); } - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); Date endDate = new Date(); Date lastUpdateDate = new Date(); - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - SimulationJobStatus newJobStatus = new SimulationJobStatus(vcServerID, oldJobStatus.getVCSimulationIdentifier(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.FAILED, + cbit.vcell.server.SimulationJobStatus newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, oldJobStatus.getVCSimulationIdentifier(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, taskID, SimulationMessage.jobFailed(failureMessage), newQueueStatus, newExeStatus); simulationDatabase.updateSimulationJobStatus(newJobStatus); diff --git a/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationExecutionStatusRecord.java b/vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/SimulationExecutionStatus.java similarity index 70% rename from vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationExecutionStatusRecord.java rename to vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/SimulationExecutionStatus.java index c70bbc9017..fddda53732 100644 --- a/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationExecutionStatusRecord.java +++ b/vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/SimulationExecutionStatus.java @@ -1,11 +1,10 @@ -package org.vcell.restq.Simulations; +package org.vcell.restq.Simulations.DTO; import cbit.vcell.server.HtcJobID; -import cbit.vcell.server.SimulationExecutionStatus; import java.time.Instant; -public record SimulationExecutionStatusRecord( +public record SimulationExecutionStatus( Instant fieldStartDate, Instant fieldLatestUpdateDate, Instant fieldEndDate, @@ -14,9 +13,9 @@ public record SimulationExecutionStatusRecord( HtcJobID fieldHtcJobID ) { - public static SimulationExecutionStatusRecord fromSimulationExecutionStatus(SimulationExecutionStatus status) { + public static SimulationExecutionStatus fromSimulationExecutionStatus(cbit.vcell.server.SimulationExecutionStatus status) { if (status == null) {return null;} - return new SimulationExecutionStatusRecord( + return new SimulationExecutionStatus( status.getStartDate() != null ? status.getStartDate().toInstant(): null, status.getLatestUpdateDate() != null ? status.getLatestUpdateDate().toInstant() : null, status.getEndDate() != null ? status.getEndDate().toInstant(): null, diff --git a/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationJobStatusRecord.java b/vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/SimulationJobStatus.java similarity index 50% rename from vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationJobStatusRecord.java rename to vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/SimulationJobStatus.java index b88e7ea86c..3dc299c92c 100644 --- a/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationJobStatusRecord.java +++ b/vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/SimulationJobStatus.java @@ -1,30 +1,26 @@ -package org.vcell.restq.Simulations; +package org.vcell.restq.Simulations.DTO; -import cbit.vcell.server.SimulationExecutionStatus; -import cbit.vcell.server.SimulationJobStatus; -import cbit.vcell.server.SimulationQueueEntryStatus; import cbit.vcell.solver.VCSimulationIdentifier; import cbit.vcell.solver.server.SimulationMessage; import java.time.Instant; -import java.time.ZoneId; -public record SimulationJobStatusRecord( +public record SimulationJobStatus( Instant fieldTimeDataStamp, VCSimulationIdentifier fieldVCSimID, Instant fieldSubmitDate, - SimulationJobStatus.SchedulerStatus fieldSchedulerStatus, + cbit.vcell.server.SimulationJobStatus.SchedulerStatus fieldSchedulerStatus, SimulationMessage fieldSimulationMessage, int fieldTaskID, String fieldServerID, int fieldJobIndex, - SimulationExecutionStatusRecord fieldSimulationExecutionStatus, - SimulationQueueEntryStatusRecord fieldSimulationQueueEntryStatus + SimulationExecutionStatus fieldSimulationExecutionStatus, + SimulationQueueEntryStatus fieldSimulationQueueEntryStatus ) { - public static SimulationJobStatusRecord fromSimulationJobStatus(SimulationJobStatus s) { + public static SimulationJobStatus fromSimulationJobStatus(cbit.vcell.server.SimulationJobStatus s) { if (s == null) {return null;} - return new SimulationJobStatusRecord( + return new SimulationJobStatus( s.getTimeDateStamp() != null ? s.getTimeDateStamp().toInstant(): null, s.getVCSimulationIdentifier(), s.getSubmitDate() != null ? s.getSubmitDate().toInstant(): null, @@ -33,8 +29,8 @@ public static SimulationJobStatusRecord fromSimulationJobStatus(SimulationJobSta s.getTaskID(), s.getServerID().toString(), s.getJobIndex(), - SimulationExecutionStatusRecord.fromSimulationExecutionStatus(s.getSimulationExecutionStatus()), - SimulationQueueEntryStatusRecord.fromStatusRecord(s.getSimulationQueueEntryStatus()) + SimulationExecutionStatus.fromSimulationExecutionStatus(s.getSimulationExecutionStatus()), + SimulationQueueEntryStatus.fromStatusRecord(s.getSimulationQueueEntryStatus()) ); } diff --git a/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationQueueEntryStatusRecord.java b/vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/SimulationQueueEntryStatus.java similarity index 60% rename from vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationQueueEntryStatusRecord.java rename to vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/SimulationQueueEntryStatus.java index 18c2f354f8..670868d42f 100644 --- a/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationQueueEntryStatusRecord.java +++ b/vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/SimulationQueueEntryStatus.java @@ -1,19 +1,18 @@ -package org.vcell.restq.Simulations; +package org.vcell.restq.Simulations.DTO; import cbit.vcell.server.SimulationJobStatus; -import cbit.vcell.server.SimulationQueueEntryStatus; import java.time.Instant; -public record SimulationQueueEntryStatusRecord( +public record SimulationQueueEntryStatus( int fieldQueuePriority, Instant fieldQueueDate, SimulationJobStatus.SimulationQueueID fieldQueueID ) { - public static SimulationQueueEntryStatusRecord fromStatusRecord(SimulationQueueEntryStatus simulationQueueEntryStatus) { + public static SimulationQueueEntryStatus fromStatusRecord(cbit.vcell.server.SimulationQueueEntryStatus simulationQueueEntryStatus) { if (simulationQueueEntryStatus == null) {return null;} - return new SimulationQueueEntryStatusRecord( + return new SimulationQueueEntryStatus( simulationQueueEntryStatus.getQueuePriority(), simulationQueueEntryStatus.getQueueDate().toInstant(), simulationQueueEntryStatus.getQueueID() diff --git a/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationStatusPersistentRecord.java b/vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/SimulationStatus.java similarity index 83% rename from vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationStatusPersistentRecord.java rename to vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/SimulationStatus.java index 99b87336fb..137a85a24a 100644 --- a/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationStatusPersistentRecord.java +++ b/vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/SimulationStatus.java @@ -1,10 +1,8 @@ -package org.vcell.restq.Simulations; +package org.vcell.restq.Simulations.DTO; import cbit.vcell.server.SimulationStatusPersistent; -import java.util.HashMap; - -public record SimulationStatusPersistentRecord( +public record SimulationStatus( Status status, String details, boolean hasData @@ -46,9 +44,9 @@ public static Status statusFromString(String statusString){ } } - public static SimulationStatusPersistentRecord fromSimulationStatusPersistent(SimulationStatusPersistent s) { + public static SimulationStatus fromSimulationStatusPersistent(SimulationStatusPersistent s) { if (s == null) return null; - return new SimulationStatusPersistentRecord( + return new SimulationStatus( Status.statusFromString(s.getStatusString()), s.getDetails(), s.getHasData() diff --git a/vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/StatusMessage.java b/vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/StatusMessage.java new file mode 100644 index 0000000000..e3d75271de --- /dev/null +++ b/vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/StatusMessage.java @@ -0,0 +1,5 @@ +package org.vcell.restq.Simulations.DTO; + +public record StatusMessage(SimulationJobStatus jobStatus, String userName, Double progress, Double timepoint) { + +} diff --git a/vcell-rest/src/main/java/org/vcell/restq/handlers/SimulationResource.java b/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationResource.java similarity index 85% rename from vcell-rest/src/main/java/org/vcell/restq/handlers/SimulationResource.java rename to vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationResource.java index 7bd13411ce..3114a7f5a0 100644 --- a/vcell-rest/src/main/java/org/vcell/restq/handlers/SimulationResource.java +++ b/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationResource.java @@ -1,4 +1,4 @@ -package org.vcell.restq.handlers; +package org.vcell.restq.Simulations; import cbit.vcell.message.VCMessagingException; import io.quarkus.security.identity.SecurityIdentity; @@ -7,9 +7,8 @@ import jakarta.ws.rs.*; import jakarta.ws.rs.core.MediaType; import org.eclipse.microprofile.openapi.annotations.Operation; -import org.vcell.restq.Simulations.SimulationStatusPersistentRecord; -import org.vcell.restq.Simulations.StatusMessage; -import org.vcell.restq.db.SimulationRestDB; +import org.vcell.restq.Simulations.DTO.SimulationStatus; +import org.vcell.restq.Simulations.DTO.StatusMessage; import org.vcell.restq.db.UserRestDB; import org.vcell.util.DataAccessException; import org.vcell.util.document.User; @@ -64,8 +63,8 @@ public ArrayList stopSimulation(@PathParam("simID") String simID) @Path("/{simID}/simulationStatus") @RolesAllowed("user") @Operation(operationId = "getSimulationStatus", summary = "Get the status of simulation running") - public SimulationStatusPersistentRecord getSimulationStatus(@PathParam("simID") String simID, - @QueryParam("bioModelID") String bioModelID, @QueryParam("mathModelID") String mathModelID){ + public SimulationStatus getSimulationStatus(@PathParam("simID") String simID, + @QueryParam("bioModelID") String bioModelID, @QueryParam("mathModelID") String mathModelID){ try { User user = userRestDB.getUserFromIdentity(securityIdentity); return simulationRestDB.getBioModelSimulationStatus(simID, bioModelID, user); diff --git a/vcell-rest/src/main/java/org/vcell/restq/db/SimulationRestDB.java b/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationRestDB.java similarity index 90% rename from vcell-rest/src/main/java/org/vcell/restq/db/SimulationRestDB.java rename to vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationRestDB.java index a02f3188fe..a6922da56e 100644 --- a/vcell-rest/src/main/java/org/vcell/restq/db/SimulationRestDB.java +++ b/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationRestDB.java @@ -1,17 +1,17 @@ -package org.vcell.restq.db; +package org.vcell.restq.Simulations; import cbit.vcell.message.VCMessagingException; import cbit.vcell.message.server.dispatcher.SimulationDatabaseDirect; import cbit.vcell.modeldb.*; -import cbit.vcell.server.SimulationStatusPersistent; -import jakarta.inject.Inject; -import org.vcell.restq.Simulations.SimulationDispatcherEngine; import cbit.vcell.solver.VCSimulationIdentifier; import jakarta.enterprise.context.ApplicationScoped; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vcell.restq.Simulations.SimulationStatusPersistentRecord; -import org.vcell.restq.Simulations.StatusMessage; +import org.vcell.restq.Simulations.Control.SimulationDispatcherEngine; +import org.vcell.restq.Simulations.DTO.SimulationStatus; +import org.vcell.restq.Simulations.DTO.StatusMessage; +import org.vcell.restq.db.AgroalConnectionFactory; +import org.vcell.restq.db.BioModelRestDB; import org.vcell.util.DataAccessException; import org.vcell.util.ObjectNotFoundException; import org.vcell.util.PermissionException; @@ -110,14 +110,14 @@ public ArrayList stopSimulation(String simId, User vcellUser) thr return simulationDispatcherEngine.onStopRequest(vcSimulationIdentifier, vcellUser, simulationDatabaseDirect, null); } - public SimulationStatusPersistentRecord getBioModelSimulationStatus(String simID, String bioModelID, User vcellUser) throws DataAccessException, SQLException { + public SimulationStatus getBioModelSimulationStatus(String simID, String bioModelID, User vcellUser) throws DataAccessException, SQLException { BioModelRep bioModelRep = bioModelRestDB.getBioModelRep(new KeyValue(bioModelID), vcellUser); User[] users = bioModelRep.getGroupUsers(); User owner = bioModelRep.getOwner(); if (!Arrays.stream(users).toList().contains(vcellUser) && !owner.compareEqual(vcellUser)){ throw new PermissionException("Not authorized to access BioModel"); } - return SimulationStatusPersistentRecord.fromSimulationStatusPersistent(databaseServerImpl.getSimulationStatus(new KeyValue(simID))); + return SimulationStatus.fromSimulationStatusPersistent(databaseServerImpl.getSimulationStatus(new KeyValue(simID))); } } diff --git a/vcell-rest/src/main/java/org/vcell/restq/Simulations/StatusMessage.java b/vcell-rest/src/main/java/org/vcell/restq/Simulations/StatusMessage.java deleted file mode 100644 index 32ee143f62..0000000000 --- a/vcell-rest/src/main/java/org/vcell/restq/Simulations/StatusMessage.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.vcell.restq.Simulations; - -import cbit.vcell.server.SimulationJobStatus; - -public record StatusMessage(SimulationJobStatusRecord jobStatus, String userName, Double progress, Double timepoint) { - -} diff --git a/vcell-rest/src/main/java/org/vcell/restq/db/BioModelRestDB.java b/vcell-rest/src/main/java/org/vcell/restq/db/BioModelRestDB.java index 698916ce30..bf8c748efd 100644 --- a/vcell-rest/src/main/java/org/vcell/restq/db/BioModelRestDB.java +++ b/vcell-rest/src/main/java/org/vcell/restq/db/BioModelRestDB.java @@ -6,7 +6,7 @@ import cbit.vcell.xml.XmlParseException; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; -import org.vcell.restq.models.BioModel; +import org.vcell.restq.Simulations.SimulationRestDB; import org.vcell.util.BigString; import org.vcell.util.DataAccessException; import org.vcell.util.ObjectNotFoundException; From d0d30cab2930145037e4e9bea2db65a427dc4434 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Wed, 26 Jun 2024 14:58:49 -0400 Subject: [PATCH 02/24] Test For Different Simulation States --- .../restq/apiclient/SimulationApiTest.java | 73 ++++++++++++++++--- 1 file changed, 64 insertions(+), 9 deletions(-) diff --git a/vcell-rest/src/test/java/org/vcell/restq/apiclient/SimulationApiTest.java b/vcell-rest/src/test/java/org/vcell/restq/apiclient/SimulationApiTest.java index 79afea0326..c06a345855 100644 --- a/vcell-rest/src/test/java/org/vcell/restq/apiclient/SimulationApiTest.java +++ b/vcell-rest/src/test/java/org/vcell/restq/apiclient/SimulationApiTest.java @@ -1,9 +1,18 @@ package org.vcell.restq.apiclient; +import cbit.rmi.event.WorkerEvent; import cbit.vcell.message.VCMessagingException; +import cbit.vcell.message.server.dispatcher.SimulationDatabaseDirect; +import cbit.vcell.messaging.server.SimulationTask; +import cbit.vcell.modeldb.AdminDBTopLevel; import cbit.vcell.modeldb.BioModelRep; +import cbit.vcell.modeldb.DatabaseServerImpl; import cbit.vcell.resource.PropertyLoader; import cbit.vcell.server.SimulationJobStatus; +import cbit.vcell.solver.Simulation; +import cbit.vcell.solver.SimulationJob; +import cbit.vcell.solver.VCSimulationIdentifier; +import cbit.vcell.solver.server.SimulationMessage; import cbit.vcell.solver.server.SimulationMessagePersistent; import cbit.vcell.xml.XmlHelper; import cbit.vcell.xml.XmlParseException; @@ -17,13 +26,14 @@ import org.vcell.restclient.api.SimulationResourceApi; import org.vcell.restclient.model.SchedulerStatus; import org.vcell.restclient.model.Status; -import org.vcell.restq.Simulations.SimulationStatusPersistentRecord; -import org.vcell.restq.Simulations.StatusMessage; +import org.vcell.restq.Simulations.Control.SimulationDispatcherEngine; +import org.vcell.restq.Simulations.DTO.SimulationStatus; +import org.vcell.restq.Simulations.DTO.StatusMessage; import org.vcell.restq.TestEndpointUtils; import org.vcell.restq.config.CDIVCellConfigProvider; import org.vcell.restq.db.AgroalConnectionFactory; import org.vcell.restq.db.BioModelRestDB; -import org.vcell.restq.db.SimulationRestDB; +import org.vcell.restq.Simulations.SimulationRestDB; import org.vcell.util.DataAccessException; import org.vcell.util.document.KeyValue; @@ -51,6 +61,11 @@ public class SimulationApiTest { @Inject SimulationRestDB simulationRestDB; + + AdminDBTopLevel adminDBTopLevel; + SimulationDatabaseDirect simulationDatabaseDirect; + DatabaseServerImpl databaseServer; + private static String previousServerID; private static String previousMongoHost; private static String previousMongoPort; @@ -97,12 +112,20 @@ public void createClients() throws ApiException, XmlParseException, DataAccessEx String testBioModel = XmlHelper.bioModelToXML(TestEndpointUtils.getTestBioModel()); String testBioModelID = bioModelRestDB.saveBioModel(TestEndpointUtils.administratorUser, testBioModel).toString(); bioModelRep = bioModelRestDB.getBioModelRep(new org.vcell.util.document.KeyValue(testBioModelID), TestEndpointUtils.administratorUser); + + databaseServer = new DatabaseServerImpl(agroalConnectionFactory, agroalConnectionFactory.getKeyFactory()); + adminDBTopLevel = new AdminDBTopLevel(agroalConnectionFactory); + simulationDatabaseDirect = new SimulationDatabaseDirect(adminDBTopLevel, databaseServer, false); } @AfterEach public void removeOIDCMappings() throws SQLException, DataAccessException { TestEndpointUtils.removeAllMappings(agroalConnectionFactory); bioModelRestDB.deleteBioModel(TestEndpointUtils.administratorUser, bioModelRep.getBmKey()); + + databaseServer = null; + adminDBTopLevel = null; + simulationDatabaseDirect = null; } @@ -111,7 +134,7 @@ public void removeOIDCMappings() throws SQLException, DataAccessException { public void testDBLayerStartStopAndStatus() throws ApiException, PropertyVetoException, XmlParseException, IOException, DataAccessException, SQLException, VCMessagingException { KeyValue simKey = bioModelRep.getSimKeyList()[0]; - SimulationStatusPersistentRecord statusPersistent = simulationRestDB.getBioModelSimulationStatus(simKey.toString(), bioModelRep.getBmKey().toString(), TestEndpointUtils.administratorUser); + SimulationStatus statusPersistent = simulationRestDB.getBioModelSimulationStatus(simKey.toString(), bioModelRep.getBmKey().toString(), TestEndpointUtils.administratorUser); Assertions.assertNull(statusPersistent); ArrayList statusMessages = simulationRestDB.startSimulation(simKey.toString(), TestEndpointUtils.administratorUser); @@ -124,14 +147,10 @@ public void testDBLayerStartStopAndStatus() throws ApiException, PropertyVetoExc simulationRestDB.stopSimulation(simKey.toString(), TestEndpointUtils.administratorUser); statusPersistent = simulationRestDB.getBioModelSimulationStatus(simKey.toString(), bioModelRep.getBmKey().toString(), TestEndpointUtils.administratorUser); - Assertions.assertEquals(SimulationStatusPersistentRecord.Status.STOPPED.statusDescription, statusPersistent.status().statusDescription); + Assertions.assertEquals(SimulationStatus.Status.STOPPED.statusDescription, statusPersistent.status().statusDescription); } - - - //TODO: Test that ensures the proper request to start or stop a simulation was made to the RPC. A different test - // class should ensure this RPC actually fulfills on its promise. Don't need actual simulations in DB for this @Test public void testStartAndStop() throws Exception { KeyValue simKey = bioModelRep.getSimKeyList()[0]; @@ -155,4 +174,40 @@ public void testStartAndStop() throws Exception { Assertions.assertEquals(SchedulerStatus.STOPPED, stopStatus.get(0).getJobStatus().getFieldSchedulerStatus()); } + + @Test + public void testDifferentSimulationStates() throws Exception{ + KeyValue simKey = bioModelRep.getSimKeyList()[0]; + + SimulationStatus statusPersistent = simulationRestDB.getBioModelSimulationStatus(simKey.toString(), bioModelRep.getBmKey().toString(), TestEndpointUtils.administratorUser); + Assertions.assertNull(statusPersistent); + + ArrayList statusMessages = simulationRestDB.startSimulation(simKey.toString(), TestEndpointUtils.administratorUser); + + Simulation simulation = XmlHelper.XMLToSim(databaseServer.getSimulationXML(TestEndpointUtils.administratorUser, simKey).toString()); + SimulationJob simulationJob = new SimulationJob(simulation, 0, null); + SimulationTask simulationTask = new SimulationTask(simulationJob, 0); + WorkerEvent workerEvent = new WorkerEvent(WorkerEvent.JOB_ACCEPTED, "testService", simulationTask, "testHost", SimulationMessage.MESSAGE_JOB_ACCEPTED); + SimulationDispatcherEngine simulationDispatcherEngine = new SimulationDispatcherEngine(); + + simulationDispatcherEngine.onWorkerEvent(workerEvent, simulationDatabaseDirect, null); + + SimulationStatus simulationStatus = simulationRestDB.getBioModelSimulationStatus(simKey.toString(), bioModelRep.getBmKey().toString(), TestEndpointUtils.administratorUser); + Assertions.assertEquals(SimulationStatus.Status.DISPATCHED, simulationStatus.status()); + + workerEvent = new WorkerEvent(WorkerEvent.JOB_PROGRESS, "testService", new VCSimulationIdentifier(simKey, TestEndpointUtils.administratorUser), + 0, "testHost", 0, .3, 1.3, SimulationMessage.MESSAGE_WORKEREVENT_PROGRESS); + + simulationDispatcherEngine.onWorkerEvent(workerEvent, simulationDatabaseDirect, null); + simulationStatus = simulationRestDB.getBioModelSimulationStatus(simKey.toString(), bioModelRep.getBmKey().toString(), TestEndpointUtils.administratorUser); + Assertions.assertEquals(SimulationStatus.Status.RUNNING, simulationStatus.status()); + + workerEvent = new WorkerEvent(WorkerEvent.JOB_COMPLETED, "testService", new VCSimulationIdentifier(simKey, TestEndpointUtils.administratorUser), + 0, "testHost", 0, 1.0, 10.0, SimulationMessage.MESSAGE_JOB_COMPLETED); + + simulationDispatcherEngine.onWorkerEvent(workerEvent, simulationDatabaseDirect, null); + simulationStatus = simulationRestDB.getBioModelSimulationStatus(simKey.toString(), bioModelRep.getBmKey().toString(), TestEndpointUtils.administratorUser); + Assertions.assertEquals(SimulationStatus.Status.COMPLETED, simulationStatus.status()); + } + } From e011f67c8433c72a30e935ed8e8903d3a683cd01 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Mon, 19 Aug 2024 10:47:26 -0400 Subject: [PATCH 03/24] Factor and Clean Up --- .../Control/SimulationStateMachine.java | 236 ++++++++---------- 1 file changed, 107 insertions(+), 129 deletions(-) diff --git a/vcell-rest/src/main/java/org/vcell/restq/Simulations/Control/SimulationStateMachine.java b/vcell-rest/src/main/java/org/vcell/restq/Simulations/Control/SimulationStateMachine.java index 262b03ac35..0f3a79dc10 100644 --- a/vcell-rest/src/main/java/org/vcell/restq/Simulations/Control/SimulationStateMachine.java +++ b/vcell-rest/src/main/java/org/vcell/restq/Simulations/Control/SimulationStateMachine.java @@ -52,6 +52,44 @@ public class SimulationStateMachine { */ private long solverProcessTimestamp; + private class CurrentState { + + public Date startDate; + public Date lastUpdateDate; + public Date endDate; + public boolean hasData; + public HtcJobID htcJobID; + public String computeHost; + public VCellServerID vcServerID; + public Date submitDate; + public Date queueDate; + public int queuePriority; + public cbit.vcell.server.SimulationJobStatus.SimulationQueueID simQueueID; + + public CurrentState(cbit.vcell.server.SimulationExecutionStatus oldSimExeStatus, + cbit.vcell.server.SimulationQueueEntryStatus oldQueueStatus, + cbit.vcell.server.SimulationJobStatus oldSimulationJobStatus){ + boolean isOldExeNull = oldSimExeStatus == null; + boolean isOldQueueNull = oldQueueStatus == null; + // + // status information (initialized as if new record) + // + startDate = !isOldExeNull && oldSimExeStatus.getStartDate()!=null ? oldSimExeStatus.getStartDate() :null; + lastUpdateDate = !isOldExeNull && oldSimExeStatus.getLatestUpdateDate()!=null ? oldSimExeStatus.getLatestUpdateDate() : null; + endDate = !isOldExeNull && oldSimExeStatus.getEndDate()!=null ? oldSimExeStatus.getEndDate() : null; + hasData = !isOldExeNull && oldSimExeStatus.hasData(); + htcJobID = !isOldExeNull && oldSimExeStatus.getHtcJobID()!=null ? oldSimExeStatus.getHtcJobID() : null; + computeHost = !isOldExeNull && oldSimExeStatus.getComputeHost()!=null ? oldSimExeStatus.getComputeHost() : null; + vcServerID = oldSimulationJobStatus.getServerID(); + submitDate = oldSimulationJobStatus.getSubmitDate(); + queueDate = !isOldQueueNull && oldQueueStatus.getQueueDate() != null ? oldQueueStatus.getQueueDate() : null; + queuePriority = !isOldQueueNull ? oldQueueStatus.getQueuePriority() : PRIORITY_DEFAULT; + simQueueID = !isOldQueueNull && oldQueueStatus.getQueueID()!=null ? oldQueueStatus.getQueueID() : cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_WAITING; + + + } + } + public SimulationStateMachine(KeyValue simKey, int jobIndex){ this.simKey = simKey; this.jobIndex = jobIndex; @@ -83,85 +121,54 @@ long getSolverProcessTimestamp() { return solverProcessTimestamp; } - public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDatabase simulationDatabase, VCMessageSession session) throws DataAccessException, VCMessagingException, SQLException { - updateSolverProcessTimestamp(); - WorkerEventMessage workerEventMessage = new WorkerEventMessage(workerEvent); - VCMongoMessage.sendWorkerEvent(workerEventMessage); - - String userName = workerEvent.getUserName(); // as the filter of the client - int workerEventTaskID = workerEvent.getTaskID(); - - if (lg.isTraceEnabled()) lg.trace("onWorkerEventMessage[" + workerEvent.getEventTypeID() + "," + workerEvent.getSimulationMessage() + "][simid=" + workerEvent.getVCSimulationDataIdentifier() + ",job=" + jobIndex + ",task=" + workerEventTaskID + "]"); - + protected boolean isWorkerEventOkay(WorkerEvent workerEvent, SimulationDatabase simulationDatabase) throws SQLException, DataAccessException { VCSimulationDataIdentifier vcSimDataID = workerEvent.getVCSimulationDataIdentifier(); + int workerEventTaskID = workerEvent.getTaskID(); if (vcSimDataID == null) { VCMongoMessage.sendInfo("onWorkerEvent() ignoring WorkerEvent - no SimID in message): "+workerEvent.show()); - return; + return false; } KeyValue simKey = vcSimDataID.getSimulationKey(); cbit.vcell.server.SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(simKey, jobIndex); if (oldSimulationJobStatus == null){ VCMongoMessage.sendInfo("onWorkerEvent() ignoring WorkerEvent, no current SimulationJobStatus: "+workerEvent.show()); - return; + return false; } - if (oldSimulationJobStatus == null || oldSimulationJobStatus.getSchedulerStatus().isDone() || oldSimulationJobStatus.getTaskID() > workerEventTaskID){ + if (oldSimulationJobStatus.getSchedulerStatus().isDone() || oldSimulationJobStatus.getTaskID() > workerEventTaskID){ VCMongoMessage.sendInfo("onWorkerEvent() ignoring outdated WorkerEvent, (currState="+oldSimulationJobStatus.getSchedulerStatus().getDescription()+"): "+workerEvent.show()); - return; + return false; } - int taskID = oldSimulationJobStatus.getTaskID(); + return true; + } + + protected cbit.vcell.server.SimulationJobStatus produceSimulationJobStatusFromWorkerEvent( + WorkerEvent workerEvent, + cbit.vcell.server.SimulationJobStatus oldSimulationJobStatus){ + + cbit.vcell.server.SimulationExecutionStatus oldSimExeStatus = oldSimulationJobStatus.getSimulationExecutionStatus(); + cbit.vcell.server.SimulationQueueEntryStatus oldQueueStatus = oldSimulationJobStatus.getSimulationQueueEntryStatus(); cbit.vcell.server.SimulationJobStatus.SchedulerStatus oldSchedulerStatus = oldSimulationJobStatus.getSchedulerStatus(); + VCSimulationDataIdentifier vcSimDataID = workerEvent.getVCSimulationDataIdentifier(); - // - // status information (initialized as if new record) - // - Date startDate = null; - Date lastUpdateDate = null; - Date endDate = null; - boolean hasData = false; - HtcJobID htcJobID = null; - String computeHost = null; - VCellServerID vcServerID = VCellServerID.getSystemServerID(); - Date submitDate = null; - Date queueDate = null; - int queuePriority = PRIORITY_DEFAULT; - cbit.vcell.server.SimulationJobStatus.SimulationQueueID simQueueID = cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_WAITING; + int taskID = oldSimulationJobStatus.getTaskID(); + CurrentState currentState = new CurrentState(oldSimExeStatus, oldQueueStatus, oldSimulationJobStatus); // - // update using previously stored status (if available). + // status information (initialized as if new record) // - cbit.vcell.server.SimulationExecutionStatus oldSimExeStatus = oldSimulationJobStatus.getSimulationExecutionStatus(); - if (oldSimExeStatus!=null && oldSimExeStatus.getStartDate()!=null){ - startDate = oldSimExeStatus.getStartDate(); - } - if (oldSimExeStatus!=null && oldSimExeStatus.getLatestUpdateDate()!=null){ - lastUpdateDate = oldSimExeStatus.getLatestUpdateDate(); - } - if (oldSimExeStatus!=null && oldSimExeStatus.getEndDate()!=null){ - endDate = oldSimExeStatus.getEndDate(); - } - if (oldSimExeStatus!=null && oldSimExeStatus.hasData()){ - hasData = true; - } - if (oldSimExeStatus!=null && oldSimExeStatus.getComputeHost()!=null){ - computeHost = oldSimExeStatus.getComputeHost(); - } - if (oldSimExeStatus!=null && oldSimExeStatus.getHtcJobID()!=null){ - htcJobID = oldSimExeStatus.getHtcJobID(); - } - vcServerID = oldSimulationJobStatus.getServerID(); - submitDate = oldSimulationJobStatus.getSubmitDate(); - cbit.vcell.server.SimulationQueueEntryStatus oldQueueStatus = oldSimulationJobStatus.getSimulationQueueEntryStatus(); - if (oldQueueStatus!=null && oldQueueStatus.getQueueDate()!=null){ - queueDate = oldQueueStatus.getQueueDate(); - } - if (oldQueueStatus!=null){ - queuePriority = oldQueueStatus.getQueuePriority(); - } - if (oldQueueStatus!=null && oldQueueStatus.getQueueID()!=null){ - simQueueID = oldQueueStatus.getQueueID(); - } + Date startDate = currentState.startDate; + Date lastUpdateDate = currentState.lastUpdateDate; + Date endDate = currentState.endDate; + boolean hasData = currentState.hasData; + HtcJobID htcJobID = currentState.htcJobID; + String computeHost = currentState.computeHost; + VCellServerID vcServerID = currentState.vcServerID; + Date submitDate = currentState.submitDate; + Date queueDate = currentState.queueDate; + int queuePriority = currentState.queuePriority; + cbit.vcell.server.SimulationJobStatus.SimulationQueueID simQueueID = currentState.simQueueID; // // update using new information from event @@ -254,8 +261,6 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba }else if (oldSchedulerStatus.isRunning()){ if (oldSimExeStatus != null) { -// Date latestUpdate = oldSimExeStatus.getLatestUpdateDate(); -// if (System.currentTimeMillis() - latestUpdate.getTime() >= MessageConstants.INTERVAL_PING_SERVER_MS * 3 / 5) { // new queue status cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); @@ -263,7 +268,6 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.RUNNING, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); } -// } } } @@ -271,28 +275,23 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ // new queue status cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - // new exe status endDate = new Date(); hasData = true; cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.COMPLETED, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); - } } else if (workerEvent.isFailedEvent()) { if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ // new queue status cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - // new exe status endDate = new Date(); cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); @@ -301,19 +300,41 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ // new queue status cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - // new exe status endDate = new Date(); - cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); SimulationMessage simulationMessage = SimulationMessage.workerFailure("solver stopped unexpectedly, "+workerEventSimulationMessage.getDisplayMessage()); - newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, taskID, simulationMessage, newQueueStatus, newExeStatus); } } + + return newJobStatus; + } + + public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDatabase simulationDatabase, VCMessageSession session) throws DataAccessException, VCMessagingException, SQLException { + updateSolverProcessTimestamp(); + WorkerEventMessage workerEventMessage = new WorkerEventMessage(workerEvent); + VCMongoMessage.sendWorkerEvent(workerEventMessage); + + String userName = workerEvent.getUserName(); // as the filter of the client + int workerEventTaskID = workerEvent.getTaskID(); + + if (lg.isTraceEnabled()) lg.trace("onWorkerEventMessage[" + workerEvent.getEventTypeID() + "," + workerEvent.getSimulationMessage() + "][simid=" + workerEvent.getVCSimulationDataIdentifier() + ",job=" + jobIndex + ",task=" + workerEventTaskID + "]"); + + if (!isWorkerEventOkay(workerEvent, simulationDatabase)){ + return; + } + + VCSimulationDataIdentifier vcSimDataID = workerEvent.getVCSimulationDataIdentifier(); + KeyValue simKey = vcSimDataID.getSimulationKey(); + cbit.vcell.server.SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(simKey, jobIndex); + + cbit.vcell.server.SimulationJobStatus.SchedulerStatus oldSchedulerStatus = oldSimulationJobStatus.getSchedulerStatus(); + cbit.vcell.server.SimulationJobStatus newJobStatus = produceSimulationJobStatusFromWorkerEvent(workerEvent, oldSimulationJobStatus); + if (newJobStatus!=null){ if (!newJobStatus.compareEqual(oldSimulationJobStatus) || workerEvent.isProgressEvent() || workerEvent.isNewDataEvent()) { Double progress = workerEvent.getProgress(); @@ -323,13 +344,16 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba runningStateInfo = new RunningStateInfo(progress,timepoint); } simulationDatabase.updateSimulationJobStatus(newJobStatus,runningStateInfo); - cbit.vcell.message.messages.StatusMessage msgForClient = new cbit.vcell.message.messages.StatusMessage(newJobStatus, userName, progress, timepoint); - msgForClient.sendToClient(session); + StatusMessage msgForClient = new StatusMessage(SimulationJobStatus.fromSimulationJobStatus(newJobStatus), userName, progress, timepoint); + + // TODO: Implement messaging to client + // msgForClient.sendToClient(session); if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); } else { simulationDatabase.updateSimulationJobStatus(newJobStatus); - cbit.vcell.message.messages.StatusMessage msgForClient = new cbit.vcell.message.messages.StatusMessage(newJobStatus, userName, null, null); - msgForClient.sendToClient(session); + StatusMessage msgForClient = new StatusMessage(SimulationJobStatus.fromSimulationJobStatus(newJobStatus), userName, null, null); + // TODO: Implement messaging to client + // msgForClient.sendToClient(session); if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); } }else if (workerEvent.isProgressEvent() || workerEvent.isNewDataEvent()){ @@ -340,8 +364,9 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba runningStateInfo = new RunningStateInfo(progress,timepoint); } simulationDatabase.updateSimulationJobStatus(oldSimulationJobStatus,runningStateInfo); - cbit.vcell.message.messages.StatusMessage msgForClient = new cbit.vcell.message.messages.StatusMessage(oldSimulationJobStatus, userName, progress, timepoint); - msgForClient.sendToClient(session); + StatusMessage msgForClient = new StatusMessage(SimulationJobStatus.fromSimulationJobStatus(oldSimulationJobStatus), userName, progress, timepoint); + // TODO: Implement messaging to client + // msgForClient.sendToClient(session); if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); }else{ VCMongoMessage.sendInfo("onWorkerEvent() ignoring WorkerEvent (currState="+oldSchedulerStatus.getDescription()+"): "+workerEvent.show()); @@ -512,19 +537,6 @@ public synchronized StatusMessage onStopRequest(User user, cbit.vcell.server.Sim cbit.vcell.server.SimulationJobStatus newJobStatus = new cbit.vcell.server.SimulationJobStatus(simJobStatus.getServerID(),simJobStatus.getVCSimulationIdentifier(),jobIndex,simJobStatus.getSubmitDate(), cbit.vcell.server.SimulationJobStatus.SchedulerStatus.STOPPED,taskID,SimulationMessage.solverStopped("simulation stopped by user"),simQueueEntryStatus,simExeStatus); - // - // send stopSimulation to serviceControl topic - // -// VCMessage msg = session.createMessage(); -// msg.setStringProperty(VCMessagingConstants.MESSAGE_TYPE_PROPERTY, MessageConstants.MESSAGE_TYPE_STOPSIMULATION_VALUE); -// msg.setLongProperty(MessageConstants.SIMKEY_PROPERTY, Long.parseLong(simKey + "")); -// msg.setIntProperty(MessageConstants.JOBINDEX_PROPERTY, jobIndex); -// msg.setIntProperty(MessageConstants.TASKID_PROPERTY, taskID); -// msg.setStringProperty(VCMessagingConstants.USERNAME_PROPERTY, user.getName()); -// if (simExeStatus.getHtcJobID()!=null){ -// msg.setStringProperty(MessageConstants.HTCJOBID_PROPERTY, simExeStatus.getHtcJobID().toDatabase()); -// } -// session.sendTopicMessage(VCellTopic.ServiceControlTopic, msg); if (lg.isTraceEnabled()) lg.trace("send " + MessageConstants.MESSAGE_TYPE_STOPSIMULATION_VALUE + " to " + VCellTopic.ServiceControlTopic.getName() + " topic"); SimulationJobStatus simulationJobStatusRecord = new SimulationJobStatus( @@ -545,53 +557,19 @@ public synchronized void onSystemAbort(cbit.vcell.server.SimulationJobStatus old int taskID = oldJobStatus.getTaskID(); - // - // status information (initialized as if new record) - // - Date startDate = null; - boolean hasData = false; - HtcJobID htcJobID = null; - String computeHost = null; - VCellServerID vcServerID = VCellServerID.getSystemServerID(); - Date submitDate = null; - Date queueDate = null; - int queuePriority = PRIORITY_DEFAULT; - - // // update using previously stored status (if available). // - cbit.vcell.server.SimulationExecutionStatus oldSimExeStatus = oldJobStatus.getSimulationExecutionStatus(); - if (oldSimExeStatus!=null && oldSimExeStatus.getStartDate()!=null){ - startDate = oldSimExeStatus.getStartDate(); - } - if (oldSimExeStatus!=null && oldSimExeStatus.hasData()){ - hasData = true; - } - if (oldSimExeStatus!=null && oldSimExeStatus.getComputeHost()!=null){ - computeHost = oldSimExeStatus.getComputeHost(); - } - if (oldSimExeStatus!=null && oldSimExeStatus.getHtcJobID()!=null){ - htcJobID = oldSimExeStatus.getHtcJobID(); - } - vcServerID = oldJobStatus.getServerID(); - submitDate = oldJobStatus.getSubmitDate(); - cbit.vcell.server.SimulationQueueEntryStatus oldQueueStatus = oldJobStatus.getSimulationQueueEntryStatus(); - if (oldQueueStatus!=null && oldQueueStatus.getQueueDate()!=null){ - queueDate = oldQueueStatus.getQueueDate(); - } - if (oldQueueStatus!=null){ - queuePriority = oldQueueStatus.getQueuePriority(); - } + CurrentState currentState = new CurrentState(oldJobStatus.getSimulationExecutionStatus(), oldJobStatus.getSimulationQueueEntryStatus(), oldJobStatus); - cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(currentState.queueDate, currentState.queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); Date endDate = new Date(); Date lastUpdateDate = new Date(); - cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(currentState.startDate, currentState.computeHost, lastUpdateDate, endDate, currentState.hasData, currentState.htcJobID); - cbit.vcell.server.SimulationJobStatus newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, oldJobStatus.getVCSimulationIdentifier(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, + cbit.vcell.server.SimulationJobStatus newJobStatus = new cbit.vcell.server.SimulationJobStatus(currentState.vcServerID, oldJobStatus.getVCSimulationIdentifier(), jobIndex, currentState.submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, taskID, SimulationMessage.jobFailed(failureMessage), newQueueStatus, newExeStatus); simulationDatabase.updateSimulationJobStatus(newJobStatus); From 228ced8863102a8e77405e11b152714c5988a4e7 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Mon, 19 Aug 2024 12:00:23 -0400 Subject: [PATCH 04/24] Move Simulation State Machine and SDE --- .../Simulations/DTO/SimulationJobStatus.java | 2 +- .../restq/Simulations/DTO/StatusMessage.java | 11 ++ .../restq/Simulations/SimulationRestDB.java | 13 +- .../restq/apiclient/SimulationApiTest.java | 4 +- .../SimulationDispatcherEngineCopy.java | 38 ++-- .../SimulationStateMachineCopy.java | 180 +++++++++--------- 6 files changed, 129 insertions(+), 119 deletions(-) rename vcell-rest/src/main/java/org/vcell/restq/Simulations/Control/SimulationDispatcherEngine.java => vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherEngineCopy.java (79%) rename vcell-rest/src/main/java/org/vcell/restq/Simulations/Control/SimulationStateMachine.java => vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineCopy.java (61%) diff --git a/vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/SimulationJobStatus.java b/vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/SimulationJobStatus.java index 3dc299c92c..eaf5c75cd5 100644 --- a/vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/SimulationJobStatus.java +++ b/vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/SimulationJobStatus.java @@ -27,7 +27,7 @@ public static SimulationJobStatus fromSimulationJobStatus(cbit.vcell.server.Simu s.getSchedulerStatus(), s.getSimulationMessage(), s.getTaskID(), - s.getServerID().toString(), + s.getServerID() != null ? s.getServerID().toString() : null, s.getJobIndex(), SimulationExecutionStatus.fromSimulationExecutionStatus(s.getSimulationExecutionStatus()), SimulationQueueEntryStatus.fromStatusRecord(s.getSimulationQueueEntryStatus()) diff --git a/vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/StatusMessage.java b/vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/StatusMessage.java index e3d75271de..8660825f2a 100644 --- a/vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/StatusMessage.java +++ b/vcell-rest/src/main/java/org/vcell/restq/Simulations/DTO/StatusMessage.java @@ -1,5 +1,16 @@ package org.vcell.restq.Simulations.DTO; +import java.util.ArrayList; + public record StatusMessage(SimulationJobStatus jobStatus, String userName, Double progress, Double timepoint) { + + public static ArrayList convertServerStatusMessages(ArrayList statusMessages){ + ArrayList dtoStatusMessage = new ArrayList<>(); + for (cbit.vcell.message.messages.StatusMessage statusMessage: statusMessages){ + dtoStatusMessage.add(new StatusMessage(SimulationJobStatus.fromSimulationJobStatus(statusMessage.getJobStatus()), statusMessage.getUserName(), + statusMessage.getProgress(), statusMessage.getTimePoint())); + } + return dtoStatusMessage; + } } diff --git a/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationRestDB.java b/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationRestDB.java index a6922da56e..c59a249aeb 100644 --- a/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationRestDB.java +++ b/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationRestDB.java @@ -7,7 +7,7 @@ import jakarta.enterprise.context.ApplicationScoped; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vcell.restq.Simulations.Control.SimulationDispatcherEngine; +import cbit.vcell.message.server.dispatcher.SimulationDispatcherEngineCopy; import org.vcell.restq.Simulations.DTO.SimulationStatus; import org.vcell.restq.Simulations.DTO.StatusMessage; import org.vcell.restq.db.AgroalConnectionFactory; @@ -31,13 +31,13 @@ public class SimulationRestDB { private final DatabaseServerImpl databaseServerImpl; private final SimulationDatabaseDirect simulationDatabaseDirect; - private final SimulationDispatcherEngine simulationDispatcherEngine; + private final SimulationDispatcherEngineCopy simulationDispatcherEngine; private final BioModelRestDB bioModelRestDB; public SimulationRestDB(AgroalConnectionFactory agroalConnectionFactory) throws DataAccessException, SQLException { databaseServerImpl = new DatabaseServerImpl(agroalConnectionFactory, agroalConnectionFactory.getKeyFactory()); simulationDatabaseDirect = new SimulationDatabaseDirect(new AdminDBTopLevel(agroalConnectionFactory), databaseServerImpl, true); - simulationDispatcherEngine = new SimulationDispatcherEngine(); + simulationDispatcherEngine = new SimulationDispatcherEngineCopy(); bioModelRestDB = new BioModelRestDB(this, databaseServerImpl); } @@ -96,8 +96,9 @@ public ArrayList startSimulation(String simId, User vcellUser) th VCSimulationIdentifier vcSimulationIdentifier = new VCSimulationIdentifier(simKey, vcellUser); SimulationRep simRep = getAndCheckSimRep(simId, vcellUser); try { - return simulationDispatcherEngine.onStartRequest(vcSimulationIdentifier, vcellUser, simRep.getScanCount(), + ArrayList statusMessages = simulationDispatcherEngine.onStartRequest(vcSimulationIdentifier, vcellUser, simRep.getScanCount(), simulationDatabaseDirect, null, null); + return StatusMessage.convertServerStatusMessages(statusMessages); } catch (VCMessagingException e) { throw new RuntimeException(e); } @@ -107,7 +108,9 @@ public ArrayList stopSimulation(String simId, User vcellUser) thr KeyValue simKey = new KeyValue(simId); getAndCheckSimRep(simId, vcellUser); VCSimulationIdentifier vcSimulationIdentifier = new VCSimulationIdentifier(simKey, vcellUser); - return simulationDispatcherEngine.onStopRequest(vcSimulationIdentifier, vcellUser, simulationDatabaseDirect, null); + ArrayList statusMessages = simulationDispatcherEngine.onStopRequest( + vcSimulationIdentifier, vcellUser, simulationDatabaseDirect, null); + return StatusMessage.convertServerStatusMessages(statusMessages); } public SimulationStatus getBioModelSimulationStatus(String simID, String bioModelID, User vcellUser) throws DataAccessException, SQLException { diff --git a/vcell-rest/src/test/java/org/vcell/restq/apiclient/SimulationApiTest.java b/vcell-rest/src/test/java/org/vcell/restq/apiclient/SimulationApiTest.java index c06a345855..3c787740a7 100644 --- a/vcell-rest/src/test/java/org/vcell/restq/apiclient/SimulationApiTest.java +++ b/vcell-rest/src/test/java/org/vcell/restq/apiclient/SimulationApiTest.java @@ -26,7 +26,7 @@ import org.vcell.restclient.api.SimulationResourceApi; import org.vcell.restclient.model.SchedulerStatus; import org.vcell.restclient.model.Status; -import org.vcell.restq.Simulations.Control.SimulationDispatcherEngine; +import cbit.vcell.message.server.dispatcher.SimulationDispatcherEngineCopy; import org.vcell.restq.Simulations.DTO.SimulationStatus; import org.vcell.restq.Simulations.DTO.StatusMessage; import org.vcell.restq.TestEndpointUtils; @@ -188,7 +188,7 @@ public void testDifferentSimulationStates() throws Exception{ SimulationJob simulationJob = new SimulationJob(simulation, 0, null); SimulationTask simulationTask = new SimulationTask(simulationJob, 0); WorkerEvent workerEvent = new WorkerEvent(WorkerEvent.JOB_ACCEPTED, "testService", simulationTask, "testHost", SimulationMessage.MESSAGE_JOB_ACCEPTED); - SimulationDispatcherEngine simulationDispatcherEngine = new SimulationDispatcherEngine(); + SimulationDispatcherEngineCopy simulationDispatcherEngine = new SimulationDispatcherEngineCopy(); simulationDispatcherEngine.onWorkerEvent(workerEvent, simulationDatabaseDirect, null); diff --git a/vcell-rest/src/main/java/org/vcell/restq/Simulations/Control/SimulationDispatcherEngine.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherEngineCopy.java similarity index 79% rename from vcell-rest/src/main/java/org/vcell/restq/Simulations/Control/SimulationDispatcherEngine.java rename to vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherEngineCopy.java index 576d52e680..6c327ab0b8 100644 --- a/vcell-rest/src/main/java/org/vcell/restq/Simulations/Control/SimulationDispatcherEngine.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherEngineCopy.java @@ -1,10 +1,9 @@ -package org.vcell.restq.Simulations.Control; +package cbit.vcell.message.server.dispatcher; import cbit.rmi.event.WorkerEvent; import cbit.vcell.message.VCMessageSession; import cbit.vcell.message.VCMessagingException; -import org.vcell.restq.Simulations.DTO.SimulationJobStatus; -import cbit.vcell.message.server.dispatcher.SimulationDatabase; +import cbit.vcell.message.messages.StatusMessage; import cbit.vcell.server.UpdateSynchronizationException; import cbit.vcell.solver.Simulation; import cbit.vcell.solver.SimulationInfo; @@ -12,7 +11,6 @@ import cbit.vcell.solver.server.SimulationMessage; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vcell.restq.Simulations.DTO.StatusMessage; import org.vcell.util.DataAccessException; import org.vcell.util.document.KeyValue; import org.vcell.util.document.User; @@ -24,34 +22,34 @@ import java.util.HashMap; import java.util.List; -public class SimulationDispatcherEngine { +public class SimulationDispatcherEngineCopy { public static final Logger lg = LogManager.getLogger(cbit.vcell.message.server.dispatcher.SimulationDispatcherEngine.class); - private HashMap> simStateMachineHash = new HashMap>(); + private HashMap> simStateMachineHash = new HashMap>(); - public SimulationDispatcherEngine() { + public SimulationDispatcherEngineCopy() { } - public SimulationStateMachine getSimulationStateMachine(KeyValue simulationKey, int jobIndex) { - List stateMachineList = simStateMachineHash.get(simulationKey); + public SimulationStateMachineCopy getSimulationStateMachine(KeyValue simulationKey, int jobIndex) { + List stateMachineList = simStateMachineHash.get(simulationKey); if (stateMachineList==null){ - stateMachineList = new ArrayList(); + stateMachineList = new ArrayList(); simStateMachineHash.put(simulationKey,stateMachineList); } - for (SimulationStateMachine stateMachine : stateMachineList){ + for (SimulationStateMachineCopy stateMachine : stateMachineList){ if (stateMachine.getJobIndex() == jobIndex){ return stateMachine; } } - SimulationStateMachine newStateMachine = new SimulationStateMachine(simulationKey, jobIndex); + SimulationStateMachineCopy newStateMachine = new SimulationStateMachineCopy(simulationKey, jobIndex); stateMachineList.add(newStateMachine); return newStateMachine; } public void onDispatch(Simulation simulation, cbit.vcell.server.SimulationJobStatus simJobStatus, SimulationDatabase simulationDatabase, VCMessageSession dispatcherQueueSession) throws VCMessagingException, DataAccessException, SQLException { KeyValue simulationKey = simJobStatus.getVCSimulationIdentifier().getSimulationKey(); - SimulationStateMachine simStateMachine = getSimulationStateMachine(simulationKey, simJobStatus.getJobIndex()); + SimulationStateMachineCopy simStateMachine = getSimulationStateMachine(simulationKey, simJobStatus.getJobIndex()); simStateMachine.onDispatch(simulation, simJobStatus, simulationDatabase, dispatcherQueueSession); } @@ -71,14 +69,14 @@ public ArrayList onStartRequest(VCSimulationIdentifier vcSimID, U if (lg.isWarnEnabled()) lg.warn("Bad simulation " + vcSimID); simJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Failed to dispatch simulation: "+ ex.getMessage()), null, null); - status.add(new StatusMessage(SimulationJobStatus.fromSimulationJobStatus(simJobStatus), user.getName(), null, null)); + status.add(new StatusMessage(simJobStatus, user.getName(), null, null)); return status; } if (simulationInfo == null) { if (lg.isWarnEnabled()) lg.warn("Can't start, simulation [" + vcSimID + "] doesn't exist in database"); simJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Can't start, simulation [" + vcSimID + "] doesn't exist"), null, null); - status.add(new StatusMessage(SimulationJobStatus.fromSimulationJobStatus(simJobStatus), user.getName(), null, null)); + status.add(new StatusMessage(simJobStatus, user.getName(), null, null)); return status; } @@ -86,12 +84,12 @@ public ArrayList onStartRequest(VCSimulationIdentifier vcSimID, U if (lg.isWarnEnabled()) lg.warn("Too many simulations (" + simulationScanCount + ") for parameter scan." + vcSimID); simJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Too many simulations (" + simulationScanCount + ") for parameter scan."), null, null); - status.add(new StatusMessage(SimulationJobStatus.fromSimulationJobStatus(simJobStatus), user.getName(), null, null)); + status.add(new StatusMessage(simJobStatus, user.getName(), null, null)); return status; } for (int simulationJobIndex = 0; simulationJobIndex < simulationScanCount; simulationJobIndex++){ - SimulationStateMachine simStateMachine = getSimulationStateMachine(simKey, simulationJobIndex); + SimulationStateMachineCopy simStateMachine = getSimulationStateMachine(simKey, simulationJobIndex); try { status.add(simStateMachine.onStartRequest(user, vcSimID, simulationDatabase, session)); }catch (UpdateSynchronizationException e){ @@ -114,7 +112,7 @@ public ArrayList onStopRequest(VCSimulationIdentifier vcSimID, Us } ArrayList stoppedSimulations = new ArrayList<>(); for (cbit.vcell.server.SimulationJobStatus simJobStatus : simJobStatusArray){ - SimulationStateMachine simStateMachine = getSimulationStateMachine(simKey, simJobStatus.getJobIndex()); + SimulationStateMachineCopy simStateMachine = getSimulationStateMachine(simKey, simJobStatus.getJobIndex()); try { stoppedSimulations.add(simStateMachine.onStopRequest(user, simJobStatus, simulationDatabase, session)); }catch (UpdateSynchronizationException e){ @@ -129,7 +127,7 @@ public void onWorkerEvent(WorkerEvent workerEvent, SimulationDatabase simulation try { KeyValue simKey = workerEvent.getVCSimulationDataIdentifier().getSimulationKey(); int jobIndex = workerEvent.getJobIndex(); - SimulationStateMachine simStateMachine = getSimulationStateMachine(simKey, jobIndex); + SimulationStateMachineCopy simStateMachine = getSimulationStateMachine(simKey, jobIndex); simStateMachine.onWorkerEvent(workerEvent, simulationDatabase, session); } catch (Exception ex) { lg.error(ex.getMessage(),ex); @@ -141,7 +139,7 @@ public void onSystemAbort(cbit.vcell.server.SimulationJobStatus jobStatus, Strin try { KeyValue simKey = jobStatus.getVCSimulationIdentifier().getSimulationKey(); int jobIndex = jobStatus.getJobIndex(); - SimulationStateMachine simStateMachine = getSimulationStateMachine(simKey, jobIndex); + SimulationStateMachineCopy simStateMachine = getSimulationStateMachine(simKey, jobIndex); simStateMachine.onSystemAbort(jobStatus, failureMessage, simulationDatabase, session); } catch (Exception ex) { lg.error(ex.getMessage(),ex); diff --git a/vcell-rest/src/main/java/org/vcell/restq/Simulations/Control/SimulationStateMachine.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineCopy.java similarity index 61% rename from vcell-rest/src/main/java/org/vcell/restq/Simulations/Control/SimulationStateMachine.java rename to vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineCopy.java index 0f3a79dc10..b6ef1a6cca 100644 --- a/vcell-rest/src/main/java/org/vcell/restq/Simulations/Control/SimulationStateMachine.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineCopy.java @@ -1,13 +1,15 @@ -package org.vcell.restq.Simulations.Control; +package cbit.vcell.message.server.dispatcher; import cbit.rmi.event.WorkerEvent; import cbit.vcell.field.FieldDataIdentifierSpec; -import cbit.vcell.message.*; +import cbit.vcell.message.VCMessageSession; +import cbit.vcell.message.VCMessagingConstants; +import cbit.vcell.message.VCMessagingException; +import cbit.vcell.message.VCellTopic; import cbit.vcell.message.messages.MessageConstants; import cbit.vcell.message.messages.SimulationTaskMessage; -import org.vcell.restq.Simulations.DTO.SimulationExecutionStatus; +import cbit.vcell.message.messages.StatusMessage; import cbit.vcell.message.messages.WorkerEventMessage; -import cbit.vcell.message.server.dispatcher.SimulationDatabase; import cbit.vcell.message.server.htc.HtcProxy; import cbit.vcell.messaging.server.SimulationTask; import cbit.vcell.mongodb.VCMongoMessage; @@ -16,9 +18,6 @@ import cbit.vcell.solver.server.SimulationMessage; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vcell.restq.Simulations.DTO.SimulationJobStatus; -import org.vcell.restq.Simulations.DTO.SimulationQueueEntryStatus; -import org.vcell.restq.Simulations.DTO.StatusMessage; import org.vcell.util.DataAccessException; import org.vcell.util.document.KeyValue; import org.vcell.util.document.User; @@ -28,7 +27,7 @@ import java.util.Arrays; import java.util.Date; -public class SimulationStateMachine { +public class SimulationStateMachineCopy { public static final Logger lg = LogManager.getLogger(cbit.vcell.message.server.dispatcher.SimulationStateMachine.class); // bitmapped counter so that allows 3 retries for each request (but preserves ordinal nature) @@ -64,11 +63,11 @@ private class CurrentState { public Date submitDate; public Date queueDate; public int queuePriority; - public cbit.vcell.server.SimulationJobStatus.SimulationQueueID simQueueID; + public SimulationJobStatus.SimulationQueueID simQueueID; - public CurrentState(cbit.vcell.server.SimulationExecutionStatus oldSimExeStatus, - cbit.vcell.server.SimulationQueueEntryStatus oldQueueStatus, - cbit.vcell.server.SimulationJobStatus oldSimulationJobStatus){ + public CurrentState(SimulationExecutionStatus oldSimExeStatus, + SimulationQueueEntryStatus oldQueueStatus, + SimulationJobStatus oldSimulationJobStatus){ boolean isOldExeNull = oldSimExeStatus == null; boolean isOldQueueNull = oldQueueStatus == null; // @@ -84,13 +83,13 @@ public CurrentState(cbit.vcell.server.SimulationExecutionStatus oldSimExeStatus, submitDate = oldSimulationJobStatus.getSubmitDate(); queueDate = !isOldQueueNull && oldQueueStatus.getQueueDate() != null ? oldQueueStatus.getQueueDate() : null; queuePriority = !isOldQueueNull ? oldQueueStatus.getQueuePriority() : PRIORITY_DEFAULT; - simQueueID = !isOldQueueNull && oldQueueStatus.getQueueID()!=null ? oldQueueStatus.getQueueID() : cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_WAITING; + simQueueID = !isOldQueueNull && oldQueueStatus.getQueueID()!=null ? oldQueueStatus.getQueueID() : SimulationJobStatus.SimulationQueueID.QUEUE_ID_WAITING; } } - public SimulationStateMachine(KeyValue simKey, int jobIndex){ + public SimulationStateMachineCopy(KeyValue simKey, int jobIndex){ this.simKey = simKey; this.jobIndex = jobIndex; updateSolverProcessTimestamp(); @@ -129,7 +128,7 @@ protected boolean isWorkerEventOkay(WorkerEvent workerEvent, SimulationDatabase return false; } KeyValue simKey = vcSimDataID.getSimulationKey(); - cbit.vcell.server.SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(simKey, jobIndex); + SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(simKey, jobIndex); if (oldSimulationJobStatus == null){ VCMongoMessage.sendInfo("onWorkerEvent() ignoring WorkerEvent, no current SimulationJobStatus: "+workerEvent.show()); @@ -142,13 +141,13 @@ protected boolean isWorkerEventOkay(WorkerEvent workerEvent, SimulationDatabase return true; } - protected cbit.vcell.server.SimulationJobStatus produceSimulationJobStatusFromWorkerEvent( + protected SimulationJobStatus produceSimulationJobStatusFromWorkerEvent( WorkerEvent workerEvent, - cbit.vcell.server.SimulationJobStatus oldSimulationJobStatus){ + SimulationJobStatus oldSimulationJobStatus){ - cbit.vcell.server.SimulationExecutionStatus oldSimExeStatus = oldSimulationJobStatus.getSimulationExecutionStatus(); - cbit.vcell.server.SimulationQueueEntryStatus oldQueueStatus = oldSimulationJobStatus.getSimulationQueueEntryStatus(); - cbit.vcell.server.SimulationJobStatus.SchedulerStatus oldSchedulerStatus = oldSimulationJobStatus.getSchedulerStatus(); + SimulationExecutionStatus oldSimExeStatus = oldSimulationJobStatus.getSimulationExecutionStatus(); + SimulationQueueEntryStatus oldQueueStatus = oldSimulationJobStatus.getSimulationQueueEntryStatus(); + SimulationJobStatus.SchedulerStatus oldSchedulerStatus = oldSimulationJobStatus.getSchedulerStatus(); VCSimulationDataIdentifier vcSimDataID = workerEvent.getVCSimulationDataIdentifier(); int taskID = oldSimulationJobStatus.getTaskID(); @@ -168,7 +167,7 @@ protected cbit.vcell.server.SimulationJobStatus produceSimulationJobStatusFromWo Date submitDate = currentState.submitDate; Date queueDate = currentState.queueDate; int queuePriority = currentState.queuePriority; - cbit.vcell.server.SimulationJobStatus.SimulationQueueID simQueueID = currentState.simQueueID; + SimulationJobStatus.SimulationQueueID simQueueID = currentState.simQueueID; // // update using new information from event @@ -185,7 +184,7 @@ protected cbit.vcell.server.SimulationJobStatus produceSimulationJobStatusFromWo } - cbit.vcell.server.SimulationJobStatus newJobStatus = null; + SimulationJobStatus newJobStatus = null; if (workerEvent.isAcceptedEvent()) { // @@ -193,15 +192,15 @@ protected cbit.vcell.server.SimulationJobStatus produceSimulationJobStatusFromWo // if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued()) { // new queue status - cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); // new exe status lastUpdateDate = new Date(); startDate = lastUpdateDate; endDate = null; - cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.DISPATCHED, + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.DISPATCHED, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); } @@ -209,35 +208,35 @@ protected cbit.vcell.server.SimulationJobStatus produceSimulationJobStatusFromWo // only update database when the job event changes from started to running. The later progress event will not be recorded. if ( oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()) { // new queue status - cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); // new exe status lastUpdateDate = new Date(); if (startDate == null){ startDate = lastUpdateDate; } - cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.RUNNING, + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); } } else if (workerEvent.isNewDataEvent()) { if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ - if (!oldSchedulerStatus.isRunning() || simQueueID != cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL || hasData==false){ + if (!oldSchedulerStatus.isRunning() || simQueueID != SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL || hasData==false){ // new queue status - cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); // new exe status if (startDate == null){ startDate = lastUpdateDate; } hasData = true; - cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.RUNNING, + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); } } @@ -246,26 +245,26 @@ protected cbit.vcell.server.SimulationJobStatus produceSimulationJobStatusFromWo if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ - if (!oldSchedulerStatus.isRunning() || simQueueID != cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL){ + if (!oldSchedulerStatus.isRunning() || simQueueID != SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL){ // new queue status - cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); // new exe status if (startDate == null){ startDate = lastUpdateDate; } - cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.RUNNING, + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); }else if (oldSchedulerStatus.isRunning()){ if (oldSimExeStatus != null) { // new queue status - cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.RUNNING, + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); } } @@ -274,38 +273,38 @@ protected cbit.vcell.server.SimulationJobStatus produceSimulationJobStatusFromWo } else if (workerEvent.isCompletedEvent()) { if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ // new queue status - cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); // new exe status endDate = new Date(); hasData = true; - cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.COMPLETED, + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.COMPLETED, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); } } else if (workerEvent.isFailedEvent()) { if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ // new queue status - cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); // new exe status endDate = new Date(); - cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.FAILED, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); } } else if (workerEvent.isWorkerExitErrorEvent()) { if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ // new queue status - cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(queueDate, queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); // new exe status endDate = new Date(); - cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); SimulationMessage simulationMessage = SimulationMessage.workerFailure("solver stopped unexpectedly, "+workerEventSimulationMessage.getDisplayMessage()); - newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.FAILED, taskID, simulationMessage, newQueueStatus, newExeStatus); } @@ -330,10 +329,10 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba VCSimulationDataIdentifier vcSimDataID = workerEvent.getVCSimulationDataIdentifier(); KeyValue simKey = vcSimDataID.getSimulationKey(); - cbit.vcell.server.SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(simKey, jobIndex); + SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(simKey, jobIndex); - cbit.vcell.server.SimulationJobStatus.SchedulerStatus oldSchedulerStatus = oldSimulationJobStatus.getSchedulerStatus(); - cbit.vcell.server.SimulationJobStatus newJobStatus = produceSimulationJobStatusFromWorkerEvent(workerEvent, oldSimulationJobStatus); + SimulationJobStatus.SchedulerStatus oldSchedulerStatus = oldSimulationJobStatus.getSchedulerStatus(); + SimulationJobStatus newJobStatus = produceSimulationJobStatusFromWorkerEvent(workerEvent, oldSimulationJobStatus); if (newJobStatus!=null){ if (!newJobStatus.compareEqual(oldSimulationJobStatus) || workerEvent.isProgressEvent() || workerEvent.isNewDataEvent()) { @@ -344,14 +343,14 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba runningStateInfo = new RunningStateInfo(progress,timepoint); } simulationDatabase.updateSimulationJobStatus(newJobStatus,runningStateInfo); - StatusMessage msgForClient = new StatusMessage(SimulationJobStatus.fromSimulationJobStatus(newJobStatus), userName, progress, timepoint); + StatusMessage msgForClient = new StatusMessage(newJobStatus, userName, progress, timepoint); // TODO: Implement messaging to client // msgForClient.sendToClient(session); if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); } else { simulationDatabase.updateSimulationJobStatus(newJobStatus); - StatusMessage msgForClient = new StatusMessage(SimulationJobStatus.fromSimulationJobStatus(newJobStatus), userName, null, null); + StatusMessage msgForClient = new StatusMessage(newJobStatus, userName, null, null); // TODO: Implement messaging to client // msgForClient.sendToClient(session); if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); @@ -364,7 +363,7 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba runningStateInfo = new RunningStateInfo(progress,timepoint); } simulationDatabase.updateSimulationJobStatus(oldSimulationJobStatus,runningStateInfo); - StatusMessage msgForClient = new StatusMessage(SimulationJobStatus.fromSimulationJobStatus(oldSimulationJobStatus), userName, progress, timepoint); + StatusMessage msgForClient = new StatusMessage(oldSimulationJobStatus, userName, progress, timepoint); // TODO: Implement messaging to client // msgForClient.sendToClient(session); if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); @@ -379,23 +378,23 @@ public synchronized StatusMessage onStartRequest(User user, VCSimulationIdentifi if (!user.equals(vcSimID.getOwner())) { lg.error(user + " is not authorized to start simulation (key=" + simKey + ")"); - cbit.vcell.server.SimulationJobStatus simulationJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, 0, null, - cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to start this simulation!"), null, null); - StatusMessage message = new StatusMessage(SimulationJobStatus.fromSimulationJobStatus(simulationJobStatus), user.getName(), null, null); + SimulationJobStatus simulationJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, 0, null, + SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to start this simulation!"), null, null); + StatusMessage message = new StatusMessage(simulationJobStatus, user.getName(), null, null); VCMongoMessage.sendInfo("onStartRequest("+vcSimID.getID()+") ignoring start simulation request - wrong user): simID="+vcSimID); return message; } - cbit.vcell.server.SimulationJobStatus newJobStatus = saveSimulationStartRequest(vcSimID, jobIndex, simulationDatabase); + SimulationJobStatus newJobStatus = saveSimulationStartRequest(vcSimID, jobIndex, simulationDatabase); - return new StatusMessage(SimulationJobStatus.fromSimulationJobStatus(newJobStatus), user.getName(), null, null); + return new StatusMessage(newJobStatus, user.getName(), null, null); } - public static cbit.vcell.server.SimulationJobStatus saveSimulationStartRequest(VCSimulationIdentifier vcSimID, int jobIndex, SimulationDatabase simulationDatabase) throws DataAccessException, SQLException { + public static SimulationJobStatus saveSimulationStartRequest(VCSimulationIdentifier vcSimID, int jobIndex, SimulationDatabase simulationDatabase) throws DataAccessException, SQLException { // // get latest simulation job task (if any). // - cbit.vcell.server.SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(vcSimID.getSimulationKey(), jobIndex); + SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(vcSimID.getSimulationKey(), jobIndex); int oldTaskID = -1; if (oldSimulationJobStatus != null){ oldTaskID = oldSimulationJobStatus.getTaskID(); @@ -418,7 +417,7 @@ public static cbit.vcell.server.SimulationJobStatus saveSimulationStartRequest(V Date currentDate = new Date(); // new queue status - cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_WAITING); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationJobStatus.SimulationQueueID.QUEUE_ID_WAITING); // new exe status Date lastUpdateDate = new Date(); @@ -428,12 +427,12 @@ public static cbit.vcell.server.SimulationJobStatus saveSimulationStartRequest(V HtcJobID htcJobID = null; boolean hasData = false; - cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); VCellServerID vcServerID = VCellServerID.getSystemServerID(); Date submitDate = currentDate; - cbit.vcell.server.SimulationJobStatus newJobStatus = new cbit.vcell.server.SimulationJobStatus(vcServerID, vcSimID, jobIndex, submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.WAITING, + SimulationJobStatus newJobStatus = new SimulationJobStatus(vcServerID, vcSimID, jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.WAITING, newTaskID, SimulationMessage.MESSAGE_JOB_WAITING, newQueueStatus, newExeStatus); simulationDatabase.insertSimulationJobStatus(newJobStatus); @@ -441,7 +440,7 @@ public static cbit.vcell.server.SimulationJobStatus saveSimulationStartRequest(V } - public synchronized void onDispatch(Simulation simulation, cbit.vcell.server.SimulationJobStatus oldSimulationJobStatus, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, DataAccessException, SQLException { + public synchronized void onDispatch(Simulation simulation, SimulationJobStatus oldSimulationJobStatus, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, DataAccessException, SQLException { updateSolverProcessTimestamp(); VCSimulationIdentifier vcSimID = oldSimulationJobStatus.getVCSimulationIdentifier(); int taskID = oldSimulationJobStatus.getTaskID(); @@ -471,23 +470,23 @@ public synchronized void onDispatch(Simulation simulation, cbit.vcell.server.Sim HtcProxy.MemLimitResults allowableMemMB = HtcProxy.getMemoryLimit(vcellUserid,simID,solverDescription, requiredMemMB, isPowerUser); - final cbit.vcell.server.SimulationJobStatus newSimJobStatus; + final SimulationJobStatus newSimJobStatus; if (requiredMemMB > allowableMemMB.getMemLimit()) { // // fail the simulation // Date currentDate = new Date(); // new queue status - cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - cbit.vcell.server.SimulationExecutionStatus newSimExeStatus = new cbit.vcell.server.SimulationExecutionStatus(null, null, new Date(), null, false, null); - newSimJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(),vcSimID,jobIndex, - oldSimulationJobStatus.getSubmitDate(), cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED,taskID, + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationExecutionStatus newSimExeStatus = new SimulationExecutionStatus(null, null, new Date(), null, false, null); + newSimJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(),vcSimID,jobIndex, + oldSimulationJobStatus.getSubmitDate(), SimulationJobStatus.SchedulerStatus.FAILED,taskID, SimulationMessage.jobFailed("simulation required "+requiredMemMB+"MB of memory, only "+allowableMemMB.getMemLimit()+"MB allowed from "+allowableMemMB.getMemLimitSource()), newQueueStatus,newSimExeStatus); simulationDatabase.updateSimulationJobStatus(newSimJobStatus); - cbit.vcell.message.messages.StatusMessage message = new cbit.vcell.message.messages.StatusMessage(newSimJobStatus, simulation.getVersion().getOwner().getName(), null, null); + StatusMessage message = new StatusMessage(newSimJobStatus, simulation.getVersion().getOwner().getName(), null, null); message.sendToClient(session); }else{ @@ -495,10 +494,10 @@ public synchronized void onDispatch(Simulation simulation, cbit.vcell.server.Sim // dispatch the simulation, new queue status // Date currentDate = new Date(); - cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_SIMULATIONJOB); - cbit.vcell.server.SimulationExecutionStatus newSimExeStatus = new cbit.vcell.server.SimulationExecutionStatus(null, null, new Date(), null, false, null); - newSimJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(),vcSimID,jobIndex, - oldSimulationJobStatus.getSubmitDate(), cbit.vcell.server.SimulationJobStatus.SchedulerStatus.DISPATCHED,taskID, + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationJobStatus.SimulationQueueID.QUEUE_ID_SIMULATIONJOB); + SimulationExecutionStatus newSimExeStatus = new SimulationExecutionStatus(null, null, new Date(), null, false, null); + newSimJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(),vcSimID,jobIndex, + oldSimulationJobStatus.getSubmitDate(), SimulationJobStatus.SchedulerStatus.DISPATCHED,taskID, SimulationMessage.MESSAGE_JOB_DISPATCHED, newQueueStatus,newSimExeStatus); @@ -515,34 +514,33 @@ public synchronized void onDispatch(Simulation simulation, cbit.vcell.server.Sim } - public synchronized StatusMessage onStopRequest(User user, cbit.vcell.server.SimulationJobStatus simJobStatus, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, DataAccessException, SQLException { + public synchronized StatusMessage onStopRequest(User user, SimulationJobStatus simJobStatus, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, DataAccessException, SQLException { updateSolverProcessTimestamp(); if (!user.equals(simJobStatus.getVCSimulationIdentifier().getOwner())) { lg.error(user + " is not authorized to stop simulation (key=" + simKey + ")"); - cbit.vcell.server.SimulationJobStatus simulationJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(), simJobStatus.getVCSimulationIdentifier(), 0, null, - cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to stop this simulation!"), null, null); + SimulationJobStatus simulationJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(), simJobStatus.getVCSimulationIdentifier(), 0, null, + SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to stop this simulation!"), null, null); VCMongoMessage.sendInfo("onStopRequest("+simJobStatus.getVCSimulationIdentifier()+") ignoring stop simulation request - wrong user)"); - return new StatusMessage(SimulationJobStatus.fromSimulationJobStatus(simulationJobStatus), user.getName(), null, null); + return new StatusMessage(simulationJobStatus, user.getName(), null, null); } // stop latest task if active - cbit.vcell.server.SimulationJobStatus.SchedulerStatus schedulerStatus = simJobStatus.getSchedulerStatus(); + SimulationJobStatus.SchedulerStatus schedulerStatus = simJobStatus.getSchedulerStatus(); int taskID = simJobStatus.getTaskID(); if (schedulerStatus.isActive()){ - cbit.vcell.server.SimulationQueueEntryStatus simQueueEntryStatus = simJobStatus.getSimulationQueueEntryStatus(); - cbit.vcell.server.SimulationExecutionStatus simExeStatus = simJobStatus.getSimulationExecutionStatus(); - cbit.vcell.server.SimulationJobStatus newJobStatus = new cbit.vcell.server.SimulationJobStatus(simJobStatus.getServerID(),simJobStatus.getVCSimulationIdentifier(),jobIndex,simJobStatus.getSubmitDate(), - cbit.vcell.server.SimulationJobStatus.SchedulerStatus.STOPPED,taskID,SimulationMessage.solverStopped("simulation stopped by user"),simQueueEntryStatus,simExeStatus); + SimulationQueueEntryStatus simQueueEntryStatus = simJobStatus.getSimulationQueueEntryStatus(); + SimulationExecutionStatus simExeStatus = simJobStatus.getSimulationExecutionStatus(); + SimulationJobStatus newJobStatus = new SimulationJobStatus(simJobStatus.getServerID(),simJobStatus.getVCSimulationIdentifier(),jobIndex,simJobStatus.getSubmitDate(), + SimulationJobStatus.SchedulerStatus.STOPPED,taskID,SimulationMessage.solverStopped("simulation stopped by user"),simQueueEntryStatus,simExeStatus); if (lg.isTraceEnabled()) lg.trace("send " + MessageConstants.MESSAGE_TYPE_STOPSIMULATION_VALUE + " to " + VCellTopic.ServiceControlTopic.getName() + " topic"); SimulationJobStatus simulationJobStatusRecord = new SimulationJobStatus( - null, new VCSimulationIdentifier(simKey, user), simJobStatus.getSubmitDate().toInstant(), - cbit.vcell.server.SimulationJobStatus.SchedulerStatus.STOPPED, simJobStatus.getSimulationMessage(), taskID, - VCellServerID.getSystemServerID().toString(), jobIndex, SimulationExecutionStatus.fromSimulationExecutionStatus(simExeStatus), SimulationQueueEntryStatus.fromStatusRecord(simQueueEntryStatus) + null, new VCSimulationIdentifier(simKey, user), jobIndex, simJobStatus.getSubmitDate(), + SimulationJobStatus.SchedulerStatus.STOPPED, taskID, simJobStatus.getSimulationMessage(), simQueueEntryStatus, simExeStatus ); simulationDatabase.updateSimulationJobStatus(newJobStatus); @@ -552,7 +550,7 @@ null, new VCSimulationIdentifier(simKey, user), simJobStatus.getSubmitDate().toI return null; } - public synchronized void onSystemAbort(cbit.vcell.server.SimulationJobStatus oldJobStatus, String failureMessage, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, UpdateSynchronizationException, DataAccessException, SQLException { + public synchronized void onSystemAbort(SimulationJobStatus oldJobStatus, String failureMessage, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, UpdateSynchronizationException, DataAccessException, SQLException { updateSolverProcessTimestamp(); int taskID = oldJobStatus.getTaskID(); @@ -562,14 +560,14 @@ public synchronized void onSystemAbort(cbit.vcell.server.SimulationJobStatus old // CurrentState currentState = new CurrentState(oldJobStatus.getSimulationExecutionStatus(), oldJobStatus.getSimulationQueueEntryStatus(), oldJobStatus); - cbit.vcell.server.SimulationQueueEntryStatus newQueueStatus = new cbit.vcell.server.SimulationQueueEntryStatus(currentState.queueDate, currentState.queuePriority, cbit.vcell.server.SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentState.queueDate, currentState.queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); Date endDate = new Date(); Date lastUpdateDate = new Date(); - cbit.vcell.server.SimulationExecutionStatus newExeStatus = new cbit.vcell.server.SimulationExecutionStatus(currentState.startDate, currentState.computeHost, lastUpdateDate, endDate, currentState.hasData, currentState.htcJobID); + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(currentState.startDate, currentState.computeHost, lastUpdateDate, endDate, currentState.hasData, currentState.htcJobID); - cbit.vcell.server.SimulationJobStatus newJobStatus = new cbit.vcell.server.SimulationJobStatus(currentState.vcServerID, oldJobStatus.getVCSimulationIdentifier(), jobIndex, currentState.submitDate, cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, + SimulationJobStatus newJobStatus = new SimulationJobStatus(currentState.vcServerID, oldJobStatus.getVCSimulationIdentifier(), jobIndex, currentState.submitDate, SimulationJobStatus.SchedulerStatus.FAILED, taskID, SimulationMessage.jobFailed(failureMessage), newQueueStatus, newExeStatus); simulationDatabase.updateSimulationJobStatus(newJobStatus); From a36537bff0af09a3d9b93bb51c9b35a8ae9101ba Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Thu, 22 Aug 2024 07:20:53 -0400 Subject: [PATCH 05/24] Mock Simulation DB and Initial State Machine Tests --- .../server/dispatcher/MockSimulationDB.java | 129 ++++++++++++++++++ .../SimulationStateMachineTest.java | 122 +++++++++++++++++ 2 files changed, 251 insertions(+) create mode 100644 vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java create mode 100644 vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java new file mode 100644 index 0000000000..d81e5f6cda --- /dev/null +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java @@ -0,0 +1,129 @@ +package cbit.vcell.message.server.dispatcher; + +import cbit.vcell.field.FieldDataIdentifierSpec; +import cbit.vcell.messaging.db.SimulationRequirements; +import cbit.vcell.server.*; +import cbit.vcell.solver.Simulation; +import cbit.vcell.solver.SimulationInfo; +import org.vcell.util.DataAccessException; +import org.vcell.util.ObjectNotFoundException; +import org.vcell.util.document.KeyValue; +import org.vcell.util.document.User; +import org.vcell.util.document.VCellServerID; + +import java.sql.SQLException; +import java.util.*; + +public class MockSimulationDB implements SimulationDatabase{ + + private final HashMap> dbTable = new HashMap<>(); + + @Override + public SimulationJobStatus getLatestSimulationJobStatus(KeyValue simKey, int jobIndex) throws DataAccessException, SQLException { + ArrayList simList = dbTable.get(simKey.toString()); + if (simList == null){ + return null; + } + SimulationJobStatus latestSim = simList.get(0); + for (SimulationJobStatus jobStatus : simList){ + if (jobStatus.getJobIndex() == jobIndex && latestSim.getSubmitDate().after(jobStatus.getSubmitDate())){ + latestSim = jobStatus; + } + } + return latestSim; + } + + @Override + public void insertSimulationJobStatus(SimulationJobStatus simulationJobStatus) throws DataAccessException, SQLException { + String simKey = simulationJobStatus.getVCSimulationIdentifier().getSimulationKey().toString(); + if (dbTable.containsKey(simKey)){ + dbTable.get(simKey).add(simulationJobStatus); + } else { + dbTable.put(simKey, new ArrayList<>(){{add(simulationJobStatus);}}); + } + } + + @Override + public SimulationJobStatus[] getActiveJobs(VCellServerID vcellServerID) throws DataAccessException, SQLException { + throw new SQLException(); + } + + @Override + public SimulationJobStatus[] queryJobs(SimpleJobStatusQuerySpec simStatusQuerySpec) throws ObjectNotFoundException, DataAccessException { + return new SimulationJobStatus[0]; + } + + @Override + public Map getSimulationRequirements(Collection simKeys) throws SQLException { + return Map.of(); + } + + @Override + public void updateSimulationJobStatus(SimulationJobStatus newSimulationJobStatus) throws DataAccessException, UpdateSynchronizationException, SQLException { + String simKey = newSimulationJobStatus.getVCSimulationIdentifier().getSimulationKey().toString(); + ArrayList jobStatuses = dbTable.get(simKey); + for (int i = 0; i < jobStatuses.size(); i++){ + SimulationJobStatus jobStatus = jobStatuses.get(i); + boolean sameJob = jobStatus.getJobIndex() == newSimulationJobStatus.getJobIndex(); + if (sameJob){ + jobStatuses.set(i,newSimulationJobStatus); + break; + } + } + } + + @Override + public void updateSimulationJobStatus(SimulationJobStatus newSimulationJobStatus, StateInfo runningStateInfo) throws DataAccessException, UpdateSynchronizationException, SQLException { + + } + + @Override + public KeyValue[] getSimulationKeysFromBiomodel(KeyValue biomodelKey) throws SQLException, DataAccessException { + throw new SQLException(); + } + + @Override + public Simulation getSimulation(User user, KeyValue simKey) throws DataAccessException { + throw new DataAccessException(); + } + + @Override + public FieldDataIdentifierSpec[] getFieldDataIdentifierSpecs(Simulation sim) throws DataAccessException { + throw new DataAccessException(); + } + + @Override + public Set getUnreferencedSimulations() throws SQLException { + return Set.of(); + } + + @Override + public User.SpecialUser getUser(String username) throws DataAccessException, SQLException { + return null; + } + + @Override + public TreeMap> getSpecialUsers() throws DataAccessException, SQLException { + return null; + } + + @Override + public SimulationInfo getSimulationInfo(User user, KeyValue simKey) throws ObjectNotFoundException, DataAccessException { + return null; + } + + @Override + public SimulationStatus[] getSimulationStatus(KeyValue[] simKeys) throws ObjectNotFoundException, DataAccessException { + return new SimulationStatus[0]; + } + + @Override + public SimulationStatus getSimulationStatus(KeyValue simulationKey) throws ObjectNotFoundException, DataAccessException { + return null; + } + + @Override + public SimpleJobStatus[] getSimpleJobStatus(User user, SimpleJobStatusQuerySpec simStatusQuerySpec) throws ObjectNotFoundException, DataAccessException { + return new SimpleJobStatus[0]; + } +} diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java new file mode 100644 index 0000000000..4f88d335e5 --- /dev/null +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java @@ -0,0 +1,122 @@ +package cbit.vcell.message.server.dispatcher; + +import cbit.rmi.event.WorkerEvent; +import cbit.vcell.server.SimulationJobStatus; +import cbit.vcell.solver.VCSimulationIdentifier; +import cbit.vcell.solver.server.SimulationMessage; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.vcell.util.DataAccessException; +import org.vcell.util.document.KeyValue; +import org.vcell.util.document.User; +import org.vcell.util.document.VCellServerID; + +import java.sql.SQLException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Date; + +@Tag("Fast") +public class SimulationStateMachineTest { + + private static final VCellServerID testVCellServerID = VCellServerID.getServerID("test"); + private static final User testUser = new User("Alice", new KeyValue("0")); + + + private MockSimulationDB simulationDB; + + @BeforeEach + public void setUp(){ + simulationDB = new MockSimulationDB(); + } + + private record ChangedStateValues( + VCSimulationIdentifier simID, + SimulationJobStatus.SchedulerStatus schedulerStatus, + int taskID, + String changesResult + ){ } + + private interface WorkerEventFactory { + WorkerEvent createChangedWorker(ChangedStateValues c); + } + + @Test + public void workerEventRejectionsTest() throws SQLException, DataAccessException { + KeyValue simKey = new KeyValue("1"); + VCSimulationIdentifier simID = new VCSimulationIdentifier(simKey, testUser); + int jobIndex = 0; + int taskID = 16; + SimulationStateMachineCopy stateMachine = new SimulationStateMachineCopy(simKey, jobIndex); + SimulationMessage acceptedSimulationMessage = SimulationMessage.workerAccepted("accepted"); + + + WorkerEventFactory workerEventFactory = (w) -> {return new WorkerEvent(WorkerEvent.JOB_ACCEPTED, simKey, w.simID, + jobIndex, "", + w.taskID, null, null, + acceptedSimulationMessage);}; + + + ArrayList changedValues = new ArrayList<>(){{ + add(new ChangedStateValues(simID, SimulationJobStatus.SchedulerStatus.RUNNING, taskID, "No old status.")); // no old status failure + add(new ChangedStateValues(simID, SimulationJobStatus.SchedulerStatus.COMPLETED, taskID, "Work is already done.")); // work is done failure + add(new ChangedStateValues(simID, SimulationJobStatus.SchedulerStatus.RUNNING, 0, "Task ID is lower")); // old status has higher number taskID failure + }}; + + for (int i = 0; i < changedValues.size(); i++){ + ChangedStateValues workerEventChangedValues = changedValues.get(i); + if (i == 1){ + simulationDB.insertSimulationJobStatus(new SimulationJobStatus(testVCellServerID, simID, jobIndex, Date.from(Instant.now()), workerEventChangedValues.schedulerStatus, taskID, + acceptedSimulationMessage, null, null)); + } else if (i > 1) { + SimulationJobStatus failingStatus = new SimulationJobStatus(testVCellServerID, simID, 0, Date.from(Instant.now()), workerEventChangedValues.schedulerStatus, taskID, + acceptedSimulationMessage, null, null); + simulationDB.updateSimulationJobStatus(failingStatus); + } + WorkerEvent workerEvent = workerEventFactory.createChangedWorker(workerEventChangedValues); + Assertions.assertFalse(stateMachine.isWorkerEventOkay(workerEvent, simulationDB), workerEventChangedValues.changesResult); + } + + ChangedStateValues passingWorkerValues= new ChangedStateValues(simID, null, taskID, ""); + WorkerEvent passingWorkerEvent = workerEventFactory.createChangedWorker(passingWorkerValues); + + for (SimulationJobStatus.SchedulerStatus passingStatus: SimulationJobStatus.SchedulerStatus.values()){ + if (!passingStatus.isDone()){ + SimulationJobStatus simulationJobStatus = new SimulationJobStatus(testVCellServerID, simID, jobIndex, Date.from(Instant.now()), passingStatus, taskID, + acceptedSimulationMessage, null, null); + simulationDB.updateSimulationJobStatus(simulationJobStatus); + Assertions.assertTrue(stateMachine.isWorkerEventOkay(passingWorkerEvent, simulationDB)); + } + } + + } + + @Test + public void stateShouldTransitionToFailure(){ + KeyValue simKey = new KeyValue("1"); + VCSimulationIdentifier simID = new VCSimulationIdentifier(simKey, testUser); + int jobIndex = 0; + int taskID = 16; + SimulationStateMachineCopy stateMachine = new SimulationStateMachineCopy(simKey, jobIndex); + } + + public void stateShouldTransitionToWaiting(){ + + } + + public void stateShouldTransitionToDispatched(){ + + } + + public void stateShouldTransitionToRunning(){ + + } + + public void stateShouldTransitionToCompleted(){ + + } + + +} From 1802cc54d3eb134272ef4a9a91d4f93bca5e4ed6 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Fri, 23 Aug 2024 12:09:22 -0400 Subject: [PATCH 06/24] Update State Machine and Mock Sim DB --- .../dispatcher/SimulationStateMachineCopy.java | 10 +++------- .../server/dispatcher/MockSimulationDB.java | 18 ++++++++++-------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineCopy.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineCopy.java index b6ef1a6cca..f87a291280 100644 --- a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineCopy.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineCopy.java @@ -141,7 +141,7 @@ protected boolean isWorkerEventOkay(WorkerEvent workerEvent, SimulationDatabase return true; } - protected SimulationJobStatus produceSimulationJobStatusFromWorkerEvent( + private SimulationJobStatus produceStateFromWorkerEvent( WorkerEvent workerEvent, SimulationJobStatus oldSimulationJobStatus){ @@ -332,7 +332,7 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(simKey, jobIndex); SimulationJobStatus.SchedulerStatus oldSchedulerStatus = oldSimulationJobStatus.getSchedulerStatus(); - SimulationJobStatus newJobStatus = produceSimulationJobStatusFromWorkerEvent(workerEvent, oldSimulationJobStatus); + SimulationJobStatus newJobStatus = produceStateFromWorkerEvent(workerEvent, oldSimulationJobStatus); if (newJobStatus!=null){ if (!newJobStatus.compareEqual(oldSimulationJobStatus) || workerEvent.isProgressEvent() || workerEvent.isNewDataEvent()) { @@ -421,13 +421,9 @@ public static SimulationJobStatus saveSimulationStartRequest(VCSimulationIdentif // new exe status Date lastUpdateDate = new Date(); - String computeHost = null; - Date startDate = null; - Date endDate = null; - HtcJobID htcJobID = null; boolean hasData = false; - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(null, null, lastUpdateDate, null, hasData, null); VCellServerID vcServerID = VCellServerID.getSystemServerID(); Date submitDate = currentDate; diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java index d81e5f6cda..121e58a3e9 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java @@ -24,9 +24,11 @@ public SimulationJobStatus getLatestSimulationJobStatus(KeyValue simKey, int job if (simList == null){ return null; } - SimulationJobStatus latestSim = simList.get(0); + SimulationJobStatus latestSim = null; for (SimulationJobStatus jobStatus : simList){ - if (jobStatus.getJobIndex() == jobIndex && latestSim.getSubmitDate().after(jobStatus.getSubmitDate())){ + boolean equalJobIndex = jobStatus.getJobIndex() == jobIndex; + boolean isLatestSimNull = latestSim == null; + if ((equalJobIndex && isLatestSimNull) || (!isLatestSimNull && equalJobIndex && latestSim.getSubmitDate().after(jobStatus.getSubmitDate()))){ latestSim = jobStatus; } } @@ -60,6 +62,11 @@ public Map getSimulationRequirements(Collectio @Override public void updateSimulationJobStatus(SimulationJobStatus newSimulationJobStatus) throws DataAccessException, UpdateSynchronizationException, SQLException { + updateSimulationJobStatus(newSimulationJobStatus, null); + } + + @Override + public void updateSimulationJobStatus(SimulationJobStatus newSimulationJobStatus, StateInfo runningStateInfo) throws DataAccessException, UpdateSynchronizationException, SQLException { String simKey = newSimulationJobStatus.getVCSimulationIdentifier().getSimulationKey().toString(); ArrayList jobStatuses = dbTable.get(simKey); for (int i = 0; i < jobStatuses.size(); i++){ @@ -72,11 +79,6 @@ public void updateSimulationJobStatus(SimulationJobStatus newSimulationJobStatus } } - @Override - public void updateSimulationJobStatus(SimulationJobStatus newSimulationJobStatus, StateInfo runningStateInfo) throws DataAccessException, UpdateSynchronizationException, SQLException { - - } - @Override public KeyValue[] getSimulationKeysFromBiomodel(KeyValue biomodelKey) throws SQLException, DataAccessException { throw new SQLException(); @@ -89,7 +91,7 @@ public Simulation getSimulation(User user, KeyValue simKey) throws DataAccessExc @Override public FieldDataIdentifierSpec[] getFieldDataIdentifierSpecs(Simulation sim) throws DataAccessException { - throw new DataAccessException(); + return null; } @Override From b40dee9bec9ac3f5a8a97bfb0c4b2952fdb27328 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Fri, 23 Aug 2024 14:00:26 -0400 Subject: [PATCH 07/24] Tests for each state the State Machine can Enter --- .../main/java/cbit/rmi/event/WorkerEvent.java | 6 + .../SimulationStateMachineTest.java | 262 ++++++++++++++---- 2 files changed, 220 insertions(+), 48 deletions(-) diff --git a/vcell-core/src/main/java/cbit/rmi/event/WorkerEvent.java b/vcell-core/src/main/java/cbit/rmi/event/WorkerEvent.java index 3c5c4a2392..660d7e9ce6 100644 --- a/vcell-core/src/main/java/cbit/rmi/event/WorkerEvent.java +++ b/vcell-core/src/main/java/cbit/rmi/event/WorkerEvent.java @@ -19,6 +19,8 @@ import cbit.vcell.solver.VCSimulationIdentifier; import cbit.vcell.solver.server.SimulationMessage; +import java.util.ArrayList; + /** * Insert the type's description here. * Creation date: (2/5/2004 12:35:20 PM) @@ -36,6 +38,10 @@ public class WorkerEvent extends MessageEvent { public static final int JOB_WORKER_EXIT_NORMAL = 1015; public static final int JOB_WORKER_EXIT_ERROR = 1016; + public static final ArrayList ALL_JOB_EVENTS = new ArrayList<>(){{add(JOB_ACCEPTED); + add(JOB_STARTING); add(JOB_DATA); add(JOB_PROGRESS); add(JOB_FAILURE); add(JOB_COMPLETED); add(JOB_WORKER_ALIVE); + add(JOB_WORKER_EXIT_NORMAL); add(JOB_WORKER_EXIT_ERROR);}}; + private VCSimulationIdentifier vcSimulationIdentifier = null; private int jobIndex = -1; private String hostName = null; diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java index 4f88d335e5..2fac63ed39 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java @@ -1,18 +1,29 @@ package cbit.vcell.message.server.dispatcher; import cbit.rmi.event.WorkerEvent; +import cbit.vcell.geometry.Geometry; +import cbit.vcell.mapping.MathSymbolMapping; +import cbit.vcell.math.*; +import cbit.vcell.mathmodel.MathModel; +import cbit.vcell.message.VCMessageSession; +import cbit.vcell.message.VCMessagingException; +import cbit.vcell.message.local.LocalVCMessageAdapter; +import cbit.vcell.message.messages.StatusMessage; +import cbit.vcell.parser.ExpressionBindingException; +import cbit.vcell.resource.PropertyLoader; +import cbit.vcell.server.HtcJobID; +import cbit.vcell.server.SimulationExecutionStatus; import cbit.vcell.server.SimulationJobStatus; +import cbit.vcell.solver.MeshSpecification; +import cbit.vcell.solver.Simulation; import cbit.vcell.solver.VCSimulationIdentifier; import cbit.vcell.solver.server.SimulationMessage; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; import org.vcell.util.DataAccessException; -import org.vcell.util.document.KeyValue; -import org.vcell.util.document.User; -import org.vcell.util.document.VCellServerID; +import org.vcell.util.ISize; +import org.vcell.util.document.*; +import java.beans.PropertyVetoException; import java.sql.SQLException; import java.time.Instant; import java.util.ArrayList; @@ -23,70 +34,121 @@ public class SimulationStateMachineTest { private static final VCellServerID testVCellServerID = VCellServerID.getServerID("test"); private static final User testUser = new User("Alice", new KeyValue("0")); + private static final VCMessageSession testMessageSession = new LocalVCMessageAdapter(null); + private static final int jobIndex = 0; + private static final int taskID = 0; + private static final KeyValue simKey = new KeyValue("1"); + private static final VCSimulationIdentifier simID = new VCSimulationIdentifier(simKey, testUser); + public static String previousServerID = ""; private MockSimulationDB simulationDB; + private SimulationStateMachineCopy stateMachine; + + @BeforeAll + public static void setSystemProperties(){ + previousServerID = PropertyLoader.getProperty(PropertyLoader.vcellServerIDProperty, ""); + PropertyLoader.setProperty(PropertyLoader.vcellServerIDProperty, ""); + } + + @AfterAll + public static void restoreSystemProperties(){ + PropertyLoader.setProperty(PropertyLoader.vcellServerIDProperty, previousServerID); + } @BeforeEach public void setUp(){ simulationDB = new MockSimulationDB(); + stateMachine = new SimulationStateMachineCopy(simKey, jobIndex); } private record ChangedStateValues( VCSimulationIdentifier simID, SimulationJobStatus.SchedulerStatus schedulerStatus, + int workerEventJob, int taskID, String changesResult ){ } - private interface WorkerEventFactory { - WorkerEvent createChangedWorker(ChangedStateValues c); + private WorkerEvent createWorkerEvent(ChangedStateValues w){ + SimulationMessage acceptedSimulationMessage = SimulationMessage.workerAccepted("accepted"); + return new WorkerEvent(w.workerEventJob, simKey, + w.simID, jobIndex, "", + w.taskID, null, null, + acceptedSimulationMessage); } - @Test - public void workerEventRejectionsTest() throws SQLException, DataAccessException { - KeyValue simKey = new KeyValue("1"); - VCSimulationIdentifier simID = new VCSimulationIdentifier(simKey, testUser); - int jobIndex = 0; - int taskID = 16; - SimulationStateMachineCopy stateMachine = new SimulationStateMachineCopy(simKey, jobIndex); - SimulationMessage acceptedSimulationMessage = SimulationMessage.workerAccepted("accepted"); + private Simulation createMockSimulation(int iSizeX, int iSizeY, int iSizeZ) throws PropertyVetoException, MathException, ExpressionBindingException { + VolVariable volVariable = new VolVariable("t", new Variable.Domain(new CompartmentSubDomain("t", 1))); + VolVariable volVariable2 = new VolVariable("b", new Variable.Domain(new CompartmentSubDomain("b", 2))); + MathSymbolMapping mathSymbolMapping = new MathSymbolMapping(); + Geometry geometry = new Geometry("T", 3); + MathModel mathModel = new MathModel(new Version("Test", testUser)); + MathDescription mathDescription = new MathDescription("Test", mathSymbolMapping); + mathDescription.setGeometry(new Geometry("T", 3)); + Simulation simulation = new Simulation(SimulationVersion.createTempSimulationVersion(), + mathDescription, mathModel); + MeshSpecification meshSpecification = new MeshSpecification(geometry); + meshSpecification.setSamplingSize(new ISize(iSizeX, iSizeY, iSizeZ)); + simulation.setMeshSpecification(meshSpecification); + mathDescription.setAllVariables(new Variable[]{volVariable, volVariable2}); + return simulation; + } + private void insertOrUpdateStatus(KeyValue simKey, int jobIndex, int taskID, User user, SimulationJobStatus.SchedulerStatus status) throws SQLException, DataAccessException { + SimulationJobStatus jobStatus = simulationDB.getLatestSimulationJobStatus(simKey, jobIndex); + VCSimulationIdentifier simID = new VCSimulationIdentifier(simKey, user); + SimulationJobStatus simulationJobStatus = new SimulationJobStatus(testVCellServerID, simID, jobIndex, Date.from(Instant.now()), status, taskID, + SimulationMessage.workerAccepted("accepted"), null, + new SimulationExecutionStatus(Date.from(Instant.now()), "", + Date.from(Instant.now()), Date.from(Instant.now()), false, new HtcJobID("2", HtcJobID.BatchSystemType.SLURM))); + if (jobStatus == null){ + simulationDB.insertSimulationJobStatus(simulationJobStatus); + } else { + simulationDB.updateSimulationJobStatus(simulationJobStatus); + } + } - WorkerEventFactory workerEventFactory = (w) -> {return new WorkerEvent(WorkerEvent.JOB_ACCEPTED, simKey, w.simID, - jobIndex, "", - w.taskID, null, null, - acceptedSimulationMessage);}; + private void insertOrUpdateStatus(KeyValue simKey, int jobIndex, int taskID, User user) throws SQLException, DataAccessException { + insertOrUpdateStatus(simKey, jobIndex, taskID, user, SimulationJobStatus.SchedulerStatus.RUNNING); + } + + /** + Defaults to a running status. + */ + private void insertOrUpdateStatus() throws SQLException, DataAccessException { + insertOrUpdateStatus(simKey, jobIndex, taskID, testUser); + } + private SimulationJobStatus getLatestJobSubmission() throws SQLException, DataAccessException { + return simulationDB.getLatestSimulationJobStatus(simKey, jobIndex); + } + + @Test + public void workerEventRejectionsTest() throws SQLException, DataAccessException { + int taskID = 16; ArrayList changedValues = new ArrayList<>(){{ - add(new ChangedStateValues(simID, SimulationJobStatus.SchedulerStatus.RUNNING, taskID, "No old status.")); // no old status failure - add(new ChangedStateValues(simID, SimulationJobStatus.SchedulerStatus.COMPLETED, taskID, "Work is already done.")); // work is done failure - add(new ChangedStateValues(simID, SimulationJobStatus.SchedulerStatus.RUNNING, 0, "Task ID is lower")); // old status has higher number taskID failure + add(new ChangedStateValues(simID, SimulationJobStatus.SchedulerStatus.RUNNING, WorkerEvent.JOB_WORKER_ALIVE, taskID, "No old status.")); // no old status failure + add(new ChangedStateValues(simID, SimulationJobStatus.SchedulerStatus.COMPLETED, WorkerEvent.JOB_WORKER_ALIVE, taskID, "Work is already done.")); // work is done failure + add(new ChangedStateValues(simID, SimulationJobStatus.SchedulerStatus.RUNNING, WorkerEvent.JOB_WORKER_ALIVE, 0, "Task ID is lower")); // old status has higher number taskID failure }}; for (int i = 0; i < changedValues.size(); i++){ ChangedStateValues workerEventChangedValues = changedValues.get(i); - if (i == 1){ - simulationDB.insertSimulationJobStatus(new SimulationJobStatus(testVCellServerID, simID, jobIndex, Date.from(Instant.now()), workerEventChangedValues.schedulerStatus, taskID, - acceptedSimulationMessage, null, null)); - } else if (i > 1) { - SimulationJobStatus failingStatus = new SimulationJobStatus(testVCellServerID, simID, 0, Date.from(Instant.now()), workerEventChangedValues.schedulerStatus, taskID, - acceptedSimulationMessage, null, null); - simulationDB.updateSimulationJobStatus(failingStatus); + if (i > 1) { + insertOrUpdateStatus(simKey, jobIndex, taskID, testUser, workerEventChangedValues.schedulerStatus); } - WorkerEvent workerEvent = workerEventFactory.createChangedWorker(workerEventChangedValues); + WorkerEvent workerEvent = createWorkerEvent(workerEventChangedValues); Assertions.assertFalse(stateMachine.isWorkerEventOkay(workerEvent, simulationDB), workerEventChangedValues.changesResult); } - ChangedStateValues passingWorkerValues= new ChangedStateValues(simID, null, taskID, ""); - WorkerEvent passingWorkerEvent = workerEventFactory.createChangedWorker(passingWorkerValues); + ChangedStateValues passingWorkerValues= new ChangedStateValues(simID, null, WorkerEvent.JOB_WORKER_ALIVE , taskID, ""); + WorkerEvent passingWorkerEvent = createWorkerEvent(passingWorkerValues); for (SimulationJobStatus.SchedulerStatus passingStatus: SimulationJobStatus.SchedulerStatus.values()){ if (!passingStatus.isDone()){ - SimulationJobStatus simulationJobStatus = new SimulationJobStatus(testVCellServerID, simID, jobIndex, Date.from(Instant.now()), passingStatus, taskID, - acceptedSimulationMessage, null, null); - simulationDB.updateSimulationJobStatus(simulationJobStatus); + insertOrUpdateStatus(simKey, jobIndex, taskID, testUser, passingStatus); Assertions.assertTrue(stateMachine.isWorkerEventOkay(passingWorkerEvent, simulationDB)); } } @@ -94,28 +156,132 @@ public void workerEventRejectionsTest() throws SQLException, DataAccessException } @Test - public void stateShouldTransitionToFailure(){ - KeyValue simKey = new KeyValue("1"); - VCSimulationIdentifier simID = new VCSimulationIdentifier(simKey, testUser); - int jobIndex = 0; - int taskID = 16; - SimulationStateMachineCopy stateMachine = new SimulationStateMachineCopy(simKey, jobIndex); - } + public void stateShouldTransitionToFailure() throws SQLException, DataAccessException, VCMessagingException, PropertyVetoException, MathException, ExpressionBindingException { + ArrayList changedValues = new ArrayList<>(){{ + add(new ChangedStateValues(simID, null, WorkerEvent.JOB_FAILURE, taskID, "The current worker has failed.")); + add(new ChangedStateValues(simID, null, WorkerEvent.JOB_WORKER_EXIT_ERROR, taskID, "The current worker exited with an error.")); + }}; - public void stateShouldTransitionToWaiting(){ + for (ChangedStateValues changedValue : changedValues){ + insertOrUpdateStatus(); + stateMachine.onWorkerEvent(createWorkerEvent(changedValue), simulationDB, null); + SimulationJobStatus result = getLatestJobSubmission(); + Assertions.assertTrue(result.getSchedulerStatus().isFailed(), changedValue.changesResult); + } + simulationDB = new MockSimulationDB(); + StatusMessage statusMessage = stateMachine.onStartRequest(new User("Bob", new KeyValue("1")), simID, simulationDB, null); + Assertions.assertTrue(statusMessage.getSimulationJobStatus().getSchedulerStatus().isFailed(), "Different from initial user that owns the simulation"); + + SimulationJobStatus jobStatus = getLatestJobSubmission(); + Assertions.assertNull(jobStatus, "If it fails on start request, there should be nothing in the DB."); + + insertOrUpdateStatus(); + Assertions.assertThrows(RuntimeException.class, + () -> {stateMachine.onStartRequest(testUser, simID, simulationDB, testMessageSession);}, + "Can't start simulation job unless previous is done."); + + insertOrUpdateStatus(); + jobStatus = getLatestJobSubmission(); + stateMachine.onSystemAbort(jobStatus, "Test Abort", simulationDB, testMessageSession); + jobStatus = getLatestJobSubmission(); + Assertions.assertTrue(jobStatus.getSchedulerStatus().isFailed()); + +// + Simulation memoryIntensiveSimulation = createMockSimulation(900, 900, 900); + + insertOrUpdateStatus(); + Assertions.assertThrows(RuntimeException.class, + () -> {stateMachine.onDispatch(memoryIntensiveSimulation, getLatestJobSubmission(), simulationDB, testMessageSession);}, + "Can't dispatch simulation that is already running."); + + insertOrUpdateStatus(simKey, jobIndex, taskID, testUser, SimulationJobStatus.SchedulerStatus.WAITING); + stateMachine.onDispatch(memoryIntensiveSimulation, getLatestJobSubmission(), simulationDB, testMessageSession); + + jobStatus = getLatestJobSubmission(); + Assertions.assertTrue(jobStatus.getSchedulerStatus().isFailed(), "Memory size to large"); + + insertOrUpdateStatus(); + statusMessage = stateMachine.onStopRequest(new User("Bob", new KeyValue("2")), getLatestJobSubmission(), simulationDB, testMessageSession); + Assertions.assertTrue(statusMessage.getSimulationJobStatus().getSchedulerStatus().isFailed(), "Stopping as another user."); + } + + @Test + public void stateShouldTransitionToWaiting() throws SQLException, VCMessagingException, DataAccessException { + stateMachine.onStartRequest(testUser, simID, simulationDB, testMessageSession); + SimulationJobStatus jobStatus = getLatestJobSubmission(); + Assertions.assertTrue(jobStatus.getSchedulerStatus().isWaiting(), "Just started new task."); } - public void stateShouldTransitionToDispatched(){ + @Test + public void stateShouldTransitionToDispatched() throws SQLException, DataAccessException, VCMessagingException, PropertyVetoException, MathException, ExpressionBindingException { + insertOrUpdateStatus(simKey, jobIndex, taskID, testUser, SimulationJobStatus.SchedulerStatus.WAITING); + WorkerEvent acceptedWorker = createWorkerEvent(new ChangedStateValues(simID, null, WorkerEvent.JOB_ACCEPTED, taskID, "Worker just got accepted")); + stateMachine.onWorkerEvent(acceptedWorker, simulationDB, testMessageSession); + SimulationJobStatus jobStatus = getLatestJobSubmission(); + Assertions.assertTrue(jobStatus.getSchedulerStatus().isDispatched(), "Job recently got accepted, only works if previous state was waiting."); + + insertOrUpdateStatus(); + stateMachine.onWorkerEvent(acceptedWorker, simulationDB, testMessageSession); + jobStatus = getLatestJobSubmission(); + Assertions.assertTrue(jobStatus.getSchedulerStatus().isRunning(), "The state has not changed from running, because something that is running can not be dispatched."); + + insertOrUpdateStatus(simKey, jobIndex, taskID, testUser, SimulationJobStatus.SchedulerStatus.WAITING); + Simulation simulation = createMockSimulation(50, 50, 50); + stateMachine.onDispatch(simulation, getLatestJobSubmission(), simulationDB, testMessageSession); + jobStatus = getLatestJobSubmission(); + Assertions.assertTrue(jobStatus.getSchedulerStatus().isDispatched()); } - public void stateShouldTransitionToRunning(){ + @Test + public void stateShouldTransitionToRunning() throws SQLException, DataAccessException, VCMessagingException { + for (int workerStatus: WorkerEvent.ALL_JOB_EVENTS){ + WorkerEvent workerEvent = createWorkerEvent(new ChangedStateValues(simID, null, workerStatus, taskID, "")); + insertOrUpdateStatus(simKey, jobIndex, taskID, testUser, SimulationJobStatus.SchedulerStatus.WAITING); + stateMachine.onWorkerEvent(workerEvent, simulationDB, testMessageSession); + SimulationJobStatus jobStatus = getLatestJobSubmission(); + if (workerEvent.isProgressEvent() || workerEvent.isNewDataEvent() || workerEvent.isStartingEvent() || workerEvent.isWorkerAliveEvent()){ + Assertions.assertTrue(jobStatus.getSchedulerStatus().isRunning()); + } else { + Assertions.assertFalse(jobStatus.getSchedulerStatus().isRunning()); + } + } + } + @Test + public void stateShouldTransitionToCompleted() throws SQLException, VCMessagingException, DataAccessException { + for (int workerStatus : WorkerEvent.ALL_JOB_EVENTS){ + WorkerEvent workerEvent = createWorkerEvent(new ChangedStateValues(simID, SimulationJobStatus.SchedulerStatus.RUNNING, workerStatus, taskID, "")); + insertOrUpdateStatus(); + stateMachine.onWorkerEvent(workerEvent, simulationDB, testMessageSession); + SimulationJobStatus jobStatus = getLatestJobSubmission(); + if (workerEvent.isCompletedEvent()){ + Assertions.assertTrue(jobStatus.getSchedulerStatus().isCompleted()); + } else { + Assertions.assertFalse(jobStatus.getSchedulerStatus().isCompleted()); + } + } } - public void stateShouldTransitionToCompleted(){ + @Test + public void stateShouldTransitionToStopped() throws SQLException, DataAccessException, VCMessagingException { + + for (SimulationJobStatus.SchedulerStatus status : SimulationJobStatus.SchedulerStatus.values()){ + insertOrUpdateStatus(simKey,jobIndex, taskID,testUser, status); + if (status.isActive()){ + stateMachine.onStopRequest(testUser, getLatestJobSubmission(), simulationDB, testMessageSession); + Assertions.assertTrue(getLatestJobSubmission().getSchedulerStatus().isStopped(), ""); + } else { + StatusMessage statusMessage = stateMachine.onStopRequest(testUser, getLatestJobSubmission(), simulationDB, testMessageSession); + Assertions.assertNull(statusMessage); + } + } + } + @Test + public void stateShouldTransitionToQueued(){ + System.out.print("Not used in state machine"); } From 1c88e93b43d90708b80d5f5f2ccdc7ad7a82f05f Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Tue, 27 Aug 2024 12:45:54 -0400 Subject: [PATCH 08/24] Partial Fix to Memory Allocation Bug Fix the memory allocation bug which allowed for users to request any amount of memory from slurm. Slurm may reject their request, but the filter placed on VCell was not stoping these requests to begin with. Fixed it for the simulation state machine, but SlurmProxy still requires some changes. --- .../cbit/vcell/resource/PropertyLoader.java | 3 + .../dispatcher/SimulationStateMachine.java | 16 +- .../SimulationStateMachineCopy.java | 16 +- .../vcell/message/server/htc/HtcProxy.java | 225 ++---------------- .../SimulationStateMachineTest.java | 15 ++ 5 files changed, 46 insertions(+), 229 deletions(-) diff --git a/vcell-core/src/main/java/cbit/vcell/resource/PropertyLoader.java b/vcell-core/src/main/java/cbit/vcell/resource/PropertyLoader.java index 1897bb9dde..041c885eb8 100644 --- a/vcell-core/src/main/java/cbit/vcell/resource/PropertyLoader.java +++ b/vcell-core/src/main/java/cbit/vcell/resource/PropertyLoader.java @@ -80,6 +80,9 @@ public static void setConfigProvider(VCellConfigProvider configProvider) { public static final String htcPbsHome = record("vcell.htc.pbs.home",ValueType.GEN); public static final String htcSgeHome = record("vcell.htc.sge.home",ValueType.GEN); public static final String htcNodeList = record("vcell.htc.nodelist",ValueType.GEN); + public static final String htcMinMemoryMB = record("vcell.htc.memory.min.mb", ValueType.INT); // minimum memory request in MB, currently 4g + public static final String htcMaxMemoryMB = record("vcell.htc.memory.max.mb", ValueType.INT); // maximum memory request in MB + public static final String htcPowerUserMemoryFloorMB = record("vcell.htc.memory.pu.floor.mb", ValueType.INT); // MIN memory allowed if declared to be a power user, currently 50g (Previously Existing Value) public static final String slurm_cmd_sbatch = record("vcell.slurm.cmd.sbatch",ValueType.GEN); public static final String slurm_cmd_scancel = record("vcell.slurm.cmd.scancel",ValueType.GEN); public static final String slurm_cmd_sacct = record("vcell.slurm.cmd.sacct",ValueType.GEN); diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java index 3df01647e5..670c3ea9c4 100644 --- a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java @@ -4,6 +4,7 @@ import java.util.Arrays; import java.util.Date; +import cbit.vcell.resource.PropertyLoader; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vcell.util.DataAccessException; @@ -482,16 +483,13 @@ public synchronized void onDispatch(Simulation simulation, SimulationJobStatus o } SimulationTask simulationTask = new SimulationTask(new SimulationJob(simulation, jobIndex, fieldDataIdentifierSpecs), taskID,null,isPowerUser); - double requiredMemMB = simulationTask.getEstimatedMemorySizeMB(); - //SimulationStateMachine ultimately instantiated from {vcellroot}/docker/build/Dockerfile-sched-dev by way of cbit.vcell.message.server.dispatcher.SimulationDispatcher - String vcellUserid = simulationTask.getUser().getName(); - KeyValue simID = simulationTask.getSimulationInfo().getSimulationVersion().getVersionKey(); - SolverDescription solverDescription = simulationTask.getSimulation().getSolverTaskDescription().getSolverDescription(); + double estimatedMemMB = simulationTask.getEstimatedMemorySizeMB(); + double htcMinMemoryMB = Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.htcMinMemoryMB)); + double htcMaxMemoryMB = Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.htcMaxMemoryMB)); + double requestedMemoryMB = Math.max(estimatedMemMB, htcMinMemoryMB); - MemLimitResults allowableMemMB = HtcProxy.getMemoryLimit(vcellUserid,simID,solverDescription, requiredMemMB, isPowerUser); - final SimulationJobStatus newSimJobStatus; - if (requiredMemMB > allowableMemMB.getMemLimit()) { + if (requestedMemoryMB > htcMaxMemoryMB) { // // fail the simulation // @@ -501,7 +499,7 @@ public synchronized void onDispatch(Simulation simulation, SimulationJobStatus o SimulationExecutionStatus newSimExeStatus = new SimulationExecutionStatus(null, null, new Date(), null, false, null); newSimJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(),vcSimID,jobIndex, oldSimulationJobStatus.getSubmitDate(),SchedulerStatus.FAILED,taskID, - SimulationMessage.jobFailed("simulation required "+requiredMemMB+"MB of memory, only "+allowableMemMB.getMemLimit()+"MB allowed from "+allowableMemMB.getMemLimitSource()), + SimulationMessage.jobFailed("simulation required "+estimatedMemMB+"MB of memory, only "+htcMaxMemoryMB+"MB allowed"), newQueueStatus,newSimExeStatus); simulationDatabase.updateSimulationJobStatus(newSimJobStatus); diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineCopy.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineCopy.java index f87a291280..347ca70708 100644 --- a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineCopy.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineCopy.java @@ -13,6 +13,7 @@ import cbit.vcell.message.server.htc.HtcProxy; import cbit.vcell.messaging.server.SimulationTask; import cbit.vcell.mongodb.VCMongoMessage; +import cbit.vcell.resource.PropertyLoader; import cbit.vcell.server.*; import cbit.vcell.solver.*; import cbit.vcell.solver.server.SimulationMessage; @@ -458,16 +459,13 @@ public synchronized void onDispatch(Simulation simulation, SimulationJobStatus o } SimulationTask simulationTask = new SimulationTask(new SimulationJob(simulation, jobIndex, fieldDataIdentifierSpecs), taskID,null,isPowerUser); - double requiredMemMB = simulationTask.getEstimatedMemorySizeMB(); - //SimulationStateMachine ultimately instantiated from {vcellroot}/docker/build/Dockerfile-sched-dev by way of cbit.vcell.message.server.dispatcher.SimulationDispatcher - String vcellUserid = simulationTask.getUser().getName(); - KeyValue simID = simulationTask.getSimulationInfo().getSimulationVersion().getVersionKey(); - SolverDescription solverDescription = simulationTask.getSimulation().getSolverTaskDescription().getSolverDescription(); - - HtcProxy.MemLimitResults allowableMemMB = HtcProxy.getMemoryLimit(vcellUserid,simID,solverDescription, requiredMemMB, isPowerUser); + double estimatedMemMB = simulationTask.getEstimatedMemorySizeMB(); + double htcMinMemoryMB = Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.htcMinMemoryMB)); + double htcMaxMemoryMB = Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.htcMaxMemoryMB)); + double requestedMemoryMB = Math.max(estimatedMemMB, htcMinMemoryMB); final SimulationJobStatus newSimJobStatus; - if (requiredMemMB > allowableMemMB.getMemLimit()) { + if (requestedMemoryMB > htcMaxMemoryMB) { // // fail the simulation // @@ -477,7 +475,7 @@ public synchronized void onDispatch(Simulation simulation, SimulationJobStatus o SimulationExecutionStatus newSimExeStatus = new SimulationExecutionStatus(null, null, new Date(), null, false, null); newSimJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(),vcSimID,jobIndex, oldSimulationJobStatus.getSubmitDate(), SimulationJobStatus.SchedulerStatus.FAILED,taskID, - SimulationMessage.jobFailed("simulation required "+requiredMemMB+"MB of memory, only "+allowableMemMB.getMemLimit()+"MB allowed from "+allowableMemMB.getMemLimitSource()), + SimulationMessage.jobFailed("simulation required "+estimatedMemMB+"MB of memory, only "+htcMaxMemoryMB+"MB allowed"), newQueueStatus,newSimExeStatus); simulationDatabase.updateSimulationJobStatus(newSimJobStatus); diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/htc/HtcProxy.java b/vcell-server/src/main/java/cbit/vcell/message/server/htc/HtcProxy.java index cb19ed6884..b47e9cf352 100644 --- a/vcell-server/src/main/java/cbit/vcell/message/server/htc/HtcProxy.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/htc/HtcProxy.java @@ -18,7 +18,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.vcell.util.BeanUtils; import org.vcell.util.document.KeyValue; import org.vcell.util.exe.ExecutableException; @@ -251,8 +250,8 @@ public static void writeUnixStyleTextFile(File file, String javaString) throws I public abstract String getSubmissionFileExtension(); public static class MemLimitResults { - private static final long FALLBACK_MEM_LIMIT_MB=4096; // MAX memory allowed if not set in limitFile, currently 4g - private static final long POWER_USER_MEMORY_FLOOR=51200; // MIN memory allowed if declared to be a power user, currently 50g + private static final long FALLBACK_MEM_LIMIT_MB= Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.htcMinMemoryMB)); // MAX memory allowed if not set in limitFile, currently 4g + private static final long POWER_USER_MEMORY_FLOOR=Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.htcPowerUserMemoryFloorMB)); // MIN memory allowed if declared to be a power user, currently 50g private long memLimit; private String memLimitSource; public MemLimitResults(long memLimit, String memLimitSource) { @@ -266,223 +265,27 @@ public long getMemLimit() { public String getMemLimitSource() { return memLimitSource; } - private static MemLimitResults getFallbackMemLimitMB(SolverDescription solverDescription,double estimatedMemSizeMB, boolean isPowerUser) { - Long result = null; - String source = null; - try { - List solverMemLimits = Files.readAllLines(Paths.get(new File("/"+System.getProperty(PropertyLoader.htcLogDirInternal)+"/slurmMinMem.txt").getAbsolutePath())); - for (Iterator iterator = solverMemLimits.iterator(); iterator.hasNext();) { - String solverAndLimit = iterator.next().trim(); - if(solverAndLimit.length()==0 || solverAndLimit.startsWith("//")) { - continue; - } - StringTokenizer st = new StringTokenizer(solverAndLimit,":"); - String limitSolver = st.nextToken(); - if(limitSolver.equalsIgnoreCase("all") && result == null) {//use all if there is not solver matching name in slurmMinMem.txt - result = Long.parseLong(st.nextToken()); - source = "used slurmMinMem.txt all"; - }else if(solverDescription != null && limitSolver.equals(solverDescription.name())) {//use matching solver mem limit from file - result = Long.parseLong(st.nextToken()); - source = "used slurmMinMem.txt "+solverDescription.name(); - break; - } - } - if(result == null) {//empty slurmMinMem.txt - result = FALLBACK_MEM_LIMIT_MB; - source = "Empty used FALLBACK_MEM_LIMIT_MB"; - } - } catch (Exception e) { - LG.debug(e); - result = FALLBACK_MEM_LIMIT_MB; - source = "Exception "+e.getClass().getSimpleName()+" used FALLBACK_MEM_LIMIT_MB"; - } - if(estimatedMemSizeMB > result) {//Use estimated if bigger - result = (long)estimatedMemSizeMB; - source = "used Estimated"; + private static MemLimitResults getJobRequestedMemoryLimit(SolverDescription solverDescription, double estimatedMemSizeMB, boolean isPowerUser) { + long batchJobMemoryLimit = FALLBACK_MEM_LIMIT_MB; + String detailedMessage = "default memory limit"; + + if(estimatedMemSizeMB > batchJobMemoryLimit) {//Use estimated if bigger + batchJobMemoryLimit = (long)estimatedMemSizeMB; + detailedMessage = "used Estimated"; } - if (isPowerUser && result < POWER_USER_MEMORY_FLOOR){ - result = (long)POWER_USER_MEMORY_FLOOR; - source = "poweruser's memory override"; + if (isPowerUser && batchJobMemoryLimit < POWER_USER_MEMORY_FLOOR){ + batchJobMemoryLimit = POWER_USER_MEMORY_FLOOR; + detailedMessage = "poweruser's memory override"; } - return new MemLimitResults(result, source); + return new MemLimitResults(batchJobMemoryLimit, detailedMessage); } } public static final boolean bDebugMemLimit = false; public static MemLimitResults getMemoryLimit(String vcellUserid, KeyValue simID, SolverDescription solverDescription ,double estimatedMemSizeMB, boolean isPowerUser) { - return MemLimitResults.getFallbackMemLimitMB(solverDescription, estimatedMemSizeMB*1.5, isPowerUser); -// boolean bUseEstimate = estimatedMemSizeMB >= MemLimitResults.getFallbackMemLimitMB(solverDescription); -// return new MemLimitResults((bUseEstimate?(long)estimatedMemSizeMB:MemLimitResults.getFallbackMemLimitMB(solverDescription)), (bUseEstimate?"used Estimated":"used FALLBACK_MEM_LIMIT")); -// //One of 5 limits are returned (ordered from highest to lowest priority): -// // MemoryMax:PerSimulation Has PropertyLoader.simPerUserMemoryLimitFile, specific user AND simID MATCHED in file (userid MemLimitMb simID) -// // MemoryMax:PerUser Has PropertyLoader.simPerUserMemoryLimitFile, specific user (but not simID) MATCHED in file (userid MemLimitMb '*') -// // MemoryMax:PerSolver Has PropertyLoader.simPerUserMemoryLimitFile, specific solverDescription (but not simID or user) MATCHED in file (solverName MemLimitMb '*') -// // MemoryMax:SimulationTask.getEstimatedMemorySizeMB() Has PropertyLoader.simPerUserMemoryLimitFile, no user or sim MATCHED in file ('defaultSimMemoryLimitMb' MemLimitMb '*') -// // estimated > MemoryMax:AllUsersMemLimit -// // MemoryMax:AllUsersMemLimit(defaultSimMemoryLimitMb) Has PropertyLoader.simPerUserMemoryLimitFile, no user or sim MATCHED in file ('defaultSimMemoryLimitMb' MemLimitMb '*') -// // estimated < MemoryMax:AllUsersMemLimit -// // MemoryMax:HtcProxy.MemLimitResults.FALLBACK_MEM_LIMIT No PropertyLoader.simPerUserMemoryLimitFile -// // estimated < FALLBACK -// -// Long defaultSimMemoryLimitMbFromFile = null; -// File memLimitFile = null; -// try { -// //${vcellroot}/docker/swarm/serverconfig-uch.sh->VCELL_SIMDATADIR_EXTERNAL=/share/apps/vcell3/users -// //${vcellroot}/docker/swarm/serverconfig-uch.sh-> VCELL_SIMDATADIR_HOST=/opt/vcelldata/users -// //${vcellroot}/docker/swarm/docker-compose.yml-> Volume map "${VCELL_SIMDATADIR_HOST}:/simdata" -// Long perUserMemMax = null; -// Long perSimMemMax = null; -// Long perSolverMax = null; -// String memLimitFileDirVal = System.getProperty(PropertyLoader.primarySimDataDirInternalProperty); -// String memLimitFileVal = System.getProperty(PropertyLoader.simPerUserMemoryLimitFile); -// if(memLimitFileDirVal != null && memLimitFileVal != null) { -// memLimitFile = new File(memLimitFileDirVal,memLimitFileVal); -// } -// if(memLimitFile != null && memLimitFile.exists()) { -// List perUserLimits = Files.readAllLines(Paths.get(memLimitFile.getAbsolutePath())); -// for (Iterator iterator = perUserLimits.iterator(); iterator.hasNext();) { -// String userAndLimit = iterator.next().trim(); -// if(userAndLimit.length()==0 || userAndLimit.startsWith("//")) { -// if(bDebugMemLimit){LG.trace("-----skipped '"+userAndLimit+"'");} -// continue; -// } -//// LG.trace("-----"+userAndLimit); -// -// StringTokenizer st = new StringTokenizer(userAndLimit); -// String limitUserid = st.nextToken(); -// if(limitUserid.equals(vcellUserid) || (solverDescription != null && limitUserid.equals(solverDescription.name()))) {//check user -// long memLimit = 0; -// try { -// memLimit = Long.parseLong(st.nextToken()); -// } catch (Exception e) { -// if(bDebugMemLimit){LG.debug("-----ERROR '"+userAndLimit+"' token memlimit not parsed");} -// //bad line in limit file, continue processing other lines -// //lg.debug(e); -// continue; -// } -// if(solverDescription != null && limitUserid.equals(solverDescription.name())) { -// perSolverMax = memLimit; -// if(bDebugMemLimit){LG.debug("-----"+"MATCH Solver "+userAndLimit);} -// continue; -// } -// //get simid -// String simSpecifier = null; -// try { -// simSpecifier = st.nextToken(); -// //check token is '*' or long -// if(!simSpecifier.equals("*") && Long.valueOf(simSpecifier).longValue() < 0 ) { -// throw new Exception(" token 'simSpecifier' expected to be '*' or simID"); -// } -// } catch (Exception e) { -// if(bDebugMemLimit){LG.debug("-----ERROR '"+userAndLimit+"' "+e.getClass().getName()+" "+e.getMessage());} -// //bad line in limit file, continue processing other lines -// //lg.debug(e); -// continue; -// } -// // * means all sims for that user, don't set if sim specific limit is already set -// if(simSpecifier.equals("*") && perSimMemMax == null) { -// perUserMemMax = memLimit;// use this unless overriden by specific simid -// if(bDebugMemLimit){LG.debug("-----"+"MATCH USER "+userAndLimit);} -// } -// //Set sim specific limit, set even if * limit has been set -// if(simID != null && simID.toString().equals(simSpecifier)) { -// perSimMemMax = memLimit;// use sim limit -// if(bDebugMemLimit){LG.debug("-----"+"MATCH SIM "+userAndLimit);} -// } -// }else if(limitUserid.equals("defaultSimMemoryLimitMb")) {//Master sim mem limit -// try { -// defaultSimMemoryLimitMbFromFile = Long.parseLong(st.nextToken()); -// if(bDebugMemLimit){LG.debug("-----"+"MATCH DEFAULT "+userAndLimit);} -// } catch (Exception e) { -// if(bDebugMemLimit){LG.debug("-----ERROR '"+userAndLimit+"' "+e.getClass().getName()+" "+e.getMessage());} -// //bad line in limit file, continue processing other lines -// //LG.debug(e); -// continue; -// } -// }else { -// if(bDebugMemLimit){LG.debug("-----"+"NO MATCH "+userAndLimit);} -// } -// } -// if(perUserMemMax != null || perSimMemMax != null) { -// long finalMax = (perSimMemMax!=null?perSimMemMax:perUserMemMax); -// if(bDebugMemLimit){LG.debug("Set memory limit for user '"+vcellUserid+"' to "+finalMax + (perSimMemMax!=null?" for simID="+simID:""));} -// return new MemLimitResults(finalMax, -// (perSimMemMax!=null? -// "MemoryMax(FILE PerSimulation):"+simID+",User='"+vcellUserid+"' from "+memLimitFile.getAbsolutePath(): -// "MemoryMax(FILE PerUser):'"+vcellUserid+"' from "+memLimitFile.getAbsolutePath())); -// }else if(perSolverMax != null) { -// if(perSolverMax == 0) {//Use estimated size always if solver had 0 for memory limit -// return new MemLimitResults( -// Math.max((long)Math.ceil(estimatedMemSizeMB*1.5), -// (defaultSimMemoryLimitMbFromFile!=null?defaultSimMemoryLimitMbFromFile:MemLimitResults.FALLBACK_MEM_LIMIT_MB)), -// "MemoryMax(FILE PerSolver ESTIMATED):'"+solverDescription.name()+"' from "+memLimitFile.getAbsolutePath()); -// }else { -// return new MemLimitResults(perSolverMax, "MemoryMax(FILE PerSolver):'"+solverDescription.name()+"' from "+memLimitFile.getAbsolutePath()); -// } -// } -// }else { -// if(bDebugMemLimit){LG.debug("-----MemLimitFile "+(memLimitFile==null?"not defined":memLimitFile.getAbsolutePath()+" not exist"));} -// } -// } catch (Exception e) { -// //ignore, try defaults -// LG.error(e); -// } -//// long estimatedMemSizeMBL = (long)Math.ceil(estimatedMemSizeMB*1.5); -// boolean bHasMemLimitFile = defaultSimMemoryLimitMbFromFile!=null; -// long maxAllowedMem = (bHasMemLimitFile?defaultSimMemoryLimitMbFromFile:MemLimitResults.FALLBACK_MEM_LIMIT_MB); -//// boolean bUseEstimated = (estimatedMemSizeMBL <= maxAllowedMem); -//// return new MemLimitResults(maxAllowedMem, -//// (bUseEstimated? -//// "MemoryMax(ESTIMATED):SimulationTask.getEstimatedMemorySizeMB()="+estimatedMemSizeMBL: -//// (bHasMemLimitFile? -//// "MemoryMax(FILE AllUsers):AllUsersMemLimit(defaultSimMemoryLimitMb) from "+memLimitFile.getAbsolutePath(): -//// "MemoryMax(HARDCODE):HtcProxy.MemLimitResults.FALLBACK_MEM_LIMIT_MB"))); -// return new MemLimitResults(maxAllowedMem, -// (bHasMemLimitFile? -// "MemoryMax(FILE AllUsers):AllUsersMemLimit(defaultSimMemoryLimitMb) from "+memLimitFile.getAbsolutePath(): -// "MemoryMax(HARDCODE):HtcProxy.MemLimitResults.FALLBACK_MEM_LIMIT_MB")); + return MemLimitResults.getJobRequestedMemoryLimit(solverDescription, estimatedMemSizeMB*1.5, isPowerUser); } -// public static boolean isStochMultiTrial(SimulationTask simTask) { -// return simTask.getSimulationJob().getSimulation().getSolverTaskDescription().getSolverDescription() == SolverDescription.StochGibson && -// simTask.getSimulationJob().getSimulation().getSolverTaskDescription().getStochOpt() != null && -// !simTask.getSimulationJob().getSimulation().getSolverTaskDescription().getStochOpt().isHistogram() && -// simTask.getSimulationJob().getSimulation().getSolverTaskDescription().getStochOpt().getNumOfTrials() > 1; -// -// } } - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java index 2fac63ed39..abcb9b061b 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java @@ -41,6 +41,9 @@ public class SimulationStateMachineTest { private static final VCSimulationIdentifier simID = new VCSimulationIdentifier(simKey, testUser); public static String previousServerID = ""; + public static String previousHtcMax = ""; + public static String previousHtcMin = ""; + public static String previousHtcPowerFloor = ""; private MockSimulationDB simulationDB; private SimulationStateMachineCopy stateMachine; @@ -49,11 +52,23 @@ public class SimulationStateMachineTest { public static void setSystemProperties(){ previousServerID = PropertyLoader.getProperty(PropertyLoader.vcellServerIDProperty, ""); PropertyLoader.setProperty(PropertyLoader.vcellServerIDProperty, ""); + + previousHtcMax = PropertyLoader.getProperty(PropertyLoader.htcMaxMemoryMB, ""); + PropertyLoader.setProperty(PropertyLoader.htcMaxMemoryMB, "4096"); + + previousHtcMin = PropertyLoader.getProperty(PropertyLoader.htcMinMemoryMB, ""); + PropertyLoader.setProperty(PropertyLoader.htcMinMemoryMB, "1024"); + + previousHtcPowerFloor = PropertyLoader.getProperty(PropertyLoader.htcPowerUserMemoryFloorMB, ""); + PropertyLoader.setProperty(PropertyLoader.htcPowerUserMemoryFloorMB, "51200"); } @AfterAll public static void restoreSystemProperties(){ PropertyLoader.setProperty(PropertyLoader.vcellServerIDProperty, previousServerID); + PropertyLoader.setProperty(PropertyLoader.htcMaxMemoryMB, previousHtcMax); + PropertyLoader.setProperty(PropertyLoader.htcMinMemoryMB, previousHtcMin); + PropertyLoader.setProperty(PropertyLoader.htcPowerUserMemoryFloorMB, previousHtcPowerFloor); } @BeforeEach From 5e9efab33a915d4bc241e1bd883abc22ac5cebd2 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Wed, 28 Aug 2024 07:30:58 -0400 Subject: [PATCH 09/24] No More Class Copies --- .../restq/Simulations/SimulationRestDB.java | 6 +- .../restq/apiclient/SimulationApiTest.java | 4 +- .../SimulationDispatcherEngine.java | 334 +++-- .../SimulationDispatcherEngineCopy.java | 148 --- .../dispatcher/SimulationStateMachine.java | 1184 ++++++++--------- .../SimulationStateMachineCopy.java | 576 -------- .../SimulationStateMachineTest.java | 4 +- 7 files changed, 717 insertions(+), 1539 deletions(-) delete mode 100644 vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherEngineCopy.java delete mode 100644 vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineCopy.java diff --git a/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationRestDB.java b/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationRestDB.java index c59a249aeb..30b54623ea 100644 --- a/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationRestDB.java +++ b/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationRestDB.java @@ -7,7 +7,7 @@ import jakarta.enterprise.context.ApplicationScoped; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import cbit.vcell.message.server.dispatcher.SimulationDispatcherEngineCopy; +import cbit.vcell.message.server.dispatcher.SimulationDispatcherEngine; import org.vcell.restq.Simulations.DTO.SimulationStatus; import org.vcell.restq.Simulations.DTO.StatusMessage; import org.vcell.restq.db.AgroalConnectionFactory; @@ -31,13 +31,13 @@ public class SimulationRestDB { private final DatabaseServerImpl databaseServerImpl; private final SimulationDatabaseDirect simulationDatabaseDirect; - private final SimulationDispatcherEngineCopy simulationDispatcherEngine; + private final SimulationDispatcherEngine simulationDispatcherEngine; private final BioModelRestDB bioModelRestDB; public SimulationRestDB(AgroalConnectionFactory agroalConnectionFactory) throws DataAccessException, SQLException { databaseServerImpl = new DatabaseServerImpl(agroalConnectionFactory, agroalConnectionFactory.getKeyFactory()); simulationDatabaseDirect = new SimulationDatabaseDirect(new AdminDBTopLevel(agroalConnectionFactory), databaseServerImpl, true); - simulationDispatcherEngine = new SimulationDispatcherEngineCopy(); + simulationDispatcherEngine = new SimulationDispatcherEngine(); bioModelRestDB = new BioModelRestDB(this, databaseServerImpl); } diff --git a/vcell-rest/src/test/java/org/vcell/restq/apiclient/SimulationApiTest.java b/vcell-rest/src/test/java/org/vcell/restq/apiclient/SimulationApiTest.java index 3c787740a7..00b683ef8f 100644 --- a/vcell-rest/src/test/java/org/vcell/restq/apiclient/SimulationApiTest.java +++ b/vcell-rest/src/test/java/org/vcell/restq/apiclient/SimulationApiTest.java @@ -26,7 +26,7 @@ import org.vcell.restclient.api.SimulationResourceApi; import org.vcell.restclient.model.SchedulerStatus; import org.vcell.restclient.model.Status; -import cbit.vcell.message.server.dispatcher.SimulationDispatcherEngineCopy; +import cbit.vcell.message.server.dispatcher.SimulationDispatcherEngine; import org.vcell.restq.Simulations.DTO.SimulationStatus; import org.vcell.restq.Simulations.DTO.StatusMessage; import org.vcell.restq.TestEndpointUtils; @@ -188,7 +188,7 @@ public void testDifferentSimulationStates() throws Exception{ SimulationJob simulationJob = new SimulationJob(simulation, 0, null); SimulationTask simulationTask = new SimulationTask(simulationJob, 0); WorkerEvent workerEvent = new WorkerEvent(WorkerEvent.JOB_ACCEPTED, "testService", simulationTask, "testHost", SimulationMessage.MESSAGE_JOB_ACCEPTED); - SimulationDispatcherEngineCopy simulationDispatcherEngine = new SimulationDispatcherEngineCopy(); + SimulationDispatcherEngine simulationDispatcherEngine = new SimulationDispatcherEngine(); simulationDispatcherEngine.onWorkerEvent(workerEvent, simulationDatabaseDirect, null); diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherEngine.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherEngine.java index 7ee67d4c84..8bcde2a22d 100644 --- a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherEngine.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherEngine.java @@ -1,175 +1,159 @@ -/* - * Copyright (C) 1999-2011 University of Connecticut Health Center - * - * Licensed under the MIT License (the "License"). - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.opensource.org/licenses/mit-license.php - */ - -package cbit.vcell.message.server.dispatcher; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vcell.util.DataAccessException; -import org.vcell.util.document.KeyValue; -import org.vcell.util.document.User; -import org.vcell.util.document.VCellServerID; - -import cbit.rmi.event.WorkerEvent; -import cbit.vcell.message.VCMessageSession; -import cbit.vcell.message.VCMessagingException; -import cbit.vcell.message.messages.StatusMessage; -import cbit.vcell.server.SimulationJobStatus; -import cbit.vcell.server.SimulationJobStatus.SchedulerStatus; -import cbit.vcell.server.UpdateSynchronizationException; -import cbit.vcell.solver.Simulation; -import cbit.vcell.solver.SimulationInfo; -import cbit.vcell.solver.VCSimulationIdentifier; -import cbit.vcell.solver.server.SimulationMessage; - -/** - * Insert the type's description here. - * Creation date: (10/18/2001 4:31:11 PM) - * @author: Jim Schaff - */ -public class SimulationDispatcherEngine { - public static final Logger lg = LogManager.getLogger(SimulationDispatcherEngine.class); - - private HashMap> simStateMachineHash = new HashMap>(); - - /** - * Scheduler constructor comment. - */ - public SimulationDispatcherEngine() { - } - - /** - * reset simulation state time stamps in case of transient error in getting running status - */ - void resetTimeStamps( ) { - long now = System.currentTimeMillis(); - for (List lst : simStateMachineHash.values()) { - for (SimulationStateMachine ssm: lst) { - ssm.setSolverProcessTimestamp(now); - } - } - } - - public SimulationStateMachine getSimulationStateMachine(KeyValue simulationKey, int jobIndex) { - List stateMachineList = simStateMachineHash.get(simulationKey); - if (stateMachineList==null){ - stateMachineList = new ArrayList(); - simStateMachineHash.put(simulationKey,stateMachineList); - } - for (SimulationStateMachine stateMachine : stateMachineList){ - if (stateMachine.getJobIndex() == jobIndex){ - return stateMachine; - } - } - SimulationStateMachine newStateMachine = new SimulationStateMachine(simulationKey, jobIndex); - stateMachineList.add(newStateMachine); - return newStateMachine; - } - - public void onDispatch(Simulation simulation, SimulationJobStatus simJobStatus, SimulationDatabase simulationDatabase, VCMessageSession dispatcherQueueSession) throws VCMessagingException, DataAccessException, SQLException{ - KeyValue simulationKey = simJobStatus.getVCSimulationIdentifier().getSimulationKey(); - SimulationStateMachine simStateMachine = getSimulationStateMachine(simulationKey, simJobStatus.getJobIndex()); - - simStateMachine.onDispatch(simulation, simJobStatus, simulationDatabase, dispatcherQueueSession); - } - - public void onStartRequest(VCSimulationIdentifier vcSimID, User user, int simulationScanCount, SimulationDatabase simulationDatabase, VCMessageSession session, VCMessageSession dispatcherQueueSession) throws VCMessagingException, DataAccessException, SQLException { - KeyValue simKey = vcSimID.getSimulationKey(); - - User.SpecialUser myUser = simulationDatabase.getUser(user.getName()); - boolean isAdmin = Arrays.asList(myUser.getMySpecials()).contains(User.SPECIAL_CLAIM.admins); - - SimulationInfo simulationInfo = null; - try { - simulationInfo = simulationDatabase.getSimulationInfo(user, simKey); - } catch (DataAccessException ex) { - if (lg.isWarnEnabled()) lg.warn("Bad simulation " + vcSimID); - StatusMessage message = new StatusMessage(new SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, - SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Failed to dispatch simulation: "+ ex.getMessage()), null, null), user.getName(), null, null); - message.sendToClient(session); - return; - } - if (simulationInfo == null) { - if (lg.isWarnEnabled()) lg.warn("Can't start, simulation [" + vcSimID + "] doesn't exist in database"); - StatusMessage message = new StatusMessage(new SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, - SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Can't start, simulation [" + vcSimID + "] doesn't exist"), null, null), user.getName(), null, null); - message.sendToClient(session); - return; - } - - if (!isAdmin && simulationScanCount > Integer.parseInt(cbit.vcell.resource.PropertyLoader.getRequiredProperty(cbit.vcell.resource.PropertyLoader.maxJobsPerScan))) { - if (lg.isWarnEnabled()) lg.warn("Too many simulations (" + simulationScanCount + ") for parameter scan." + vcSimID); - StatusMessage message = new StatusMessage(new SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, - SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Too many simulations (" + simulationScanCount + ") for parameter scan."), null, null), user.getName(), null, null); - message.sendToClient(session); - return; - } - - for (int jobIndex = 0; jobIndex < simulationScanCount; jobIndex++){ - SimulationStateMachine simStateMachine = getSimulationStateMachine(simKey, jobIndex); - try { - simStateMachine.onStartRequest(user, vcSimID, simulationDatabase, session); - }catch (UpdateSynchronizationException e){ - simStateMachine.onStartRequest(user, vcSimID, simulationDatabase, session); - } - } - } - - - public void onStopRequest(VCSimulationIdentifier vcSimID, User user, SimulationDatabase simulationDatabase, VCMessageSession session) throws DataAccessException, VCMessagingException, SQLException { - KeyValue simKey = vcSimID.getSimulationKey(); - - SimulationJobStatus[] allActiveSimJobStatusArray = simulationDatabase.getActiveJobs(VCellServerID.getSystemServerID()); - ArrayList simJobStatusArray = new ArrayList(); - for (SimulationJobStatus activeSimJobStatus : allActiveSimJobStatusArray){ - if (activeSimJobStatus.getVCSimulationIdentifier().getSimulationKey().equals(vcSimID.getSimulationKey())){ - simJobStatusArray.add(activeSimJobStatus); - } - } - for (SimulationJobStatus simJobStatus : simJobStatusArray){ - SimulationStateMachine simStateMachine = getSimulationStateMachine(simKey, simJobStatus.getJobIndex()); - try { - simStateMachine.onStopRequest(user, simJobStatus, simulationDatabase, session); - }catch (UpdateSynchronizationException e){ - simStateMachine.onStopRequest(user, simJobStatus, simulationDatabase, session); - } - } - } - - - public void onWorkerEvent(WorkerEvent workerEvent, SimulationDatabase simulationDatabase, VCMessageSession session) { - try { - KeyValue simKey = workerEvent.getVCSimulationDataIdentifier().getSimulationKey(); - int jobIndex = workerEvent.getJobIndex(); - SimulationStateMachine simStateMachine = getSimulationStateMachine(simKey, jobIndex); - simStateMachine.onWorkerEvent(workerEvent, simulationDatabase, session); - } catch (Exception ex) { - lg.error(ex.getMessage(),ex); - } - } - - - public void onSystemAbort(SimulationJobStatus jobStatus, String failureMessage, SimulationDatabase simulationDatabase, VCMessageSession session) { - try { - KeyValue simKey = jobStatus.getVCSimulationIdentifier().getSimulationKey(); - int jobIndex = jobStatus.getJobIndex(); - SimulationStateMachine simStateMachine = getSimulationStateMachine(simKey, jobIndex); - simStateMachine.onSystemAbort(jobStatus, failureMessage, simulationDatabase, session); - } catch (Exception ex) { - lg.error(ex.getMessage(),ex); - } - } - -} +package cbit.vcell.message.server.dispatcher; + +import cbit.rmi.event.WorkerEvent; +import cbit.vcell.message.VCMessageSession; +import cbit.vcell.message.VCMessagingException; +import cbit.vcell.message.messages.StatusMessage; +import cbit.vcell.server.UpdateSynchronizationException; +import cbit.vcell.solver.Simulation; +import cbit.vcell.solver.SimulationInfo; +import cbit.vcell.solver.VCSimulationIdentifier; +import cbit.vcell.solver.server.SimulationMessage; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vcell.util.DataAccessException; +import org.vcell.util.document.KeyValue; +import org.vcell.util.document.User; +import org.vcell.util.document.VCellServerID; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +public class SimulationDispatcherEngine { + public static final Logger lg = LogManager.getLogger(cbit.vcell.message.server.dispatcher.SimulationDispatcherEngine.class); + + private HashMap> simStateMachineHash = new HashMap>(); + + /** + * reset simulation state time stamps in case of transient error in getting running status + */ + void resetTimeStamps( ) { + long now = System.currentTimeMillis(); + for (List lst : simStateMachineHash.values()) { + for (SimulationStateMachine ssm: lst) { + ssm.setSolverProcessTimestamp(now); + } + } + } + + public SimulationDispatcherEngine() { + } + + public SimulationStateMachine getSimulationStateMachine(KeyValue simulationKey, int jobIndex) { + List stateMachineList = simStateMachineHash.get(simulationKey); + if (stateMachineList==null){ + stateMachineList = new ArrayList(); + simStateMachineHash.put(simulationKey,stateMachineList); + } + for (SimulationStateMachine stateMachine : stateMachineList){ + if (stateMachine.getJobIndex() == jobIndex){ + return stateMachine; + } + } + SimulationStateMachine newStateMachine = new SimulationStateMachine(simulationKey, jobIndex); + stateMachineList.add(newStateMachine); + return newStateMachine; + } + + public void onDispatch(Simulation simulation, cbit.vcell.server.SimulationJobStatus simJobStatus, SimulationDatabase simulationDatabase, VCMessageSession dispatcherQueueSession) throws VCMessagingException, DataAccessException, SQLException { + KeyValue simulationKey = simJobStatus.getVCSimulationIdentifier().getSimulationKey(); + SimulationStateMachine simStateMachine = getSimulationStateMachine(simulationKey, simJobStatus.getJobIndex()); + + simStateMachine.onDispatch(simulation, simJobStatus, simulationDatabase, dispatcherQueueSession); + } + + public ArrayList onStartRequest(VCSimulationIdentifier vcSimID, User user, int simulationScanCount, SimulationDatabase simulationDatabase, VCMessageSession session, VCMessageSession dispatcherQueueSession) throws VCMessagingException, DataAccessException, SQLException { + KeyValue simKey = vcSimID.getSimulationKey(); + + User.SpecialUser myUser = simulationDatabase.getUser(user.getName()); + boolean isAdmin = Arrays.asList(myUser.getMySpecials()).contains(User.SPECIAL_CLAIM.admins); + + SimulationInfo simulationInfo = null; + cbit.vcell.server.SimulationJobStatus simJobStatus = null; + ArrayList status = new ArrayList<>(); + try { + simulationInfo = simulationDatabase.getSimulationInfo(user, simKey); + } catch (DataAccessException ex) { + if (lg.isWarnEnabled()) lg.warn("Bad simulation " + vcSimID); + simJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, + cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Failed to dispatch simulation: "+ ex.getMessage()), null, null); + status.add(new StatusMessage(simJobStatus, user.getName(), null, null)); + return status; + } + if (simulationInfo == null) { + if (lg.isWarnEnabled()) lg.warn("Can't start, simulation [" + vcSimID + "] doesn't exist in database"); + simJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, + cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Can't start, simulation [" + vcSimID + "] doesn't exist"), null, null); + status.add(new StatusMessage(simJobStatus, user.getName(), null, null)); + return status; + } + + if (!isAdmin && simulationScanCount > Integer.parseInt(cbit.vcell.resource.PropertyLoader.getRequiredProperty(cbit.vcell.resource.PropertyLoader.maxJobsPerScan))) { + if (lg.isWarnEnabled()) lg.warn("Too many simulations (" + simulationScanCount + ") for parameter scan." + vcSimID); + simJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, + cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Too many simulations (" + simulationScanCount + ") for parameter scan."), null, null); + status.add(new StatusMessage(simJobStatus, user.getName(), null, null)); + return status; + } + + for (int simulationJobIndex = 0; simulationJobIndex < simulationScanCount; simulationJobIndex++){ + SimulationStateMachine simStateMachine = getSimulationStateMachine(simKey, simulationJobIndex); + try { + status.add(simStateMachine.onStartRequest(user, vcSimID, simulationDatabase, session)); + }catch (UpdateSynchronizationException e){ + status.add(simStateMachine.onStartRequest(user, vcSimID, simulationDatabase, session)); + } + } + return status; + } + + + public ArrayList onStopRequest(VCSimulationIdentifier vcSimID, User user, SimulationDatabase simulationDatabase, VCMessageSession session) throws DataAccessException, VCMessagingException, SQLException { + KeyValue simKey = vcSimID.getSimulationKey(); + + cbit.vcell.server.SimulationJobStatus[] allActiveSimJobStatusArray = simulationDatabase.getActiveJobs(VCellServerID.getSystemServerID()); + ArrayList simJobStatusArray = new ArrayList(); + for (cbit.vcell.server.SimulationJobStatus activeSimJobStatus : allActiveSimJobStatusArray){ + if (activeSimJobStatus.getVCSimulationIdentifier().getSimulationKey().equals(vcSimID.getSimulationKey())){ + simJobStatusArray.add(activeSimJobStatus); + } + } + ArrayList stoppedSimulations = new ArrayList<>(); + for (cbit.vcell.server.SimulationJobStatus simJobStatus : simJobStatusArray){ + SimulationStateMachine simStateMachine = getSimulationStateMachine(simKey, simJobStatus.getJobIndex()); + try { + stoppedSimulations.add(simStateMachine.onStopRequest(user, simJobStatus, simulationDatabase, session)); + }catch (UpdateSynchronizationException e){ + stoppedSimulations.add(simStateMachine.onStopRequest(user, simJobStatus, simulationDatabase, session)); + } + } + return stoppedSimulations; + } + + + public void onWorkerEvent(WorkerEvent workerEvent, SimulationDatabase simulationDatabase, VCMessageSession session) { + try { + KeyValue simKey = workerEvent.getVCSimulationDataIdentifier().getSimulationKey(); + int jobIndex = workerEvent.getJobIndex(); + SimulationStateMachine simStateMachine = getSimulationStateMachine(simKey, jobIndex); + simStateMachine.onWorkerEvent(workerEvent, simulationDatabase, session); + } catch (Exception ex) { + lg.error(ex.getMessage(),ex); + } + } + + + public void onSystemAbort(cbit.vcell.server.SimulationJobStatus jobStatus, String failureMessage, SimulationDatabase simulationDatabase, VCMessageSession session) { + try { + KeyValue simKey = jobStatus.getVCSimulationIdentifier().getSimulationKey(); + int jobIndex = jobStatus.getJobIndex(); + SimulationStateMachine simStateMachine = getSimulationStateMachine(simKey, jobIndex); + simStateMachine.onSystemAbort(jobStatus, failureMessage, simulationDatabase, session); + } catch (Exception ex) { + lg.error(ex.getMessage(),ex); + } + } +} diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherEngineCopy.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherEngineCopy.java deleted file mode 100644 index 6c327ab0b8..0000000000 --- a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherEngineCopy.java +++ /dev/null @@ -1,148 +0,0 @@ -package cbit.vcell.message.server.dispatcher; - -import cbit.rmi.event.WorkerEvent; -import cbit.vcell.message.VCMessageSession; -import cbit.vcell.message.VCMessagingException; -import cbit.vcell.message.messages.StatusMessage; -import cbit.vcell.server.UpdateSynchronizationException; -import cbit.vcell.solver.Simulation; -import cbit.vcell.solver.SimulationInfo; -import cbit.vcell.solver.VCSimulationIdentifier; -import cbit.vcell.solver.server.SimulationMessage; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vcell.util.DataAccessException; -import org.vcell.util.document.KeyValue; -import org.vcell.util.document.User; -import org.vcell.util.document.VCellServerID; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; - -public class SimulationDispatcherEngineCopy { - public static final Logger lg = LogManager.getLogger(cbit.vcell.message.server.dispatcher.SimulationDispatcherEngine.class); - - private HashMap> simStateMachineHash = new HashMap>(); - - - public SimulationDispatcherEngineCopy() { - } - - public SimulationStateMachineCopy getSimulationStateMachine(KeyValue simulationKey, int jobIndex) { - List stateMachineList = simStateMachineHash.get(simulationKey); - if (stateMachineList==null){ - stateMachineList = new ArrayList(); - simStateMachineHash.put(simulationKey,stateMachineList); - } - for (SimulationStateMachineCopy stateMachine : stateMachineList){ - if (stateMachine.getJobIndex() == jobIndex){ - return stateMachine; - } - } - SimulationStateMachineCopy newStateMachine = new SimulationStateMachineCopy(simulationKey, jobIndex); - stateMachineList.add(newStateMachine); - return newStateMachine; - } - - public void onDispatch(Simulation simulation, cbit.vcell.server.SimulationJobStatus simJobStatus, SimulationDatabase simulationDatabase, VCMessageSession dispatcherQueueSession) throws VCMessagingException, DataAccessException, SQLException { - KeyValue simulationKey = simJobStatus.getVCSimulationIdentifier().getSimulationKey(); - SimulationStateMachineCopy simStateMachine = getSimulationStateMachine(simulationKey, simJobStatus.getJobIndex()); - - simStateMachine.onDispatch(simulation, simJobStatus, simulationDatabase, dispatcherQueueSession); - } - - public ArrayList onStartRequest(VCSimulationIdentifier vcSimID, User user, int simulationScanCount, SimulationDatabase simulationDatabase, VCMessageSession session, VCMessageSession dispatcherQueueSession) throws VCMessagingException, DataAccessException, SQLException { - KeyValue simKey = vcSimID.getSimulationKey(); - - User.SpecialUser myUser = simulationDatabase.getUser(user.getName()); - boolean isAdmin = Arrays.asList(myUser.getMySpecials()).contains(User.SPECIAL_CLAIM.admins); - - SimulationInfo simulationInfo = null; - cbit.vcell.server.SimulationJobStatus simJobStatus = null; - ArrayList status = new ArrayList<>(); - try { - simulationInfo = simulationDatabase.getSimulationInfo(user, simKey); - } catch (DataAccessException ex) { - if (lg.isWarnEnabled()) lg.warn("Bad simulation " + vcSimID); - simJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, - cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Failed to dispatch simulation: "+ ex.getMessage()), null, null); - status.add(new StatusMessage(simJobStatus, user.getName(), null, null)); - return status; - } - if (simulationInfo == null) { - if (lg.isWarnEnabled()) lg.warn("Can't start, simulation [" + vcSimID + "] doesn't exist in database"); - simJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, - cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Can't start, simulation [" + vcSimID + "] doesn't exist"), null, null); - status.add(new StatusMessage(simJobStatus, user.getName(), null, null)); - return status; - } - - if (!isAdmin && simulationScanCount > Integer.parseInt(cbit.vcell.resource.PropertyLoader.getRequiredProperty(cbit.vcell.resource.PropertyLoader.maxJobsPerScan))) { - if (lg.isWarnEnabled()) lg.warn("Too many simulations (" + simulationScanCount + ") for parameter scan." + vcSimID); - simJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, - cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Too many simulations (" + simulationScanCount + ") for parameter scan."), null, null); - status.add(new StatusMessage(simJobStatus, user.getName(), null, null)); - return status; - } - - for (int simulationJobIndex = 0; simulationJobIndex < simulationScanCount; simulationJobIndex++){ - SimulationStateMachineCopy simStateMachine = getSimulationStateMachine(simKey, simulationJobIndex); - try { - status.add(simStateMachine.onStartRequest(user, vcSimID, simulationDatabase, session)); - }catch (UpdateSynchronizationException e){ - status.add(simStateMachine.onStartRequest(user, vcSimID, simulationDatabase, session)); - } - } - return status; - } - - - public ArrayList onStopRequest(VCSimulationIdentifier vcSimID, User user, SimulationDatabase simulationDatabase, VCMessageSession session) throws DataAccessException, VCMessagingException, SQLException { - KeyValue simKey = vcSimID.getSimulationKey(); - - cbit.vcell.server.SimulationJobStatus[] allActiveSimJobStatusArray = simulationDatabase.getActiveJobs(VCellServerID.getSystemServerID()); - ArrayList simJobStatusArray = new ArrayList(); - for (cbit.vcell.server.SimulationJobStatus activeSimJobStatus : allActiveSimJobStatusArray){ - if (activeSimJobStatus.getVCSimulationIdentifier().getSimulationKey().equals(vcSimID.getSimulationKey())){ - simJobStatusArray.add(activeSimJobStatus); - } - } - ArrayList stoppedSimulations = new ArrayList<>(); - for (cbit.vcell.server.SimulationJobStatus simJobStatus : simJobStatusArray){ - SimulationStateMachineCopy simStateMachine = getSimulationStateMachine(simKey, simJobStatus.getJobIndex()); - try { - stoppedSimulations.add(simStateMachine.onStopRequest(user, simJobStatus, simulationDatabase, session)); - }catch (UpdateSynchronizationException e){ - stoppedSimulations.add(simStateMachine.onStopRequest(user, simJobStatus, simulationDatabase, session)); - } - } - return stoppedSimulations; - } - - - public void onWorkerEvent(WorkerEvent workerEvent, SimulationDatabase simulationDatabase, VCMessageSession session) { - try { - KeyValue simKey = workerEvent.getVCSimulationDataIdentifier().getSimulationKey(); - int jobIndex = workerEvent.getJobIndex(); - SimulationStateMachineCopy simStateMachine = getSimulationStateMachine(simKey, jobIndex); - simStateMachine.onWorkerEvent(workerEvent, simulationDatabase, session); - } catch (Exception ex) { - lg.error(ex.getMessage(),ex); - } - } - - - public void onSystemAbort(cbit.vcell.server.SimulationJobStatus jobStatus, String failureMessage, SimulationDatabase simulationDatabase, VCMessageSession session) { - try { - KeyValue simKey = jobStatus.getVCSimulationIdentifier().getSimulationKey(); - int jobIndex = jobStatus.getJobIndex(); - SimulationStateMachineCopy simStateMachine = getSimulationStateMachine(simKey, jobIndex); - simStateMachine.onSystemAbort(jobStatus, failureMessage, simulationDatabase, session); - } catch (Exception ex) { - lg.error(ex.getMessage(),ex); - } - } -} diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java index 670c3ea9c4..286a66a4fd 100644 --- a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java @@ -1,20 +1,7 @@ package cbit.vcell.message.server.dispatcher; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.Date; - -import cbit.vcell.resource.PropertyLoader; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vcell.util.DataAccessException; -import org.vcell.util.document.KeyValue; -import org.vcell.util.document.User; -import org.vcell.util.document.VCellServerID; - import cbit.rmi.event.WorkerEvent; import cbit.vcell.field.FieldDataIdentifierSpec; -import cbit.vcell.message.VCMessage; import cbit.vcell.message.VCMessageSession; import cbit.vcell.message.VCMessagingConstants; import cbit.vcell.message.VCMessagingException; @@ -23,635 +10,566 @@ import cbit.vcell.message.messages.SimulationTaskMessage; import cbit.vcell.message.messages.StatusMessage; import cbit.vcell.message.messages.WorkerEventMessage; -import cbit.vcell.message.server.htc.HtcProxy; -import cbit.vcell.message.server.htc.HtcProxy.MemLimitResults; import cbit.vcell.messaging.server.SimulationTask; import cbit.vcell.mongodb.VCMongoMessage; -import cbit.vcell.server.HtcJobID; -import cbit.vcell.server.RunningStateInfo; -import cbit.vcell.server.SimulationExecutionStatus; -import cbit.vcell.server.SimulationJobStatus; -import cbit.vcell.server.SimulationJobStatus.SchedulerStatus; -import cbit.vcell.server.SimulationQueueEntryStatus; -import cbit.vcell.server.SimulationStatus; -import cbit.vcell.server.UpdateSynchronizationException; -import cbit.vcell.solver.Simulation; -import cbit.vcell.solver.SimulationJob; -import cbit.vcell.solver.SolverDescription; -import cbit.vcell.solver.VCSimulationDataIdentifier; -import cbit.vcell.solver.VCSimulationIdentifier; +import cbit.vcell.resource.PropertyLoader; +import cbit.vcell.server.*; +import cbit.vcell.solver.*; import cbit.vcell.solver.server.SimulationMessage; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vcell.util.DataAccessException; +import org.vcell.util.document.KeyValue; +import org.vcell.util.document.User; +import org.vcell.util.document.VCellServerID; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Date; public class SimulationStateMachine { - public static final Logger lg = LogManager.getLogger(SimulationStateMachine.class); - - // bitmapped counter so that allows 3 retries for each request (but preserves ordinal nature) - // bits 0-3: retry count - // bits 4-31: submit - // max retries must be less than 15. - public static final int TASKID_USERCOUNTER_MASK = SimulationStatus.TASKID_USERCOUNTER_MASK; - public static final int TASKID_RETRYCOUNTER_MASK = SimulationStatus.TASKID_RETRYCOUNTER_MASK; - public static final int TASKID_USERINCREMENT = SimulationStatus.TASKID_USERINCREMENT; - - public static final int PRIORITY_LOW = 0; - public static final int PRIORITY_DEFAULT = 5; - public static final int PRIORITY_HIGH = 9; - - private final KeyValue simKey; - private final int jobIndex; - - /** - * in memory storage of last time information about this job was received or status was unknown due - * to transient failure or system restart - */ - private long solverProcessTimestamp; - - public SimulationStateMachine(KeyValue simKey, int jobIndex){ - this.simKey = simKey; - this.jobIndex = jobIndex; - updateSolverProcessTimestamp(); - } - - /* - public SimulationStateMachine(SimulationJobStatus[] simJobStatus) { - this(simJobStatus[0].getVCSimulationIdentifier().getSimulationKey(),simJobStatus[0].getJobIndex()); - } - */ - - /** - * set in memory last update time to now - */ - private void updateSolverProcessTimestamp( ) { - solverProcessTimestamp = System.currentTimeMillis(); - } - - /** - * set to specified time (for mass setting) - * @param solverProcessTimestamp - */ - void setSolverProcessTimestamp(long solverProcessTimestamp) { - this.solverProcessTimestamp = solverProcessTimestamp; - } - - public KeyValue getSimKey() { - return simKey; - } - - public int getJobIndex() { - return jobIndex; - } - -// public List getStateMachineTransitions() { -// return stateMachineTransitions; -// } - -// public String show(){ -// StringBuffer buffer = new StringBuffer(); -// buffer.append("SimulationStateMachine for SimID='"+simKey+"', jobIndex="+jobIndex+"\n"); -// for (StateMachineTransition stateMachineTransition : stateMachineTransitions){ -// buffer.append(stateMachineTransition+"\n"); -// } -// return buffer.toString(); -// } - -// private void addStateMachineTransition(StateMachineTransition stateMachineTransition){ -// stateMachineTransitions.add(stateMachineTransition); -// } - - /** - * return last time a status update was received in memory - * @return time since information last changed about this task - */ - long getSolverProcessTimestamp() { - return solverProcessTimestamp; - } - - public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDatabase simulationDatabase, VCMessageSession session) throws DataAccessException, VCMessagingException, SQLException { - updateSolverProcessTimestamp(); - WorkerEventMessage workerEventMessage = new WorkerEventMessage(workerEvent); - VCMongoMessage.sendWorkerEvent(workerEventMessage); - - String userName = workerEvent.getUserName(); // as the filter of the client - int workerEventTaskID = workerEvent.getTaskID(); - - if (lg.isTraceEnabled()) lg.trace("onWorkerEventMessage[" + workerEvent.getEventTypeID() + "," + workerEvent.getSimulationMessage() + "][simid=" + workerEvent.getVCSimulationDataIdentifier() + ",job=" + jobIndex + ",task=" + workerEventTaskID + "]"); - - VCSimulationDataIdentifier vcSimDataID = workerEvent.getVCSimulationDataIdentifier(); - if (vcSimDataID == null) { - VCMongoMessage.sendInfo("onWorkerEvent() ignoring WorkerEvent - no SimID in message): "+workerEvent.show()); - return; - } - KeyValue simKey = vcSimDataID.getSimulationKey(); - SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(simKey, jobIndex); - - if (oldSimulationJobStatus == null){ - VCMongoMessage.sendInfo("onWorkerEvent() ignoring WorkerEvent, no current SimulationJobStatus: "+workerEvent.show()); - return; - } - if (oldSimulationJobStatus == null || oldSimulationJobStatus.getSchedulerStatus().isDone() || oldSimulationJobStatus.getTaskID() > workerEventTaskID){ - VCMongoMessage.sendInfo("onWorkerEvent() ignoring outdated WorkerEvent, (currState="+oldSimulationJobStatus.getSchedulerStatus().getDescription()+"): "+workerEvent.show()); - return; - } - int taskID = oldSimulationJobStatus.getTaskID(); - SchedulerStatus oldSchedulerStatus = oldSimulationJobStatus.getSchedulerStatus(); - - // - // status information (initialized as if new record) - // - Date startDate = null; - Date lastUpdateDate = null; - Date endDate = null; - boolean hasData = false; - HtcJobID htcJobID = null; - String computeHost = null; - VCellServerID vcServerID = VCellServerID.getSystemServerID(); - Date submitDate = null; - Date queueDate = null; - int queuePriority = PRIORITY_DEFAULT; - SimulationJobStatus.SimulationQueueID simQueueID = SimulationJobStatus.SimulationQueueID.QUEUE_ID_WAITING; - - - // - // update using previously stored status (if available). - // - SimulationExecutionStatus oldSimExeStatus = oldSimulationJobStatus.getSimulationExecutionStatus(); - if (oldSimExeStatus!=null && oldSimExeStatus.getStartDate()!=null){ - startDate = oldSimExeStatus.getStartDate(); - } - if (oldSimExeStatus!=null && oldSimExeStatus.getLatestUpdateDate()!=null){ - lastUpdateDate = oldSimExeStatus.getLatestUpdateDate(); - } - if (oldSimExeStatus!=null && oldSimExeStatus.getEndDate()!=null){ - endDate = oldSimExeStatus.getEndDate(); - } - if (oldSimExeStatus!=null && oldSimExeStatus.hasData()){ - hasData = true; - } - if (oldSimExeStatus!=null && oldSimExeStatus.getComputeHost()!=null){ - computeHost = oldSimExeStatus.getComputeHost(); - } - if (oldSimExeStatus!=null && oldSimExeStatus.getHtcJobID()!=null){ - htcJobID = oldSimExeStatus.getHtcJobID(); - } - vcServerID = oldSimulationJobStatus.getServerID(); - submitDate = oldSimulationJobStatus.getSubmitDate(); - SimulationQueueEntryStatus oldQueueStatus = oldSimulationJobStatus.getSimulationQueueEntryStatus(); - if (oldQueueStatus!=null && oldQueueStatus.getQueueDate()!=null){ - queueDate = oldQueueStatus.getQueueDate(); - } - if (oldQueueStatus!=null){ - queuePriority = oldQueueStatus.getQueuePriority(); - } - if (oldQueueStatus!=null && oldQueueStatus.getQueueID()!=null){ - simQueueID = oldQueueStatus.getQueueID(); - } - - // - // update using new information from event - // - if (workerEvent.getHtcJobID()!=null){ - htcJobID = workerEvent.getHtcJobID(); - } - if (workerEvent.getHostName()!=null){ - computeHost = workerEvent.getHostName(); - } - SimulationMessage workerEventSimulationMessage = workerEvent.getSimulationMessage(); - if (workerEventSimulationMessage.getHtcJobId()!=null){ - htcJobID = workerEventSimulationMessage.getHtcJobId(); - } - - - SimulationJobStatus newJobStatus = null; - - if (workerEvent.isAcceptedEvent()) { - // - // job message accepted by HtcSimulationWorker and sent to Scheduler (PBS/SGE/SLURM) (with a htcJobID) ... previous state should be "WAITING" - // - if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued()) { - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - - // new exe status - lastUpdateDate = new Date(); - startDate = lastUpdateDate; - endDate = null; - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.DISPATCHED, - taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); - } - - } else if (workerEvent.isStartingEvent()) { - // only update database when the job event changes from started to running. The later progress event will not be recorded. - if ( oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()) { - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - - // new exe status - lastUpdateDate = new Date(); - if (startDate == null){ - startDate = lastUpdateDate; - } - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.RUNNING, - taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); - } - - } else if (workerEvent.isNewDataEvent()) { - if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ - - if (!oldSchedulerStatus.isRunning() || simQueueID != SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL || hasData==false){ - - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - - // new exe status - if (startDate == null){ - startDate = lastUpdateDate; - } - hasData = true; - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.RUNNING, - taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); - } - } - - } else if (workerEvent.isProgressEvent() || workerEvent.isWorkerAliveEvent()) { - if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ - - - if (!oldSchedulerStatus.isRunning() || simQueueID != SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL){ - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - - // new exe status - if (startDate == null){ - startDate = lastUpdateDate; - } - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.RUNNING, - taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); - - }else if (oldSchedulerStatus.isRunning()){ - if (oldSimExeStatus != null) { -// Date latestUpdate = oldSimExeStatus.getLatestUpdateDate(); -// if (System.currentTimeMillis() - latestUpdate.getTime() >= MessageConstants.INTERVAL_PING_SERVER_MS * 3 / 5) { - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.RUNNING, - taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); - } -// } - } - } - - } else if (workerEvent.isCompletedEvent()) { - if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - - // new exe status - endDate = new Date(); - hasData = true; - - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.COMPLETED, - taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); - - } - - } else if (workerEvent.isFailedEvent()) { - if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - - // new exe status - endDate = new Date(); - - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.FAILED, - taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); - - } - } else if (workerEvent.isWorkerExitErrorEvent()) { - if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - - // new exe status - endDate = new Date(); - - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - - SimulationMessage simulationMessage = SimulationMessage.workerFailure("solver stopped unexpectedly, "+workerEventSimulationMessage.getDisplayMessage()); - - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.FAILED, - taskID, simulationMessage, newQueueStatus, newExeStatus); - - } - } - if (newJobStatus!=null){ - if (!newJobStatus.compareEqual(oldSimulationJobStatus) || workerEvent.isProgressEvent() || workerEvent.isNewDataEvent()) { - Double progress = workerEvent.getProgress(); - Double timepoint = workerEvent.getTimePoint(); - RunningStateInfo runningStateInfo = null; - if (progress != null && timepoint != null){ - runningStateInfo = new RunningStateInfo(progress,timepoint); - } - simulationDatabase.updateSimulationJobStatus(newJobStatus,runningStateInfo); - StatusMessage msgForClient = new StatusMessage(newJobStatus, userName, progress, timepoint); - msgForClient.sendToClient(session); - if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); - } else { - simulationDatabase.updateSimulationJobStatus(newJobStatus); - StatusMessage msgForClient = new StatusMessage(newJobStatus, userName, null, null); - msgForClient.sendToClient(session); - if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); - } - }else if (workerEvent.isProgressEvent() || workerEvent.isNewDataEvent()){ - Double progress = workerEvent.getProgress(); - Double timepoint = workerEvent.getTimePoint(); - RunningStateInfo runningStateInfo = null; - if (progress!=null && timepoint!=null){ - runningStateInfo = new RunningStateInfo(progress,timepoint); - } - simulationDatabase.updateSimulationJobStatus(oldSimulationJobStatus,runningStateInfo); - StatusMessage msgForClient = new StatusMessage(oldSimulationJobStatus, userName, progress, timepoint); - msgForClient.sendToClient(session); - if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); - }else{ - VCMongoMessage.sendInfo("onWorkerEvent() ignoring WorkerEvent (currState="+oldSchedulerStatus.getDescription()+"): "+workerEvent.show()); - } + public static final Logger lg = LogManager.getLogger(cbit.vcell.message.server.dispatcher.SimulationStateMachine.class); + + // bitmapped counter so that allows 3 retries for each request (but preserves ordinal nature) + // bits 0-3: retry count + // bits 4-31: submit + // max retries must be less than 15. + public static final int TASKID_USERCOUNTER_MASK = SimulationStatus.TASKID_USERCOUNTER_MASK; + public static final int TASKID_RETRYCOUNTER_MASK = SimulationStatus.TASKID_RETRYCOUNTER_MASK; + public static final int TASKID_USERINCREMENT = SimulationStatus.TASKID_USERINCREMENT; + + public static final int PRIORITY_LOW = 0; + public static final int PRIORITY_DEFAULT = 5; + public static final int PRIORITY_HIGH = 9; + + private final KeyValue simKey; + private final int jobIndex; + + /** + * in memory storage of last time information about this job was received or status was unknown due + * to transient failure or system restart + */ + private long solverProcessTimestamp; + + private class CurrentState { + + public Date startDate; + public Date lastUpdateDate; + public Date endDate; + public boolean hasData; + public HtcJobID htcJobID; + public String computeHost; + public VCellServerID vcServerID; + public Date submitDate; + public Date queueDate; + public int queuePriority; + public SimulationJobStatus.SimulationQueueID simQueueID; + + public CurrentState(SimulationExecutionStatus oldSimExeStatus, + SimulationQueueEntryStatus oldQueueStatus, + SimulationJobStatus oldSimulationJobStatus){ + boolean isOldExeNull = oldSimExeStatus == null; + boolean isOldQueueNull = oldQueueStatus == null; + // + // status information (initialized as if new record) + // + startDate = !isOldExeNull && oldSimExeStatus.getStartDate()!=null ? oldSimExeStatus.getStartDate() :null; + lastUpdateDate = !isOldExeNull && oldSimExeStatus.getLatestUpdateDate()!=null ? oldSimExeStatus.getLatestUpdateDate() : null; + endDate = !isOldExeNull && oldSimExeStatus.getEndDate()!=null ? oldSimExeStatus.getEndDate() : null; + hasData = !isOldExeNull && oldSimExeStatus.hasData(); + htcJobID = !isOldExeNull && oldSimExeStatus.getHtcJobID()!=null ? oldSimExeStatus.getHtcJobID() : null; + computeHost = !isOldExeNull && oldSimExeStatus.getComputeHost()!=null ? oldSimExeStatus.getComputeHost() : null; + vcServerID = oldSimulationJobStatus.getServerID(); + submitDate = oldSimulationJobStatus.getSubmitDate(); + queueDate = !isOldQueueNull && oldQueueStatus.getQueueDate() != null ? oldQueueStatus.getQueueDate() : null; + queuePriority = !isOldQueueNull ? oldQueueStatus.getQueuePriority() : PRIORITY_DEFAULT; + simQueueID = !isOldQueueNull && oldQueueStatus.getQueueID()!=null ? oldQueueStatus.getQueueID() : SimulationJobStatus.SimulationQueueID.QUEUE_ID_WAITING; + + + } + } + + public SimulationStateMachine(KeyValue simKey, int jobIndex){ + this.simKey = simKey; + this.jobIndex = jobIndex; + updateSolverProcessTimestamp(); + } + + private void updateSolverProcessTimestamp( ) { + solverProcessTimestamp = System.currentTimeMillis(); + } + + /** + * set to specified time (for mass setting) + * @param solverProcessTimestamp + */ + void setSolverProcessTimestamp(long solverProcessTimestamp) { + this.solverProcessTimestamp = solverProcessTimestamp; + } + + public KeyValue getSimKey() { + return simKey; + } + + public int getJobIndex() { + return jobIndex; + } + + + long getSolverProcessTimestamp() { + return solverProcessTimestamp; + } + + protected boolean isWorkerEventOkay(WorkerEvent workerEvent, SimulationDatabase simulationDatabase) throws SQLException, DataAccessException { + VCSimulationDataIdentifier vcSimDataID = workerEvent.getVCSimulationDataIdentifier(); + int workerEventTaskID = workerEvent.getTaskID(); + if (vcSimDataID == null) { + VCMongoMessage.sendInfo("onWorkerEvent() ignoring WorkerEvent - no SimID in message): "+workerEvent.show()); + return false; + } + KeyValue simKey = vcSimDataID.getSimulationKey(); + SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(simKey, jobIndex); + + if (oldSimulationJobStatus == null){ + VCMongoMessage.sendInfo("onWorkerEvent() ignoring WorkerEvent, no current SimulationJobStatus: "+workerEvent.show()); + return false; + } + if (oldSimulationJobStatus.getSchedulerStatus().isDone() || oldSimulationJobStatus.getTaskID() > workerEventTaskID){ + VCMongoMessage.sendInfo("onWorkerEvent() ignoring outdated WorkerEvent, (currState="+oldSimulationJobStatus.getSchedulerStatus().getDescription()+"): "+workerEvent.show()); + return false; + } + return true; + } + + private SimulationJobStatus produceStateFromWorkerEvent( + WorkerEvent workerEvent, + SimulationJobStatus oldSimulationJobStatus){ + + SimulationExecutionStatus oldSimExeStatus = oldSimulationJobStatus.getSimulationExecutionStatus(); + SimulationQueueEntryStatus oldQueueStatus = oldSimulationJobStatus.getSimulationQueueEntryStatus(); + SimulationJobStatus.SchedulerStatus oldSchedulerStatus = oldSimulationJobStatus.getSchedulerStatus(); + VCSimulationDataIdentifier vcSimDataID = workerEvent.getVCSimulationDataIdentifier(); + + int taskID = oldSimulationJobStatus.getTaskID(); + + CurrentState currentState = new CurrentState(oldSimExeStatus, oldQueueStatus, oldSimulationJobStatus); + + // + // status information (initialized as if new record) + // + Date startDate = currentState.startDate; + Date lastUpdateDate = currentState.lastUpdateDate; + Date endDate = currentState.endDate; + boolean hasData = currentState.hasData; + HtcJobID htcJobID = currentState.htcJobID; + String computeHost = currentState.computeHost; + VCellServerID vcServerID = currentState.vcServerID; + Date submitDate = currentState.submitDate; + Date queueDate = currentState.queueDate; + int queuePriority = currentState.queuePriority; + SimulationJobStatus.SimulationQueueID simQueueID = currentState.simQueueID; + + // + // update using new information from event + // + if (workerEvent.getHtcJobID()!=null){ + htcJobID = workerEvent.getHtcJobID(); + } + if (workerEvent.getHostName()!=null){ + computeHost = workerEvent.getHostName(); + } + SimulationMessage workerEventSimulationMessage = workerEvent.getSimulationMessage(); + if (workerEventSimulationMessage.getHtcJobId()!=null){ + htcJobID = workerEventSimulationMessage.getHtcJobId(); + } + + + SimulationJobStatus newJobStatus = null; + + if (workerEvent.isAcceptedEvent()) { + // + // job message accepted by HtcSimulationWorker and sent to Scheduler (PBS/SGE/SLURM) (with a htcJobID) ... previous state should be "WAITING" + // + if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued()) { + // new queue status + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + + // new exe status + lastUpdateDate = new Date(); + startDate = lastUpdateDate; + endDate = null; + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.DISPATCHED, + taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); + } + + } else if (workerEvent.isStartingEvent()) { + // only update database when the job event changes from started to running. The later progress event will not be recorded. + if ( oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()) { + // new queue status + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + + // new exe status + lastUpdateDate = new Date(); + if (startDate == null){ + startDate = lastUpdateDate; + } + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, + taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); + } + + } else if (workerEvent.isNewDataEvent()) { + if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ + + if (!oldSchedulerStatus.isRunning() || simQueueID != SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL || hasData==false){ + + // new queue status + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + + // new exe status + if (startDate == null){ + startDate = lastUpdateDate; + } + hasData = true; + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, + taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); + } + } + + } else if (workerEvent.isProgressEvent() || workerEvent.isWorkerAliveEvent()) { + if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ + + + if (!oldSchedulerStatus.isRunning() || simQueueID != SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL){ + // new queue status + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + + // new exe status + if (startDate == null){ + startDate = lastUpdateDate; + } + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, + taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); + + }else if (oldSchedulerStatus.isRunning()){ + if (oldSimExeStatus != null) { + // new queue status + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, + taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); + } + } + } + + } else if (workerEvent.isCompletedEvent()) { + if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ + // new queue status + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + // new exe status + endDate = new Date(); + hasData = true; + + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.COMPLETED, + taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); + } + + } else if (workerEvent.isFailedEvent()) { + if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ + // new queue status + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + // new exe status + endDate = new Date(); + + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.FAILED, + taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); + + } + } else if (workerEvent.isWorkerExitErrorEvent()) { + if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ + // new queue status + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + // new exe status + endDate = new Date(); + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); + + SimulationMessage simulationMessage = SimulationMessage.workerFailure("solver stopped unexpectedly, "+workerEventSimulationMessage.getDisplayMessage()); + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.FAILED, + taskID, simulationMessage, newQueueStatus, newExeStatus); + + } + } + + return newJobStatus; + } + + public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDatabase simulationDatabase, VCMessageSession session) throws DataAccessException, VCMessagingException, SQLException { + updateSolverProcessTimestamp(); + WorkerEventMessage workerEventMessage = new WorkerEventMessage(workerEvent); + VCMongoMessage.sendWorkerEvent(workerEventMessage); + + String userName = workerEvent.getUserName(); // as the filter of the client + int workerEventTaskID = workerEvent.getTaskID(); + + if (lg.isTraceEnabled()) lg.trace("onWorkerEventMessage[" + workerEvent.getEventTypeID() + "," + workerEvent.getSimulationMessage() + "][simid=" + workerEvent.getVCSimulationDataIdentifier() + ",job=" + jobIndex + ",task=" + workerEventTaskID + "]"); + + if (!isWorkerEventOkay(workerEvent, simulationDatabase)){ + return; + } + + VCSimulationDataIdentifier vcSimDataID = workerEvent.getVCSimulationDataIdentifier(); + KeyValue simKey = vcSimDataID.getSimulationKey(); + SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(simKey, jobIndex); + + SimulationJobStatus.SchedulerStatus oldSchedulerStatus = oldSimulationJobStatus.getSchedulerStatus(); + SimulationJobStatus newJobStatus = produceStateFromWorkerEvent(workerEvent, oldSimulationJobStatus); + + if (newJobStatus!=null){ + if (!newJobStatus.compareEqual(oldSimulationJobStatus) || workerEvent.isProgressEvent() || workerEvent.isNewDataEvent()) { + Double progress = workerEvent.getProgress(); + Double timepoint = workerEvent.getTimePoint(); + RunningStateInfo runningStateInfo = null; + if (progress != null && timepoint != null){ + runningStateInfo = new RunningStateInfo(progress,timepoint); + } + simulationDatabase.updateSimulationJobStatus(newJobStatus,runningStateInfo); + StatusMessage msgForClient = new StatusMessage(newJobStatus, userName, progress, timepoint); + + // TODO: Implement messaging to client + // msgForClient.sendToClient(session); + if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); + } else { + simulationDatabase.updateSimulationJobStatus(newJobStatus); + StatusMessage msgForClient = new StatusMessage(newJobStatus, userName, null, null); + // TODO: Implement messaging to client + // msgForClient.sendToClient(session); + if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); + } + }else if (workerEvent.isProgressEvent() || workerEvent.isNewDataEvent()){ + Double progress = workerEvent.getProgress(); + Double timepoint = workerEvent.getTimePoint(); + RunningStateInfo runningStateInfo = null; + if (progress!=null && timepoint!=null){ + runningStateInfo = new RunningStateInfo(progress,timepoint); + } + simulationDatabase.updateSimulationJobStatus(oldSimulationJobStatus,runningStateInfo); + StatusMessage msgForClient = new StatusMessage(oldSimulationJobStatus, userName, progress, timepoint); + // TODO: Implement messaging to client + // msgForClient.sendToClient(session); + if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); + }else{ + VCMongoMessage.sendInfo("onWorkerEvent() ignoring WorkerEvent (currState="+oldSchedulerStatus.getDescription()+"): "+workerEvent.show()); + } // addStateMachineTransition(new StateMachineTransition(new WorkerStateMachineEvent(taskID, workerEvent), oldSimulationJobStatus, newJobStatus)); - } - - public synchronized void onStartRequest(User user, VCSimulationIdentifier vcSimID, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, DataAccessException, SQLException { - - if (!user.equals(vcSimID.getOwner())) { - lg.error(user + " is not authorized to start simulation (key=" + simKey + ")"); - StatusMessage message = new StatusMessage(new SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, 0, null, - SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to start this simulation!"), null, null), user.getName(), null, null); - message.sendToClient(session); - VCMongoMessage.sendInfo("onStartRequest("+vcSimID.getID()+") ignoring start simulation request - wrong user): simID="+vcSimID); - return; - } - - SimulationJobStatus newJobStatus = saveSimulationStartRequest(vcSimID, jobIndex, simulationDatabase); -// addStateMachineTransition(new StateMachineTransition(new StartStateMachineEvent(newTaskID), oldSimulationJobStatus, newJobStatus)); - - StatusMessage message = new StatusMessage(newJobStatus, user.getName(), null, null); - message.sendToClient(session); - } - - public static SimulationJobStatus saveSimulationStartRequest(VCSimulationIdentifier vcSimID, int jobIndex, SimulationDatabase simulationDatabase) throws DataAccessException, SQLException { - // - // get latest simulation job task (if any). - // - SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(vcSimID.getSimulationKey(), jobIndex); - int oldTaskID = -1; - if (oldSimulationJobStatus != null){ - oldTaskID = oldSimulationJobStatus.getTaskID(); - } - // if already started by another thread - if (oldSimulationJobStatus != null && !oldSimulationJobStatus.getSchedulerStatus().isDone()) { - VCMongoMessage.sendInfo("onStartRequest("+ vcSimID.getID()+") ignoring start simulation request - (currentSimJobStatus:"+oldSimulationJobStatus.getSchedulerStatus().getDescription()+"): simID="+ vcSimID); - throw new RuntimeException("Can't start, simulation[" + vcSimID + "] job [" + jobIndex + "] task [" + oldTaskID + "] is running already ("+oldSimulationJobStatus.getSchedulerStatus().getDescription()+")"); - } - - int newTaskID; - - if (oldTaskID > -1){ - // calculate new task - newTaskID = (oldTaskID & SimulationStatus.TASKID_USERCOUNTER_MASK) + SimulationStatus.TASKID_USERINCREMENT; - }else{ - // first task, start with 0 - newTaskID = 0; - } - - Date currentDate = new Date(); - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationJobStatus.SimulationQueueID.QUEUE_ID_WAITING); - - // new exe status - Date lastUpdateDate = new Date(); - String computeHost = null; - Date startDate = null; - Date endDate = null; - HtcJobID htcJobID = null; - boolean hasData = false; - - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - - VCellServerID vcServerID = VCellServerID.getSystemServerID(); - Date submitDate = currentDate; - - SimulationJobStatus newJobStatus = new SimulationJobStatus(vcServerID, vcSimID, jobIndex, submitDate, SchedulerStatus.WAITING, - newTaskID, SimulationMessage.MESSAGE_JOB_WAITING, newQueueStatus, newExeStatus); - - simulationDatabase.insertSimulationJobStatus(newJobStatus); - return newJobStatus; - } - - - public synchronized void onDispatch(Simulation simulation, SimulationJobStatus oldSimulationJobStatus, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, DataAccessException, SQLException { - updateSolverProcessTimestamp(); - VCSimulationIdentifier vcSimID = oldSimulationJobStatus.getVCSimulationIdentifier(); - int taskID = oldSimulationJobStatus.getTaskID(); - - if (!oldSimulationJobStatus.getSchedulerStatus().isWaiting()) { - VCMongoMessage.sendInfo("onDispatch("+vcSimID.getID()+") Can't start, simulation[" + vcSimID + "] job [" + jobIndex + "] task [" + taskID + "] is already dispatched ("+oldSimulationJobStatus.getSchedulerStatus().getDescription()+")"); - throw new RuntimeException("Can't start, simulation[" + vcSimID + "] job [" + jobIndex + "] task [" + taskID + "] is already dispatched ("+oldSimulationJobStatus.getSchedulerStatus().getDescription()+")"); - } - - FieldDataIdentifierSpec[] fieldDataIdentifierSpecs = simulationDatabase.getFieldDataIdentifierSpecs(simulation); - //Check if user wants long running sims activated in SlurmProxy.generateScript(...) - //only happens if user is allowed to be power user (entry in vc_specialusers table) and - //has checked the 'timeoutDisabledCheckBox' in SolverTaskDescriptionAdvancedPanel on the client-side GUI - boolean isPowerUser = simulation.getSolverTaskDescription().isTimeoutDisabled();//Set from GUI - if(isPowerUser) {//Check if user allowed to be power user for 'special1' long running sims (see User.SPECIALS and vc_specialusers table) - User.SpecialUser myUser = simulationDatabase.getUser(simulation.getVersion().getOwner().getName()); - //'powerUsers' (previously called 'special1') assigned to users by request to allow long running sims - isPowerUser = isPowerUser && Arrays.asList(myUser.getMySpecials()).contains(User.SPECIAL_CLAIM.powerUsers); - } - SimulationTask simulationTask = new SimulationTask(new SimulationJob(simulation, jobIndex, fieldDataIdentifierSpecs), taskID,null,isPowerUser); - - double estimatedMemMB = simulationTask.getEstimatedMemorySizeMB(); - double htcMinMemoryMB = Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.htcMinMemoryMB)); - double htcMaxMemoryMB = Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.htcMaxMemoryMB)); - double requestedMemoryMB = Math.max(estimatedMemMB, htcMinMemoryMB); - - final SimulationJobStatus newSimJobStatus; - if (requestedMemoryMB > htcMaxMemoryMB) { - // - // fail the simulation - // - Date currentDate = new Date(); - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - SimulationExecutionStatus newSimExeStatus = new SimulationExecutionStatus(null, null, new Date(), null, false, null); - newSimJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(),vcSimID,jobIndex, - oldSimulationJobStatus.getSubmitDate(),SchedulerStatus.FAILED,taskID, - SimulationMessage.jobFailed("simulation required "+estimatedMemMB+"MB of memory, only "+htcMaxMemoryMB+"MB allowed"), - newQueueStatus,newSimExeStatus); - - simulationDatabase.updateSimulationJobStatus(newSimJobStatus); - - StatusMessage message = new StatusMessage(newSimJobStatus, simulation.getVersion().getOwner().getName(), null, null); - message.sendToClient(session); - - }else{ - // - // dispatch the simulation, new queue status - // - Date currentDate = new Date(); - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationJobStatus.SimulationQueueID.QUEUE_ID_SIMULATIONJOB); - SimulationExecutionStatus newSimExeStatus = new SimulationExecutionStatus(null, null, new Date(), null, false, null); - newSimJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(),vcSimID,jobIndex, - oldSimulationJobStatus.getSubmitDate(),SchedulerStatus.DISPATCHED,taskID, - SimulationMessage.MESSAGE_JOB_DISPATCHED, - newQueueStatus,newSimExeStatus); - - SimulationTaskMessage simTaskMessage = new SimulationTaskMessage(simulationTask); - simTaskMessage.sendSimulationTask(session); - - simulationDatabase.updateSimulationJobStatus(newSimJobStatus); - - StatusMessage message = new StatusMessage(newSimJobStatus, simulation.getVersion().getOwner().getName(), null, null); - message.sendToClient(session); - - } + } + + public synchronized StatusMessage onStartRequest(User user, VCSimulationIdentifier vcSimID, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, DataAccessException, SQLException { + + if (!user.equals(vcSimID.getOwner())) { + lg.error(user + " is not authorized to start simulation (key=" + simKey + ")"); + SimulationJobStatus simulationJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, 0, null, + SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to start this simulation!"), null, null); + StatusMessage message = new StatusMessage(simulationJobStatus, user.getName(), null, null); + VCMongoMessage.sendInfo("onStartRequest("+vcSimID.getID()+") ignoring start simulation request - wrong user): simID="+vcSimID); + return message; + } + + SimulationJobStatus newJobStatus = saveSimulationStartRequest(vcSimID, jobIndex, simulationDatabase); + + return new StatusMessage(newJobStatus, user.getName(), null, null); + } + + public static SimulationJobStatus saveSimulationStartRequest(VCSimulationIdentifier vcSimID, int jobIndex, SimulationDatabase simulationDatabase) throws DataAccessException, SQLException { + // + // get latest simulation job task (if any). + // + SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(vcSimID.getSimulationKey(), jobIndex); + int oldTaskID = -1; + if (oldSimulationJobStatus != null){ + oldTaskID = oldSimulationJobStatus.getTaskID(); + } + // if already started by another thread + if (oldSimulationJobStatus != null && !oldSimulationJobStatus.getSchedulerStatus().isDone()) { + VCMongoMessage.sendInfo("onStartRequest("+ vcSimID.getID()+") ignoring start simulation request - (currentSimJobStatus:"+oldSimulationJobStatus.getSchedulerStatus().getDescription()+"): simID="+ vcSimID); + throw new RuntimeException("Can't start, simulation[" + vcSimID + "] job [" + jobIndex + "] task [" + oldTaskID + "] is running already ("+oldSimulationJobStatus.getSchedulerStatus().getDescription()+")"); + } + + int newTaskID; + + if (oldTaskID > -1){ + // calculate new task + newTaskID = (oldTaskID & SimulationStatus.TASKID_USERCOUNTER_MASK) + SimulationStatus.TASKID_USERINCREMENT; + }else{ + // first task, start with 0 + newTaskID = 0; + } + + Date currentDate = new Date(); + // new queue status + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationJobStatus.SimulationQueueID.QUEUE_ID_WAITING); + + // new exe status + Date lastUpdateDate = new Date(); + boolean hasData = false; + + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(null, null, lastUpdateDate, null, hasData, null); + + VCellServerID vcServerID = VCellServerID.getSystemServerID(); + Date submitDate = currentDate; + + SimulationJobStatus newJobStatus = new SimulationJobStatus(vcServerID, vcSimID, jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.WAITING, + newTaskID, SimulationMessage.MESSAGE_JOB_WAITING, newQueueStatus, newExeStatus); + + simulationDatabase.insertSimulationJobStatus(newJobStatus); + return newJobStatus; + } + + + public synchronized void onDispatch(Simulation simulation, SimulationJobStatus oldSimulationJobStatus, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, DataAccessException, SQLException { + updateSolverProcessTimestamp(); + VCSimulationIdentifier vcSimID = oldSimulationJobStatus.getVCSimulationIdentifier(); + int taskID = oldSimulationJobStatus.getTaskID(); + + if (!oldSimulationJobStatus.getSchedulerStatus().isWaiting()) { + VCMongoMessage.sendInfo("onDispatch("+vcSimID.getID()+") Can't start, simulation[" + vcSimID + "] job [" + jobIndex + "] task [" + taskID + "] is already dispatched ("+oldSimulationJobStatus.getSchedulerStatus().getDescription()+")"); + throw new RuntimeException("Can't start, simulation[" + vcSimID + "] job [" + jobIndex + "] task [" + taskID + "] is already dispatched ("+oldSimulationJobStatus.getSchedulerStatus().getDescription()+")"); + } + + FieldDataIdentifierSpec[] fieldDataIdentifierSpecs = simulationDatabase.getFieldDataIdentifierSpecs(simulation); + //Check if user wants long running sims activated in SlurmProxy.generateScript(...) + //only happens if user is allowed to be power user (entry in vc_specialusers table) and + //has checked the 'timeoutDisabledCheckBox' in SolverTaskDescriptionAdvancedPanel on the client-side GUI + boolean isPowerUser = simulation.getSolverTaskDescription().isTimeoutDisabled();//Set from GUI + if(isPowerUser) {//Check if user allowed to be power user for 'special1' long running sims (see User.SPECIALS and vc_specialusers table) + User.SpecialUser myUser = simulationDatabase.getUser(simulation.getVersion().getOwner().getName()); + //'powerUsers' (previously called 'special1') assigned to users by request to allow long running sims + isPowerUser = isPowerUser && Arrays.asList(myUser.getMySpecials()).contains(User.SPECIAL_CLAIM.powerUsers); + } + SimulationTask simulationTask = new SimulationTask(new SimulationJob(simulation, jobIndex, fieldDataIdentifierSpecs), taskID,null,isPowerUser); + + double estimatedMemMB = simulationTask.getEstimatedMemorySizeMB(); + double htcMinMemoryMB = Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.htcMinMemoryMB)); + double htcMaxMemoryMB = Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.htcMaxMemoryMB)); + double requestedMemoryMB = Math.max(estimatedMemMB, htcMinMemoryMB); + + final SimulationJobStatus newSimJobStatus; + if (requestedMemoryMB > htcMaxMemoryMB) { + // + // fail the simulation + // + Date currentDate = new Date(); + // new queue status + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationExecutionStatus newSimExeStatus = new SimulationExecutionStatus(null, null, new Date(), null, false, null); + newSimJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(),vcSimID,jobIndex, + oldSimulationJobStatus.getSubmitDate(), SimulationJobStatus.SchedulerStatus.FAILED,taskID, + SimulationMessage.jobFailed("simulation required "+estimatedMemMB+"MB of memory, only "+htcMaxMemoryMB+"MB allowed"), + newQueueStatus,newSimExeStatus); + + simulationDatabase.updateSimulationJobStatus(newSimJobStatus); + + StatusMessage message = new StatusMessage(newSimJobStatus, simulation.getVersion().getOwner().getName(), null, null); + message.sendToClient(session); + + }else{ + // + // dispatch the simulation, new queue status + // + Date currentDate = new Date(); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationJobStatus.SimulationQueueID.QUEUE_ID_SIMULATIONJOB); + SimulationExecutionStatus newSimExeStatus = new SimulationExecutionStatus(null, null, new Date(), null, false, null); + newSimJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(),vcSimID,jobIndex, + oldSimulationJobStatus.getSubmitDate(), SimulationJobStatus.SchedulerStatus.DISPATCHED,taskID, + SimulationMessage.MESSAGE_JOB_DISPATCHED, + newQueueStatus,newSimExeStatus); + + SimulationTaskMessage simTaskMessage = new SimulationTaskMessage(simulationTask); + simTaskMessage.sendSimulationTask(session); + + simulationDatabase.updateSimulationJobStatus(newSimJobStatus); + + cbit.vcell.message.messages.StatusMessage message = new cbit.vcell.message.messages.StatusMessage(newSimJobStatus, simulation.getVersion().getOwner().getName(), null, null); + message.sendToClient(session); + + } // addStateMachineTransition(new StateMachineTransition(new DispatchStateMachineEvent(taskID), oldSimulationJobStatus, newSimJobStatus)); - } - - public synchronized void onStopRequest(User user, SimulationJobStatus simJobStatus, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, DataAccessException, SQLException { - updateSolverProcessTimestamp(); - - if (!user.equals(simJobStatus.getVCSimulationIdentifier().getOwner())) { - lg.error(user + " is not authorized to stop simulation (key=" + simKey + ")"); - StatusMessage message = new StatusMessage(new SimulationJobStatus(VCellServerID.getSystemServerID(), simJobStatus.getVCSimulationIdentifier(), 0, null, - SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to stop this simulation!"), null, null), user.getName(), null, null); - message.sendToClient(session); - VCMongoMessage.sendInfo("onStopRequest("+simJobStatus.getVCSimulationIdentifier()+") ignoring stop simulation request - wrong user)"); - return; - } - - // stop latest task if active - SchedulerStatus schedulerStatus = simJobStatus.getSchedulerStatus(); - int taskID = simJobStatus.getTaskID(); - - if (schedulerStatus.isActive()){ - SimulationQueueEntryStatus simQueueEntryStatus = simJobStatus.getSimulationQueueEntryStatus(); - SimulationExecutionStatus simExeStatus = simJobStatus.getSimulationExecutionStatus(); - SimulationJobStatus newJobStatus = new SimulationJobStatus(simJobStatus.getServerID(),simJobStatus.getVCSimulationIdentifier(),jobIndex,simJobStatus.getSubmitDate(), - SchedulerStatus.STOPPED,taskID,SimulationMessage.solverStopped("simulation stopped by user"),simQueueEntryStatus,simExeStatus); - - // - // send stopSimulation to serviceControl topic - // - if (lg.isTraceEnabled()) lg.trace("send " + MessageConstants.MESSAGE_TYPE_STOPSIMULATION_VALUE + " to " + VCellTopic.ServiceControlTopic.getName() + " topic"); - VCMessage msg = session.createMessage(); - msg.setStringProperty(VCMessagingConstants.MESSAGE_TYPE_PROPERTY, MessageConstants.MESSAGE_TYPE_STOPSIMULATION_VALUE); - msg.setLongProperty(MessageConstants.SIMKEY_PROPERTY, Long.parseLong(simKey + "")); - msg.setIntProperty(MessageConstants.JOBINDEX_PROPERTY, jobIndex); - msg.setIntProperty(MessageConstants.TASKID_PROPERTY, taskID); - msg.setStringProperty(VCMessagingConstants.USERNAME_PROPERTY, user.getName()); - if (simExeStatus.getHtcJobID()!=null){ - msg.setStringProperty(MessageConstants.HTCJOBID_PROPERTY, simExeStatus.getHtcJobID().toDatabase()); - } - session.sendTopicMessage(VCellTopic.ServiceControlTopic, msg); - - simulationDatabase.updateSimulationJobStatus(newJobStatus); -// addStateMachineTransition(new StateMachineTransition(new StopStateMachineEvent(taskID), simJobStatus, newJobStatus)); - - // update client - StatusMessage message = new StatusMessage(newJobStatus, user.getName(), null, null); - message.sendToClient(session); - } - } - - public synchronized void onSystemAbort(SimulationJobStatus oldJobStatus, String failureMessage, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, UpdateSynchronizationException, DataAccessException, SQLException { - updateSolverProcessTimestamp(); - - int taskID = oldJobStatus.getTaskID(); - - // - // status information (initialized as if new record) - // - Date startDate = null; - boolean hasData = false; - HtcJobID htcJobID = null; - String computeHost = null; - VCellServerID vcServerID = VCellServerID.getSystemServerID(); - Date submitDate = null; - Date queueDate = null; - int queuePriority = PRIORITY_DEFAULT; - - - // - // update using previously stored status (if available). - // - SimulationExecutionStatus oldSimExeStatus = oldJobStatus.getSimulationExecutionStatus(); - if (oldSimExeStatus!=null && oldSimExeStatus.getStartDate()!=null){ - startDate = oldSimExeStatus.getStartDate(); - } - if (oldSimExeStatus!=null && oldSimExeStatus.hasData()){ - hasData = true; - } - if (oldSimExeStatus!=null && oldSimExeStatus.getComputeHost()!=null){ - computeHost = oldSimExeStatus.getComputeHost(); - } - if (oldSimExeStatus!=null && oldSimExeStatus.getHtcJobID()!=null){ - htcJobID = oldSimExeStatus.getHtcJobID(); - } - vcServerID = oldJobStatus.getServerID(); - submitDate = oldJobStatus.getSubmitDate(); - SimulationQueueEntryStatus oldQueueStatus = oldJobStatus.getSimulationQueueEntryStatus(); - if (oldQueueStatus!=null && oldQueueStatus.getQueueDate()!=null){ - queueDate = oldQueueStatus.getQueueDate(); - } - if (oldQueueStatus!=null){ - queuePriority = oldQueueStatus.getQueuePriority(); - } - - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - - Date endDate = new Date(); - Date lastUpdateDate = new Date(); - - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - - SimulationJobStatus newJobStatus = new SimulationJobStatus(vcServerID, oldJobStatus.getVCSimulationIdentifier(), jobIndex, submitDate, SchedulerStatus.FAILED, - taskID, SimulationMessage.jobFailed(failureMessage), newQueueStatus, newExeStatus); - - simulationDatabase.updateSimulationJobStatus(newJobStatus); + } + + public synchronized StatusMessage onStopRequest(User user, SimulationJobStatus simJobStatus, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, DataAccessException, SQLException { + updateSolverProcessTimestamp(); + + if (!user.equals(simJobStatus.getVCSimulationIdentifier().getOwner())) { + lg.error(user + " is not authorized to stop simulation (key=" + simKey + ")"); + SimulationJobStatus simulationJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(), simJobStatus.getVCSimulationIdentifier(), 0, null, + SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to stop this simulation!"), null, null); + + VCMongoMessage.sendInfo("onStopRequest("+simJobStatus.getVCSimulationIdentifier()+") ignoring stop simulation request - wrong user)"); + return new StatusMessage(simulationJobStatus, user.getName(), null, null); + } + + // stop latest task if active + SimulationJobStatus.SchedulerStatus schedulerStatus = simJobStatus.getSchedulerStatus(); + int taskID = simJobStatus.getTaskID(); + + if (schedulerStatus.isActive()){ + SimulationQueueEntryStatus simQueueEntryStatus = simJobStatus.getSimulationQueueEntryStatus(); + SimulationExecutionStatus simExeStatus = simJobStatus.getSimulationExecutionStatus(); + SimulationJobStatus newJobStatus = new SimulationJobStatus(simJobStatus.getServerID(),simJobStatus.getVCSimulationIdentifier(),jobIndex,simJobStatus.getSubmitDate(), + SimulationJobStatus.SchedulerStatus.STOPPED,taskID,SimulationMessage.solverStopped("simulation stopped by user"),simQueueEntryStatus,simExeStatus); + + + if (lg.isTraceEnabled()) lg.trace("send " + MessageConstants.MESSAGE_TYPE_STOPSIMULATION_VALUE + " to " + VCellTopic.ServiceControlTopic.getName() + " topic"); + SimulationJobStatus simulationJobStatusRecord = new SimulationJobStatus( + null, new VCSimulationIdentifier(simKey, user), jobIndex, simJobStatus.getSubmitDate(), + SimulationJobStatus.SchedulerStatus.STOPPED, taskID, simJobStatus.getSimulationMessage(), simQueueEntryStatus, simExeStatus + ); + + simulationDatabase.updateSimulationJobStatus(newJobStatus); + + return new StatusMessage(simulationJobStatusRecord, user.getName(), null, null); + } + return null; + } + + public synchronized void onSystemAbort(SimulationJobStatus oldJobStatus, String failureMessage, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, UpdateSynchronizationException, DataAccessException, SQLException { + updateSolverProcessTimestamp(); + + int taskID = oldJobStatus.getTaskID(); + + // + // update using previously stored status (if available). + // + CurrentState currentState = new CurrentState(oldJobStatus.getSimulationExecutionStatus(), oldJobStatus.getSimulationQueueEntryStatus(), oldJobStatus); + + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentState.queueDate, currentState.queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + + Date endDate = new Date(); + Date lastUpdateDate = new Date(); + + SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(currentState.startDate, currentState.computeHost, lastUpdateDate, endDate, currentState.hasData, currentState.htcJobID); + + SimulationJobStatus newJobStatus = new SimulationJobStatus(currentState.vcServerID, oldJobStatus.getVCSimulationIdentifier(), jobIndex, currentState.submitDate, SimulationJobStatus.SchedulerStatus.FAILED, + taskID, SimulationMessage.jobFailed(failureMessage), newQueueStatus, newExeStatus); + + simulationDatabase.updateSimulationJobStatus(newJobStatus); // addStateMachineTransition(new StateMachineTransition(new AbortStateMachineEvent(taskID, failureMessage), oldJobStatus, newJobStatus)); - String userName = VCMessagingConstants.USERNAME_PROPERTY_VALUE_ALL; - StatusMessage msgForClient = new StatusMessage(newJobStatus, userName, null, null); - msgForClient.sendToClient(session); - if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); - } - -// public int getLatestKnownTaskID() { -// int taskID = -1; -// for (StateMachineTransition transition : stateMachineTransitions){ -// if (transition.event.taskID!=null && transition.event.taskID>taskID){ -// taskID = transition.event.taskID; -// } -// if (transition.newSimJobStatus!=null && transition.newSimJobStatus.getTaskID()>taskID){ -// taskID = transition.newSimJobStatus.getTaskID(); -// } -// } -// return taskID; -// } -// + String userName = VCMessagingConstants.USERNAME_PROPERTY_VALUE_ALL; + cbit.vcell.message.messages.StatusMessage msgForClient = new cbit.vcell.message.messages.StatusMessage(newJobStatus, userName, null, null); + msgForClient.sendToClient(session); + if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); + } + } diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineCopy.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineCopy.java deleted file mode 100644 index 347ca70708..0000000000 --- a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineCopy.java +++ /dev/null @@ -1,576 +0,0 @@ -package cbit.vcell.message.server.dispatcher; - -import cbit.rmi.event.WorkerEvent; -import cbit.vcell.field.FieldDataIdentifierSpec; -import cbit.vcell.message.VCMessageSession; -import cbit.vcell.message.VCMessagingConstants; -import cbit.vcell.message.VCMessagingException; -import cbit.vcell.message.VCellTopic; -import cbit.vcell.message.messages.MessageConstants; -import cbit.vcell.message.messages.SimulationTaskMessage; -import cbit.vcell.message.messages.StatusMessage; -import cbit.vcell.message.messages.WorkerEventMessage; -import cbit.vcell.message.server.htc.HtcProxy; -import cbit.vcell.messaging.server.SimulationTask; -import cbit.vcell.mongodb.VCMongoMessage; -import cbit.vcell.resource.PropertyLoader; -import cbit.vcell.server.*; -import cbit.vcell.solver.*; -import cbit.vcell.solver.server.SimulationMessage; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vcell.util.DataAccessException; -import org.vcell.util.document.KeyValue; -import org.vcell.util.document.User; -import org.vcell.util.document.VCellServerID; - -import java.sql.SQLException; -import java.util.Arrays; -import java.util.Date; - -public class SimulationStateMachineCopy { - public static final Logger lg = LogManager.getLogger(cbit.vcell.message.server.dispatcher.SimulationStateMachine.class); - - // bitmapped counter so that allows 3 retries for each request (but preserves ordinal nature) - // bits 0-3: retry count - // bits 4-31: submit - // max retries must be less than 15. - public static final int TASKID_USERCOUNTER_MASK = SimulationStatus.TASKID_USERCOUNTER_MASK; - public static final int TASKID_RETRYCOUNTER_MASK = SimulationStatus.TASKID_RETRYCOUNTER_MASK; - public static final int TASKID_USERINCREMENT = SimulationStatus.TASKID_USERINCREMENT; - - public static final int PRIORITY_LOW = 0; - public static final int PRIORITY_DEFAULT = 5; - public static final int PRIORITY_HIGH = 9; - - private final KeyValue simKey; - private final int jobIndex; - - /** - * in memory storage of last time information about this job was received or status was unknown due - * to transient failure or system restart - */ - private long solverProcessTimestamp; - - private class CurrentState { - - public Date startDate; - public Date lastUpdateDate; - public Date endDate; - public boolean hasData; - public HtcJobID htcJobID; - public String computeHost; - public VCellServerID vcServerID; - public Date submitDate; - public Date queueDate; - public int queuePriority; - public SimulationJobStatus.SimulationQueueID simQueueID; - - public CurrentState(SimulationExecutionStatus oldSimExeStatus, - SimulationQueueEntryStatus oldQueueStatus, - SimulationJobStatus oldSimulationJobStatus){ - boolean isOldExeNull = oldSimExeStatus == null; - boolean isOldQueueNull = oldQueueStatus == null; - // - // status information (initialized as if new record) - // - startDate = !isOldExeNull && oldSimExeStatus.getStartDate()!=null ? oldSimExeStatus.getStartDate() :null; - lastUpdateDate = !isOldExeNull && oldSimExeStatus.getLatestUpdateDate()!=null ? oldSimExeStatus.getLatestUpdateDate() : null; - endDate = !isOldExeNull && oldSimExeStatus.getEndDate()!=null ? oldSimExeStatus.getEndDate() : null; - hasData = !isOldExeNull && oldSimExeStatus.hasData(); - htcJobID = !isOldExeNull && oldSimExeStatus.getHtcJobID()!=null ? oldSimExeStatus.getHtcJobID() : null; - computeHost = !isOldExeNull && oldSimExeStatus.getComputeHost()!=null ? oldSimExeStatus.getComputeHost() : null; - vcServerID = oldSimulationJobStatus.getServerID(); - submitDate = oldSimulationJobStatus.getSubmitDate(); - queueDate = !isOldQueueNull && oldQueueStatus.getQueueDate() != null ? oldQueueStatus.getQueueDate() : null; - queuePriority = !isOldQueueNull ? oldQueueStatus.getQueuePriority() : PRIORITY_DEFAULT; - simQueueID = !isOldQueueNull && oldQueueStatus.getQueueID()!=null ? oldQueueStatus.getQueueID() : SimulationJobStatus.SimulationQueueID.QUEUE_ID_WAITING; - - - } - } - - public SimulationStateMachineCopy(KeyValue simKey, int jobIndex){ - this.simKey = simKey; - this.jobIndex = jobIndex; - updateSolverProcessTimestamp(); - } - - private void updateSolverProcessTimestamp( ) { - solverProcessTimestamp = System.currentTimeMillis(); - } - - /** - * set to specified time (for mass setting) - * @param solverProcessTimestamp - */ - void setSolverProcessTimestamp(long solverProcessTimestamp) { - this.solverProcessTimestamp = solverProcessTimestamp; - } - - public KeyValue getSimKey() { - return simKey; - } - - public int getJobIndex() { - return jobIndex; - } - - - long getSolverProcessTimestamp() { - return solverProcessTimestamp; - } - - protected boolean isWorkerEventOkay(WorkerEvent workerEvent, SimulationDatabase simulationDatabase) throws SQLException, DataAccessException { - VCSimulationDataIdentifier vcSimDataID = workerEvent.getVCSimulationDataIdentifier(); - int workerEventTaskID = workerEvent.getTaskID(); - if (vcSimDataID == null) { - VCMongoMessage.sendInfo("onWorkerEvent() ignoring WorkerEvent - no SimID in message): "+workerEvent.show()); - return false; - } - KeyValue simKey = vcSimDataID.getSimulationKey(); - SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(simKey, jobIndex); - - if (oldSimulationJobStatus == null){ - VCMongoMessage.sendInfo("onWorkerEvent() ignoring WorkerEvent, no current SimulationJobStatus: "+workerEvent.show()); - return false; - } - if (oldSimulationJobStatus.getSchedulerStatus().isDone() || oldSimulationJobStatus.getTaskID() > workerEventTaskID){ - VCMongoMessage.sendInfo("onWorkerEvent() ignoring outdated WorkerEvent, (currState="+oldSimulationJobStatus.getSchedulerStatus().getDescription()+"): "+workerEvent.show()); - return false; - } - return true; - } - - private SimulationJobStatus produceStateFromWorkerEvent( - WorkerEvent workerEvent, - SimulationJobStatus oldSimulationJobStatus){ - - SimulationExecutionStatus oldSimExeStatus = oldSimulationJobStatus.getSimulationExecutionStatus(); - SimulationQueueEntryStatus oldQueueStatus = oldSimulationJobStatus.getSimulationQueueEntryStatus(); - SimulationJobStatus.SchedulerStatus oldSchedulerStatus = oldSimulationJobStatus.getSchedulerStatus(); - VCSimulationDataIdentifier vcSimDataID = workerEvent.getVCSimulationDataIdentifier(); - - int taskID = oldSimulationJobStatus.getTaskID(); - - CurrentState currentState = new CurrentState(oldSimExeStatus, oldQueueStatus, oldSimulationJobStatus); - - // - // status information (initialized as if new record) - // - Date startDate = currentState.startDate; - Date lastUpdateDate = currentState.lastUpdateDate; - Date endDate = currentState.endDate; - boolean hasData = currentState.hasData; - HtcJobID htcJobID = currentState.htcJobID; - String computeHost = currentState.computeHost; - VCellServerID vcServerID = currentState.vcServerID; - Date submitDate = currentState.submitDate; - Date queueDate = currentState.queueDate; - int queuePriority = currentState.queuePriority; - SimulationJobStatus.SimulationQueueID simQueueID = currentState.simQueueID; - - // - // update using new information from event - // - if (workerEvent.getHtcJobID()!=null){ - htcJobID = workerEvent.getHtcJobID(); - } - if (workerEvent.getHostName()!=null){ - computeHost = workerEvent.getHostName(); - } - SimulationMessage workerEventSimulationMessage = workerEvent.getSimulationMessage(); - if (workerEventSimulationMessage.getHtcJobId()!=null){ - htcJobID = workerEventSimulationMessage.getHtcJobId(); - } - - - SimulationJobStatus newJobStatus = null; - - if (workerEvent.isAcceptedEvent()) { - // - // job message accepted by HtcSimulationWorker and sent to Scheduler (PBS/SGE/SLURM) (with a htcJobID) ... previous state should be "WAITING" - // - if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued()) { - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - - // new exe status - lastUpdateDate = new Date(); - startDate = lastUpdateDate; - endDate = null; - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.DISPATCHED, - taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); - } - - } else if (workerEvent.isStartingEvent()) { - // only update database when the job event changes from started to running. The later progress event will not be recorded. - if ( oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()) { - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - - // new exe status - lastUpdateDate = new Date(); - if (startDate == null){ - startDate = lastUpdateDate; - } - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, - taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); - } - - } else if (workerEvent.isNewDataEvent()) { - if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ - - if (!oldSchedulerStatus.isRunning() || simQueueID != SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL || hasData==false){ - - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - - // new exe status - if (startDate == null){ - startDate = lastUpdateDate; - } - hasData = true; - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, - taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); - } - } - - } else if (workerEvent.isProgressEvent() || workerEvent.isWorkerAliveEvent()) { - if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ - - - if (!oldSchedulerStatus.isRunning() || simQueueID != SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL){ - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - - // new exe status - if (startDate == null){ - startDate = lastUpdateDate; - } - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, - taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); - - }else if (oldSchedulerStatus.isRunning()){ - if (oldSimExeStatus != null) { - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, - taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); - } - } - } - - } else if (workerEvent.isCompletedEvent()) { - if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - // new exe status - endDate = new Date(); - hasData = true; - - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.COMPLETED, - taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); - } - - } else if (workerEvent.isFailedEvent()) { - if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - // new exe status - endDate = new Date(); - - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.FAILED, - taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); - - } - } else if (workerEvent.isWorkerExitErrorEvent()) { - if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - // new exe status - endDate = new Date(); - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - - SimulationMessage simulationMessage = SimulationMessage.workerFailure("solver stopped unexpectedly, "+workerEventSimulationMessage.getDisplayMessage()); - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.FAILED, - taskID, simulationMessage, newQueueStatus, newExeStatus); - - } - } - - return newJobStatus; - } - - public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDatabase simulationDatabase, VCMessageSession session) throws DataAccessException, VCMessagingException, SQLException { - updateSolverProcessTimestamp(); - WorkerEventMessage workerEventMessage = new WorkerEventMessage(workerEvent); - VCMongoMessage.sendWorkerEvent(workerEventMessage); - - String userName = workerEvent.getUserName(); // as the filter of the client - int workerEventTaskID = workerEvent.getTaskID(); - - if (lg.isTraceEnabled()) lg.trace("onWorkerEventMessage[" + workerEvent.getEventTypeID() + "," + workerEvent.getSimulationMessage() + "][simid=" + workerEvent.getVCSimulationDataIdentifier() + ",job=" + jobIndex + ",task=" + workerEventTaskID + "]"); - - if (!isWorkerEventOkay(workerEvent, simulationDatabase)){ - return; - } - - VCSimulationDataIdentifier vcSimDataID = workerEvent.getVCSimulationDataIdentifier(); - KeyValue simKey = vcSimDataID.getSimulationKey(); - SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(simKey, jobIndex); - - SimulationJobStatus.SchedulerStatus oldSchedulerStatus = oldSimulationJobStatus.getSchedulerStatus(); - SimulationJobStatus newJobStatus = produceStateFromWorkerEvent(workerEvent, oldSimulationJobStatus); - - if (newJobStatus!=null){ - if (!newJobStatus.compareEqual(oldSimulationJobStatus) || workerEvent.isProgressEvent() || workerEvent.isNewDataEvent()) { - Double progress = workerEvent.getProgress(); - Double timepoint = workerEvent.getTimePoint(); - RunningStateInfo runningStateInfo = null; - if (progress != null && timepoint != null){ - runningStateInfo = new RunningStateInfo(progress,timepoint); - } - simulationDatabase.updateSimulationJobStatus(newJobStatus,runningStateInfo); - StatusMessage msgForClient = new StatusMessage(newJobStatus, userName, progress, timepoint); - - // TODO: Implement messaging to client - // msgForClient.sendToClient(session); - if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); - } else { - simulationDatabase.updateSimulationJobStatus(newJobStatus); - StatusMessage msgForClient = new StatusMessage(newJobStatus, userName, null, null); - // TODO: Implement messaging to client - // msgForClient.sendToClient(session); - if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); - } - }else if (workerEvent.isProgressEvent() || workerEvent.isNewDataEvent()){ - Double progress = workerEvent.getProgress(); - Double timepoint = workerEvent.getTimePoint(); - RunningStateInfo runningStateInfo = null; - if (progress!=null && timepoint!=null){ - runningStateInfo = new RunningStateInfo(progress,timepoint); - } - simulationDatabase.updateSimulationJobStatus(oldSimulationJobStatus,runningStateInfo); - StatusMessage msgForClient = new StatusMessage(oldSimulationJobStatus, userName, progress, timepoint); - // TODO: Implement messaging to client - // msgForClient.sendToClient(session); - if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); - }else{ - VCMongoMessage.sendInfo("onWorkerEvent() ignoring WorkerEvent (currState="+oldSchedulerStatus.getDescription()+"): "+workerEvent.show()); - } -// addStateMachineTransition(new StateMachineTransition(new WorkerStateMachineEvent(taskID, workerEvent), oldSimulationJobStatus, newJobStatus)); - - } - - public synchronized StatusMessage onStartRequest(User user, VCSimulationIdentifier vcSimID, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, DataAccessException, SQLException { - - if (!user.equals(vcSimID.getOwner())) { - lg.error(user + " is not authorized to start simulation (key=" + simKey + ")"); - SimulationJobStatus simulationJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, 0, null, - SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to start this simulation!"), null, null); - StatusMessage message = new StatusMessage(simulationJobStatus, user.getName(), null, null); - VCMongoMessage.sendInfo("onStartRequest("+vcSimID.getID()+") ignoring start simulation request - wrong user): simID="+vcSimID); - return message; - } - - SimulationJobStatus newJobStatus = saveSimulationStartRequest(vcSimID, jobIndex, simulationDatabase); - - return new StatusMessage(newJobStatus, user.getName(), null, null); - } - - public static SimulationJobStatus saveSimulationStartRequest(VCSimulationIdentifier vcSimID, int jobIndex, SimulationDatabase simulationDatabase) throws DataAccessException, SQLException { - // - // get latest simulation job task (if any). - // - SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(vcSimID.getSimulationKey(), jobIndex); - int oldTaskID = -1; - if (oldSimulationJobStatus != null){ - oldTaskID = oldSimulationJobStatus.getTaskID(); - } - // if already started by another thread - if (oldSimulationJobStatus != null && !oldSimulationJobStatus.getSchedulerStatus().isDone()) { - VCMongoMessage.sendInfo("onStartRequest("+ vcSimID.getID()+") ignoring start simulation request - (currentSimJobStatus:"+oldSimulationJobStatus.getSchedulerStatus().getDescription()+"): simID="+ vcSimID); - throw new RuntimeException("Can't start, simulation[" + vcSimID + "] job [" + jobIndex + "] task [" + oldTaskID + "] is running already ("+oldSimulationJobStatus.getSchedulerStatus().getDescription()+")"); - } - - int newTaskID; - - if (oldTaskID > -1){ - // calculate new task - newTaskID = (oldTaskID & SimulationStatus.TASKID_USERCOUNTER_MASK) + SimulationStatus.TASKID_USERINCREMENT; - }else{ - // first task, start with 0 - newTaskID = 0; - } - - Date currentDate = new Date(); - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationJobStatus.SimulationQueueID.QUEUE_ID_WAITING); - - // new exe status - Date lastUpdateDate = new Date(); - boolean hasData = false; - - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(null, null, lastUpdateDate, null, hasData, null); - - VCellServerID vcServerID = VCellServerID.getSystemServerID(); - Date submitDate = currentDate; - - SimulationJobStatus newJobStatus = new SimulationJobStatus(vcServerID, vcSimID, jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.WAITING, - newTaskID, SimulationMessage.MESSAGE_JOB_WAITING, newQueueStatus, newExeStatus); - - simulationDatabase.insertSimulationJobStatus(newJobStatus); - return newJobStatus; - } - - - public synchronized void onDispatch(Simulation simulation, SimulationJobStatus oldSimulationJobStatus, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, DataAccessException, SQLException { - updateSolverProcessTimestamp(); - VCSimulationIdentifier vcSimID = oldSimulationJobStatus.getVCSimulationIdentifier(); - int taskID = oldSimulationJobStatus.getTaskID(); - - if (!oldSimulationJobStatus.getSchedulerStatus().isWaiting()) { - VCMongoMessage.sendInfo("onDispatch("+vcSimID.getID()+") Can't start, simulation[" + vcSimID + "] job [" + jobIndex + "] task [" + taskID + "] is already dispatched ("+oldSimulationJobStatus.getSchedulerStatus().getDescription()+")"); - throw new RuntimeException("Can't start, simulation[" + vcSimID + "] job [" + jobIndex + "] task [" + taskID + "] is already dispatched ("+oldSimulationJobStatus.getSchedulerStatus().getDescription()+")"); - } - - FieldDataIdentifierSpec[] fieldDataIdentifierSpecs = simulationDatabase.getFieldDataIdentifierSpecs(simulation); - //Check if user wants long running sims activated in SlurmProxy.generateScript(...) - //only happens if user is allowed to be power user (entry in vc_specialusers table) and - //has checked the 'timeoutDisabledCheckBox' in SolverTaskDescriptionAdvancedPanel on the client-side GUI - boolean isPowerUser = simulation.getSolverTaskDescription().isTimeoutDisabled();//Set from GUI - if(isPowerUser) {//Check if user allowed to be power user for 'special1' long running sims (see User.SPECIALS and vc_specialusers table) - User.SpecialUser myUser = simulationDatabase.getUser(simulation.getVersion().getOwner().getName()); - //'powerUsers' (previously called 'special1') assigned to users by request to allow long running sims - isPowerUser = isPowerUser && Arrays.asList(myUser.getMySpecials()).contains(User.SPECIAL_CLAIM.powerUsers); - } - SimulationTask simulationTask = new SimulationTask(new SimulationJob(simulation, jobIndex, fieldDataIdentifierSpecs), taskID,null,isPowerUser); - - double estimatedMemMB = simulationTask.getEstimatedMemorySizeMB(); - double htcMinMemoryMB = Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.htcMinMemoryMB)); - double htcMaxMemoryMB = Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.htcMaxMemoryMB)); - double requestedMemoryMB = Math.max(estimatedMemMB, htcMinMemoryMB); - - final SimulationJobStatus newSimJobStatus; - if (requestedMemoryMB > htcMaxMemoryMB) { - // - // fail the simulation - // - Date currentDate = new Date(); - // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - SimulationExecutionStatus newSimExeStatus = new SimulationExecutionStatus(null, null, new Date(), null, false, null); - newSimJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(),vcSimID,jobIndex, - oldSimulationJobStatus.getSubmitDate(), SimulationJobStatus.SchedulerStatus.FAILED,taskID, - SimulationMessage.jobFailed("simulation required "+estimatedMemMB+"MB of memory, only "+htcMaxMemoryMB+"MB allowed"), - newQueueStatus,newSimExeStatus); - - simulationDatabase.updateSimulationJobStatus(newSimJobStatus); - - StatusMessage message = new StatusMessage(newSimJobStatus, simulation.getVersion().getOwner().getName(), null, null); - message.sendToClient(session); - - }else{ - // - // dispatch the simulation, new queue status - // - Date currentDate = new Date(); - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationJobStatus.SimulationQueueID.QUEUE_ID_SIMULATIONJOB); - SimulationExecutionStatus newSimExeStatus = new SimulationExecutionStatus(null, null, new Date(), null, false, null); - newSimJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(),vcSimID,jobIndex, - oldSimulationJobStatus.getSubmitDate(), SimulationJobStatus.SchedulerStatus.DISPATCHED,taskID, - SimulationMessage.MESSAGE_JOB_DISPATCHED, - newQueueStatus,newSimExeStatus); - - SimulationTaskMessage simTaskMessage = new SimulationTaskMessage(simulationTask); - simTaskMessage.sendSimulationTask(session); - - simulationDatabase.updateSimulationJobStatus(newSimJobStatus); - - cbit.vcell.message.messages.StatusMessage message = new cbit.vcell.message.messages.StatusMessage(newSimJobStatus, simulation.getVersion().getOwner().getName(), null, null); - message.sendToClient(session); - - } -// addStateMachineTransition(new StateMachineTransition(new DispatchStateMachineEvent(taskID), oldSimulationJobStatus, newSimJobStatus)); - - } - - public synchronized StatusMessage onStopRequest(User user, SimulationJobStatus simJobStatus, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, DataAccessException, SQLException { - updateSolverProcessTimestamp(); - - if (!user.equals(simJobStatus.getVCSimulationIdentifier().getOwner())) { - lg.error(user + " is not authorized to stop simulation (key=" + simKey + ")"); - SimulationJobStatus simulationJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(), simJobStatus.getVCSimulationIdentifier(), 0, null, - SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to stop this simulation!"), null, null); - - VCMongoMessage.sendInfo("onStopRequest("+simJobStatus.getVCSimulationIdentifier()+") ignoring stop simulation request - wrong user)"); - return new StatusMessage(simulationJobStatus, user.getName(), null, null); - } - - // stop latest task if active - SimulationJobStatus.SchedulerStatus schedulerStatus = simJobStatus.getSchedulerStatus(); - int taskID = simJobStatus.getTaskID(); - - if (schedulerStatus.isActive()){ - SimulationQueueEntryStatus simQueueEntryStatus = simJobStatus.getSimulationQueueEntryStatus(); - SimulationExecutionStatus simExeStatus = simJobStatus.getSimulationExecutionStatus(); - SimulationJobStatus newJobStatus = new SimulationJobStatus(simJobStatus.getServerID(),simJobStatus.getVCSimulationIdentifier(),jobIndex,simJobStatus.getSubmitDate(), - SimulationJobStatus.SchedulerStatus.STOPPED,taskID,SimulationMessage.solverStopped("simulation stopped by user"),simQueueEntryStatus,simExeStatus); - - - if (lg.isTraceEnabled()) lg.trace("send " + MessageConstants.MESSAGE_TYPE_STOPSIMULATION_VALUE + " to " + VCellTopic.ServiceControlTopic.getName() + " topic"); - SimulationJobStatus simulationJobStatusRecord = new SimulationJobStatus( - null, new VCSimulationIdentifier(simKey, user), jobIndex, simJobStatus.getSubmitDate(), - SimulationJobStatus.SchedulerStatus.STOPPED, taskID, simJobStatus.getSimulationMessage(), simQueueEntryStatus, simExeStatus - ); - - simulationDatabase.updateSimulationJobStatus(newJobStatus); - - return new StatusMessage(simulationJobStatusRecord, user.getName(), null, null); - } - return null; - } - - public synchronized void onSystemAbort(SimulationJobStatus oldJobStatus, String failureMessage, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, UpdateSynchronizationException, DataAccessException, SQLException { - updateSolverProcessTimestamp(); - - int taskID = oldJobStatus.getTaskID(); - - // - // update using previously stored status (if available). - // - CurrentState currentState = new CurrentState(oldJobStatus.getSimulationExecutionStatus(), oldJobStatus.getSimulationQueueEntryStatus(), oldJobStatus); - - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentState.queueDate, currentState.queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); - - Date endDate = new Date(); - Date lastUpdateDate = new Date(); - - SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(currentState.startDate, currentState.computeHost, lastUpdateDate, endDate, currentState.hasData, currentState.htcJobID); - - SimulationJobStatus newJobStatus = new SimulationJobStatus(currentState.vcServerID, oldJobStatus.getVCSimulationIdentifier(), jobIndex, currentState.submitDate, SimulationJobStatus.SchedulerStatus.FAILED, - taskID, SimulationMessage.jobFailed(failureMessage), newQueueStatus, newExeStatus); - - simulationDatabase.updateSimulationJobStatus(newJobStatus); -// addStateMachineTransition(new StateMachineTransition(new AbortStateMachineEvent(taskID, failureMessage), oldJobStatus, newJobStatus)); - - String userName = VCMessagingConstants.USERNAME_PROPERTY_VALUE_ALL; - cbit.vcell.message.messages.StatusMessage msgForClient = new cbit.vcell.message.messages.StatusMessage(newJobStatus, userName, null, null); - msgForClient.sendToClient(session); - if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); - } - -} diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java index abcb9b061b..e48a74f558 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java @@ -46,7 +46,7 @@ public class SimulationStateMachineTest { public static String previousHtcPowerFloor = ""; private MockSimulationDB simulationDB; - private SimulationStateMachineCopy stateMachine; + private SimulationStateMachine stateMachine; @BeforeAll public static void setSystemProperties(){ @@ -74,7 +74,7 @@ public static void restoreSystemProperties(){ @BeforeEach public void setUp(){ simulationDB = new MockSimulationDB(); - stateMachine = new SimulationStateMachineCopy(simKey, jobIndex); + stateMachine = new SimulationStateMachine(simKey, jobIndex); } private record ChangedStateValues( From fc10c42befd4f4cbbdecbdaf1ff91a8ad44da1cc Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Thu, 29 Aug 2024 07:06:31 -0400 Subject: [PATCH 10/24] Test Messages Sent to Client --- .../dispatcher/SimulationStateMachine.java | 27 +++-- .../dispatcher/MockVCMessageSession.java | 113 ++++++++++++++++++ .../SimulationStateMachineTest.java | 43 ++++++- 3 files changed, 167 insertions(+), 16 deletions(-) create mode 100644 vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockVCMessageSession.java diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java index 286a66a4fd..30a056e40d 100644 --- a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java @@ -346,13 +346,13 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba StatusMessage msgForClient = new StatusMessage(newJobStatus, userName, progress, timepoint); // TODO: Implement messaging to client - // msgForClient.sendToClient(session); + msgForClient.sendToClient(session); if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); } else { simulationDatabase.updateSimulationJobStatus(newJobStatus); StatusMessage msgForClient = new StatusMessage(newJobStatus, userName, null, null); // TODO: Implement messaging to client - // msgForClient.sendToClient(session); + msgForClient.sendToClient(session); if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); } }else if (workerEvent.isProgressEvent() || workerEvent.isNewDataEvent()){ @@ -365,7 +365,7 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba simulationDatabase.updateSimulationJobStatus(oldSimulationJobStatus,runningStateInfo); StatusMessage msgForClient = new StatusMessage(oldSimulationJobStatus, userName, progress, timepoint); // TODO: Implement messaging to client - // msgForClient.sendToClient(session); + msgForClient.sendToClient(session); if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); }else{ VCMongoMessage.sendInfo("onWorkerEvent() ignoring WorkerEvent (currState="+oldSchedulerStatus.getDescription()+"): "+workerEvent.show()); @@ -376,18 +376,20 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba public synchronized StatusMessage onStartRequest(User user, VCSimulationIdentifier vcSimID, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, DataAccessException, SQLException { + StatusMessage statusMessage; if (!user.equals(vcSimID.getOwner())) { lg.error(user + " is not authorized to start simulation (key=" + simKey + ")"); SimulationJobStatus simulationJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, 0, null, SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to start this simulation!"), null, null); - StatusMessage message = new StatusMessage(simulationJobStatus, user.getName(), null, null); + statusMessage = new StatusMessage(simulationJobStatus, user.getName(), null, null); VCMongoMessage.sendInfo("onStartRequest("+vcSimID.getID()+") ignoring start simulation request - wrong user): simID="+vcSimID); - return message; + statusMessage.sendToClient(session); + return statusMessage; } - SimulationJobStatus newJobStatus = saveSimulationStartRequest(vcSimID, jobIndex, simulationDatabase); - - return new StatusMessage(newJobStatus, user.getName(), null, null); + statusMessage = new StatusMessage(newJobStatus, user.getName(), null, null); + statusMessage.sendToClient(session); + return statusMessage; } public static SimulationJobStatus saveSimulationStartRequest(VCSimulationIdentifier vcSimID, int jobIndex, SimulationDatabase simulationDatabase) throws DataAccessException, SQLException { @@ -510,13 +512,16 @@ public synchronized void onDispatch(Simulation simulation, SimulationJobStatus o public synchronized StatusMessage onStopRequest(User user, SimulationJobStatus simJobStatus, SimulationDatabase simulationDatabase, VCMessageSession session) throws VCMessagingException, DataAccessException, SQLException { updateSolverProcessTimestamp(); + StatusMessage statusMessage; if (!user.equals(simJobStatus.getVCSimulationIdentifier().getOwner())) { lg.error(user + " is not authorized to stop simulation (key=" + simKey + ")"); SimulationJobStatus simulationJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(), simJobStatus.getVCSimulationIdentifier(), 0, null, SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to stop this simulation!"), null, null); VCMongoMessage.sendInfo("onStopRequest("+simJobStatus.getVCSimulationIdentifier()+") ignoring stop simulation request - wrong user)"); - return new StatusMessage(simulationJobStatus, user.getName(), null, null); + statusMessage = new StatusMessage(simulationJobStatus, user.getName(), null, null); + statusMessage.sendToClient(session); + return statusMessage; } // stop latest task if active @@ -537,8 +542,10 @@ null, new VCSimulationIdentifier(simKey, user), jobIndex, simJobStatus.getSubmit ); simulationDatabase.updateSimulationJobStatus(newJobStatus); + statusMessage = new StatusMessage(simulationJobStatusRecord, user.getName(), null, null); + statusMessage.sendToClient(session); - return new StatusMessage(simulationJobStatusRecord, user.getName(), null, null); + return statusMessage; } return null; } diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockVCMessageSession.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockVCMessageSession.java new file mode 100644 index 0000000000..092022373e --- /dev/null +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockVCMessageSession.java @@ -0,0 +1,113 @@ +package cbit.vcell.message.server.dispatcher; + +import cbit.vcell.message.*; +import cbit.vcell.message.jms.VCMessageJms; +import org.apache.activemq.command.ActiveMQObjectMessage; +import org.apache.activemq.command.ActiveMQTextMessage; +import org.vcell.util.document.UserLoginInfo; + +import javax.jms.*; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Queue; + +public class MockVCMessageSession implements VCMessageSession { + + private Connection connection; + private Session session; + + public MockVCMessageSession(){ + try { +// connection = ActiveMQConnection.makeConnection(""); +// session = connection.createSession(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private final HashMap> topics = new HashMap<>(){{ + put(VCellTopic.ClientStatusTopic, new LinkedList<>()); + put(VCellTopic.ServiceControlTopic, new LinkedList<>()); + }}; + private final HashMap> queues = new HashMap<>(){{ + put(VCellQueue.WorkerEventQueue, new LinkedList<>()); + put(VCellQueue.DbRequestQueue, new LinkedList<>()); + put(VCellQueue.DataRequestQueue, new LinkedList<>()); + put(VCellQueue.SimReqQueue, new LinkedList<>()); + put(VCellQueue.SimJobQueue, new LinkedList<>()); + }}; + + @Override + public Object sendRpcMessage(VCellQueue queue, VCRpcRequest vcRpcRequest, boolean returnRequired, long timeoutMS, String[] specialProperties, Object[] specialValues, UserLoginInfo userLoginInfo) throws VCMessagingException, VCMessagingInvocationTargetException { + return null; + } + + @Override + public void sendQueueMessage(VCellQueue queue, VCMessage message, Boolean persistent, Long clientTimeoutMS) throws VCMessagingException { + queues.get(queue).add(message); + } + + @Override + public void sendTopicMessage(VCellTopic topic, VCMessage message) throws VCMessagingException { + topics.get(topic).add(message); + } + + @Override + public void rollback() { + + } + + @Override + public void commit() { + + } + + @Override + public VCMessage createTextMessage(String text) { + TextMessage textMessage = new ActiveMQTextMessage(); + try { + textMessage.setText(text); + } catch (JMSException e) { + throw new RuntimeException(e); + } + return new VCMessageJms(textMessage, null); + } + + @Override + public VCMessage createMessage() { + return null; + } + + @Override + public VCMessage createObjectMessage(Serializable object) { + ObjectMessage objectMessage = new ActiveMQObjectMessage(); + try { + objectMessage.setObjectProperty(VCMessageJms.BLOB_MESSAGE_FILE_NAME, ""); + } catch (JMSException e) { + throw new RuntimeException(e); + } + return new VCMessageJms(objectMessage, object, null); + + } + + @Override + public VCMessagingDelegate getDelegate() { + return null; + } + + @Override + public void close() { + + } + + public VCMessage getTopicMessage(VCellTopic vCellTopic){ + return topics.get(vCellTopic).remove(); + } + + public VCMessage getQueueMessage(VCellQueue vCellQueue){ + return queues.get(vCellQueue).remove(); + } + +} diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java index e48a74f558..1307c36115 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java @@ -5,9 +5,8 @@ import cbit.vcell.mapping.MathSymbolMapping; import cbit.vcell.math.*; import cbit.vcell.mathmodel.MathModel; -import cbit.vcell.message.VCMessageSession; import cbit.vcell.message.VCMessagingException; -import cbit.vcell.message.local.LocalVCMessageAdapter; +import cbit.vcell.message.VCellTopic; import cbit.vcell.message.messages.StatusMessage; import cbit.vcell.parser.ExpressionBindingException; import cbit.vcell.resource.PropertyLoader; @@ -28,13 +27,14 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Date; +import java.util.NoSuchElementException; @Tag("Fast") public class SimulationStateMachineTest { private static final VCellServerID testVCellServerID = VCellServerID.getServerID("test"); private static final User testUser = new User("Alice", new KeyValue("0")); - private static final VCMessageSession testMessageSession = new LocalVCMessageAdapter(null); + private static final MockVCMessageSession testMessageSession = new MockVCMessageSession(); private static final int jobIndex = 0; private static final int taskID = 0; private static final KeyValue simKey = new KeyValue("1"); @@ -44,6 +44,7 @@ public class SimulationStateMachineTest { public static String previousHtcMax = ""; public static String previousHtcMin = ""; public static String previousHtcPowerFloor = ""; + public static String previousMongoBlob = ""; private MockSimulationDB simulationDB; private SimulationStateMachine stateMachine; @@ -61,6 +62,9 @@ public static void setSystemProperties(){ previousHtcPowerFloor = PropertyLoader.getProperty(PropertyLoader.htcPowerUserMemoryFloorMB, ""); PropertyLoader.setProperty(PropertyLoader.htcPowerUserMemoryFloorMB, "51200"); + + previousMongoBlob = PropertyLoader.getProperty(PropertyLoader.jmsBlobMessageUseMongo, ""); + PropertyLoader.setProperty(PropertyLoader.jmsBlobMessageUseMongo, ""); } @AfterAll @@ -69,6 +73,7 @@ public static void restoreSystemProperties(){ PropertyLoader.setProperty(PropertyLoader.htcMaxMemoryMB, previousHtcMax); PropertyLoader.setProperty(PropertyLoader.htcMinMemoryMB, previousHtcMin); PropertyLoader.setProperty(PropertyLoader.htcPowerUserMemoryFloorMB, previousHtcPowerFloor); + PropertyLoader.setProperty(PropertyLoader.jmsBlobMessageUseMongo, previousMongoBlob); } @BeforeEach @@ -139,6 +144,10 @@ private SimulationJobStatus getLatestJobSubmission() throws SQLException, DataAc return simulationDB.getLatestSimulationJobStatus(simKey, jobIndex); } + private SimulationJobStatus getClientTopicMessage(){ + return (SimulationJobStatus) testMessageSession.getTopicMessage(VCellTopic.ClientStatusTopic).getObjectContent(); + } + @Test public void workerEventRejectionsTest() throws SQLException, DataAccessException { int taskID = 16; @@ -179,28 +188,33 @@ public void stateShouldTransitionToFailure() throws SQLException, DataAccessExce for (ChangedStateValues changedValue : changedValues){ insertOrUpdateStatus(); - stateMachine.onWorkerEvent(createWorkerEvent(changedValue), simulationDB, null); + stateMachine.onWorkerEvent(createWorkerEvent(changedValue), simulationDB, testMessageSession); SimulationJobStatus result = getLatestJobSubmission(); Assertions.assertTrue(result.getSchedulerStatus().isFailed(), changedValue.changesResult); + Assertions.assertTrue(getClientTopicMessage().getSchedulerStatus().isFailed(), changedValue.changesResult); } simulationDB = new MockSimulationDB(); - StatusMessage statusMessage = stateMachine.onStartRequest(new User("Bob", new KeyValue("1")), simID, simulationDB, null); + StatusMessage statusMessage = stateMachine.onStartRequest(new User("Bob", new KeyValue("1")), simID, simulationDB, testMessageSession); Assertions.assertTrue(statusMessage.getSimulationJobStatus().getSchedulerStatus().isFailed(), "Different from initial user that owns the simulation"); SimulationJobStatus jobStatus = getLatestJobSubmission(); Assertions.assertNull(jobStatus, "If it fails on start request, there should be nothing in the DB."); + Assertions.assertTrue(getClientTopicMessage().getSchedulerStatus().isFailed(), "Only the client receives start request failure status."); insertOrUpdateStatus(); Assertions.assertThrows(RuntimeException.class, () -> {stateMachine.onStartRequest(testUser, simID, simulationDB, testMessageSession);}, "Can't start simulation job unless previous is done."); + Assertions.assertThrows(NoSuchElementException.class,() -> getClientTopicMessage().getSchedulerStatus().isFailed(), "No message sent to client."); + insertOrUpdateStatus(); jobStatus = getLatestJobSubmission(); stateMachine.onSystemAbort(jobStatus, "Test Abort", simulationDB, testMessageSession); jobStatus = getLatestJobSubmission(); Assertions.assertTrue(jobStatus.getSchedulerStatus().isFailed()); + Assertions.assertTrue(getClientTopicMessage().getSchedulerStatus().isFailed(), "On abort client gets failed status."); // Simulation memoryIntensiveSimulation = createMockSimulation(900, 900, 900); @@ -209,16 +223,18 @@ public void stateShouldTransitionToFailure() throws SQLException, DataAccessExce Assertions.assertThrows(RuntimeException.class, () -> {stateMachine.onDispatch(memoryIntensiveSimulation, getLatestJobSubmission(), simulationDB, testMessageSession);}, "Can't dispatch simulation that is already running."); + Assertions.assertThrows(NoSuchElementException.class, () -> getClientTopicMessage().getSchedulerStatus().isFailed(), "Client receives failure because simulation is already running."); insertOrUpdateStatus(simKey, jobIndex, taskID, testUser, SimulationJobStatus.SchedulerStatus.WAITING); stateMachine.onDispatch(memoryIntensiveSimulation, getLatestJobSubmission(), simulationDB, testMessageSession); - jobStatus = getLatestJobSubmission(); Assertions.assertTrue(jobStatus.getSchedulerStatus().isFailed(), "Memory size to large"); + Assertions.assertTrue(getClientTopicMessage().getSchedulerStatus().isFailed(), "Failed because of memory size."); insertOrUpdateStatus(); statusMessage = stateMachine.onStopRequest(new User("Bob", new KeyValue("2")), getLatestJobSubmission(), simulationDB, testMessageSession); Assertions.assertTrue(statusMessage.getSimulationJobStatus().getSchedulerStatus().isFailed(), "Stopping as another user."); + Assertions.assertTrue(getClientTopicMessage().getSchedulerStatus().isFailed(), "Can't stop as another user."); } @Test @@ -226,6 +242,7 @@ public void stateShouldTransitionToWaiting() throws SQLException, VCMessagingExc stateMachine.onStartRequest(testUser, simID, simulationDB, testMessageSession); SimulationJobStatus jobStatus = getLatestJobSubmission(); Assertions.assertTrue(jobStatus.getSchedulerStatus().isWaiting(), "Just started new task."); + Assertions.assertTrue(getClientTopicMessage().getSchedulerStatus().isWaiting()); } @Test @@ -235,6 +252,7 @@ public void stateShouldTransitionToDispatched() throws SQLException, DataAccessE stateMachine.onWorkerEvent(acceptedWorker, simulationDB, testMessageSession); SimulationJobStatus jobStatus = getLatestJobSubmission(); Assertions.assertTrue(jobStatus.getSchedulerStatus().isDispatched(), "Job recently got accepted, only works if previous state was waiting."); + Assertions.assertTrue(getClientTopicMessage().getSchedulerStatus().isDispatched()); insertOrUpdateStatus(); stateMachine.onWorkerEvent(acceptedWorker, simulationDB, testMessageSession); @@ -247,6 +265,7 @@ public void stateShouldTransitionToDispatched() throws SQLException, DataAccessE stateMachine.onDispatch(simulation, getLatestJobSubmission(), simulationDB, testMessageSession); jobStatus = getLatestJobSubmission(); Assertions.assertTrue(jobStatus.getSchedulerStatus().isDispatched()); + Assertions.assertTrue(getClientTopicMessage().getSchedulerStatus().isDispatched()); } @Test @@ -258,8 +277,12 @@ public void stateShouldTransitionToRunning() throws SQLException, DataAccessExce SimulationJobStatus jobStatus = getLatestJobSubmission(); if (workerEvent.isProgressEvent() || workerEvent.isNewDataEvent() || workerEvent.isStartingEvent() || workerEvent.isWorkerAliveEvent()){ Assertions.assertTrue(jobStatus.getSchedulerStatus().isRunning()); + Assertions.assertTrue(getClientTopicMessage().getSchedulerStatus().isRunning()); } else { Assertions.assertFalse(jobStatus.getSchedulerStatus().isRunning()); + try { + Assertions.assertFalse(getClientTopicMessage().getSchedulerStatus().isRunning()); + } catch (NoSuchElementException ignored){} } } } @@ -273,8 +296,12 @@ public void stateShouldTransitionToCompleted() throws SQLException, VCMessagingE SimulationJobStatus jobStatus = getLatestJobSubmission(); if (workerEvent.isCompletedEvent()){ Assertions.assertTrue(jobStatus.getSchedulerStatus().isCompleted()); + Assertions.assertTrue(getClientTopicMessage().getSchedulerStatus().isCompleted()); } else { Assertions.assertFalse(jobStatus.getSchedulerStatus().isCompleted()); + try { + Assertions.assertFalse(getClientTopicMessage().getSchedulerStatus().isCompleted()); + } catch (NoSuchElementException ignored){} } } } @@ -287,9 +314,13 @@ public void stateShouldTransitionToStopped() throws SQLException, DataAccessExce if (status.isActive()){ stateMachine.onStopRequest(testUser, getLatestJobSubmission(), simulationDB, testMessageSession); Assertions.assertTrue(getLatestJobSubmission().getSchedulerStatus().isStopped(), ""); + Assertions.assertTrue(getClientTopicMessage().getSchedulerStatus().isStopped()); } else { StatusMessage statusMessage = stateMachine.onStopRequest(testUser, getLatestJobSubmission(), simulationDB, testMessageSession); Assertions.assertNull(statusMessage); + try { + Assertions.assertFalse(getClientTopicMessage().getSchedulerStatus().isCompleted()); + } catch (NoSuchElementException ignored){} } } } From 872d04044c43ec50e849fa32a2c3062beedeb66e Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Thu, 29 Aug 2024 08:30:28 -0400 Subject: [PATCH 11/24] Mock VCMessage Session for Now --- .../java/org/vcell/restq/Simulations/SimulationRestDB.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationRestDB.java b/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationRestDB.java index 30b54623ea..25dfe415ca 100644 --- a/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationRestDB.java +++ b/vcell-rest/src/main/java/org/vcell/restq/Simulations/SimulationRestDB.java @@ -1,6 +1,8 @@ package org.vcell.restq.Simulations; +import cbit.vcell.message.VCMessageSession; import cbit.vcell.message.VCMessagingException; +import cbit.vcell.message.local.LocalVCMessageAdapter; import cbit.vcell.message.server.dispatcher.SimulationDatabaseDirect; import cbit.vcell.modeldb.*; import cbit.vcell.solver.VCSimulationIdentifier; @@ -33,6 +35,7 @@ public class SimulationRestDB { private final SimulationDatabaseDirect simulationDatabaseDirect; private final SimulationDispatcherEngine simulationDispatcherEngine; private final BioModelRestDB bioModelRestDB; + private final VCMessageSession mockSession = new LocalVCMessageAdapter(null); // TODO: Replace with actual session public SimulationRestDB(AgroalConnectionFactory agroalConnectionFactory) throws DataAccessException, SQLException { databaseServerImpl = new DatabaseServerImpl(agroalConnectionFactory, agroalConnectionFactory.getKeyFactory()); @@ -97,7 +100,7 @@ public ArrayList startSimulation(String simId, User vcellUser) th SimulationRep simRep = getAndCheckSimRep(simId, vcellUser); try { ArrayList statusMessages = simulationDispatcherEngine.onStartRequest(vcSimulationIdentifier, vcellUser, simRep.getScanCount(), - simulationDatabaseDirect, null, null); + simulationDatabaseDirect, mockSession, null); return StatusMessage.convertServerStatusMessages(statusMessages); } catch (VCMessagingException e) { throw new RuntimeException(e); @@ -109,7 +112,7 @@ public ArrayList stopSimulation(String simId, User vcellUser) thr getAndCheckSimRep(simId, vcellUser); VCSimulationIdentifier vcSimulationIdentifier = new VCSimulationIdentifier(simKey, vcellUser); ArrayList statusMessages = simulationDispatcherEngine.onStopRequest( - vcSimulationIdentifier, vcellUser, simulationDatabaseDirect, null); + vcSimulationIdentifier, vcellUser, simulationDatabaseDirect, mockSession); return StatusMessage.convertServerStatusMessages(statusMessages); } From 182e83587e9b877acdd0e22f49b4034a3ffa2e9d Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Fri, 30 Aug 2024 15:10:59 -0400 Subject: [PATCH 12/24] Separate Simulation Dispatcher into Two Classes --- .../dispatcher/SimulationDispatcher.java | 94 +++++-------------- .../dispatcher/SimulationDispatcherMain.java | 79 ++++++++++++++++ 2 files changed, 102 insertions(+), 71 deletions(-) create mode 100644 vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherMain.java diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java index 09bbce31fb..786a7a99b6 100644 --- a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java @@ -29,21 +29,17 @@ import cbit.vcell.modeldb.AdminDBTopLevel; import cbit.vcell.modeldb.DatabaseServerImpl; import cbit.vcell.mongodb.VCMongoMessage; -import cbit.vcell.resource.OperatingSystemInfo; import cbit.vcell.resource.PropertyLoader; import cbit.vcell.server.*; import cbit.vcell.server.SimulationJobStatus.SchedulerStatus; import cbit.vcell.solver.Simulation; import cbit.vcell.solver.VCSimulationIdentifier; import com.google.gson.Gson; -import com.google.inject.Guice; -import com.google.inject.Injector; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vcell.db.ConnectionFactory; import org.vcell.db.DatabaseService; import org.vcell.db.KeyFactory; -import org.vcell.dependency.server.VCellServerModule; import org.vcell.util.DataAccessException; import org.vcell.util.PermissionException; import org.vcell.util.document.KeyValue; @@ -99,6 +95,7 @@ public class SimulationDispatcher { private final HtcProxy htcProxy; public static Logger lg = LogManager.getLogger(SimulationDispatcher.class); + public final SimulationService simServiceImpl; public class SimulationServiceImpl implements SimulationService { @@ -547,25 +544,37 @@ public Thread newThread(Runnable r) { } } - /** - * Scheduler constructor comment. - */ - public SimulationDispatcher() throws Exception { + public static SimulationDispatcher simulationDispatcherCreator(SimulationDatabase simulationDatabase, VCMessagingService messagingServiceInternal, + VCMessagingService messagingServiceSim){ + return new SimulationDispatcher(simulationDatabase, messagingServiceInternal, messagingServiceSim); + } + + public static SimulationDispatcher simulationDispatcherCreator() throws SQLException, DataAccessException { ConnectionFactory conFactory = DatabaseService.getInstance().createConnectionFactory(); KeyFactory keyFactory = conFactory.getKeyFactory(); DatabaseServerImpl databaseServerImpl = new DatabaseServerImpl(conFactory, keyFactory); AdminDBTopLevel adminDbTopLevel = new AdminDBTopLevel(conFactory); - this.simulationDatabase = new SimulationDatabaseDirect(adminDbTopLevel, databaseServerImpl, true); + SimulationDatabase simulationDatabase = new SimulationDatabaseDirect(adminDbTopLevel, databaseServerImpl, true); - this.vcMessagingService_int = new VCMessagingServiceActiveMQ(); + VCMessagingService vcMessagingServiceInternal = new VCMessagingServiceActiveMQ(); String jmshost_int = PropertyLoader.getRequiredProperty(PropertyLoader.jmsIntHostInternal); int jmsport_int = Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.jmsIntPortInternal)); - this.vcMessagingService_int.setConfiguration(new ServerMessagingDelegate(), jmshost_int, jmsport_int); + vcMessagingServiceInternal.setConfiguration(new ServerMessagingDelegate(), jmshost_int, jmsport_int); - this.vcMessagingService_sim = new VCMessagingServiceActiveMQ(); + VCMessagingService vcMessagingServiceSim = new VCMessagingServiceActiveMQ(); String jmshost_sim = PropertyLoader.getRequiredProperty(PropertyLoader.jmsSimHostInternal); int jmsport_sim = Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.jmsSimPortInternal)); - this.vcMessagingService_sim.setConfiguration(new ServerMessagingDelegate(), jmshost_sim, jmsport_sim); + vcMessagingServiceSim.setConfiguration(new ServerMessagingDelegate(), jmshost_sim, jmsport_sim); + + return SimulationDispatcher.simulationDispatcherCreator(simulationDatabase, vcMessagingServiceInternal, vcMessagingServiceSim); + } + + private SimulationDispatcher(SimulationDatabase simulationDatabase, VCMessagingService messagingServiceInternal, + VCMessagingService messagingServiceSim){ + this.simulationDatabase = simulationDatabase; + this.vcMessagingService_int = messagingServiceInternal; + this.vcMessagingService_sim = messagingServiceSim; + QueueListener workerEventListener = new QueueListener() { public void onQueueMessage(VCMessage vcMessage, VCMessageSession session) throws RollbackException { onWorkerEventMessage(vcMessage, session); @@ -579,7 +588,7 @@ public void onQueueMessage(VCMessage vcMessage, VCMessageSession session) throws // // set up consumer for Simulation Request (non-blocking RPC) messages // - SimulationService simServiceImpl = new SimulationServiceImpl(); + simServiceImpl = new SimulationServiceImpl(); VCMessageSelector simRequestSelector = null; threadName = "Sim Request Consumer"; @@ -600,11 +609,6 @@ public void onQueueMessage(VCMessage vcMessage, VCMessageSession session) throws } - public void init() { - - } - - /** * @param vcMessage * @param session @@ -657,57 +661,5 @@ private void traceThread(Object source) { " commencing run cycle at " + new SimpleDateFormat("k:m:s").format(new Date( )) ); } } - - /** - * Starts the application. - * @param args an array of command-line arguments - */ - public static void main(java.lang.String[] args) { - - if (args.length != 0) { - System.out.println("No arguments expected: " + SimulationDispatcher.class.getName()); - System.exit(1); - } - - try { - OperatingSystemInfo.getInstance(); - PropertyLoader.loadProperties(REQUIRED_SERVICE_PROPERTIES); - - Injector injector = Guice.createInjector(new VCellServerModule()); - - SimulationDispatcher simulationDispatcher = injector.getInstance(SimulationDispatcher.class); - simulationDispatcher.init(); - - } catch (Throwable e) { - lg.error("uncaught exception initializing SimulationDispatcher: "+e.getLocalizedMessage(), e); - System.exit(1); - } - } - - - private static final String REQUIRED_SERVICE_PROPERTIES[] = { - PropertyLoader.vcellServerIDProperty, - PropertyLoader.installationRoot, - PropertyLoader.dbConnectURL, - PropertyLoader.dbDriverName, - PropertyLoader.dbUserid, - PropertyLoader.dbPasswordFile, - PropertyLoader.userTimezone, - PropertyLoader.mongodbHostInternal, - PropertyLoader.mongodbPortInternal, - PropertyLoader.mongodbDatabase, - PropertyLoader.jmsIntHostInternal, - PropertyLoader.jmsIntPortInternal, - PropertyLoader.jmsSimHostInternal, - PropertyLoader.jmsSimPortInternal, - PropertyLoader.jmsUser, - PropertyLoader.jmsPasswordFile, - PropertyLoader.htcUser, - PropertyLoader.jmsBlobMessageUseMongo, - PropertyLoader.maxJobsPerScan, - PropertyLoader.maxOdeJobsPerUser, - PropertyLoader.maxPdeJobsPerUser, - PropertyLoader.slurm_partition - }; } diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherMain.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherMain.java new file mode 100644 index 0000000000..a23560f10b --- /dev/null +++ b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherMain.java @@ -0,0 +1,79 @@ +/* + * Copyright (C) 1999-2011 University of Connecticut Health Center + * + * Licensed under the MIT License (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.opensource.org/licenses/mit-license.php + */ + +package cbit.vcell.message.server.dispatcher; + +import cbit.vcell.resource.OperatingSystemInfo; +import cbit.vcell.resource.PropertyLoader; +import com.google.inject.Guice; +import com.google.inject.Injector; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vcell.dependency.server.VCellServerModule; + +/** + * Insert the type's description here. + * Creation date: (10/18/2001 4:31:11 PM) + * @author: Jim Schaff + */ +public class SimulationDispatcherMain { + public static Logger lg = LogManager.getLogger(SimulationDispatcher.class); + /** + * Starts the application. + * @param args an array of command-line arguments + */ + public static void main(String[] args) { + + if (args.length != 0) { + System.out.println("No arguments expected: " + SimulationDispatcherMain.class.getName()); + System.exit(1); + } + + try { + OperatingSystemInfo.getInstance(); + PropertyLoader.loadProperties(REQUIRED_SERVICE_PROPERTIES); + + Injector injector = Guice.createInjector(new VCellServerModule()); + + SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(); + injector.injectMembers(simulationDispatcher); + } catch (Throwable e) { + lg.error("uncaught exception initializing SimulationDispatcher: "+e.getLocalizedMessage(), e); + System.exit(1); + } + } + + + private static final String REQUIRED_SERVICE_PROPERTIES[] = { + PropertyLoader.vcellServerIDProperty, + PropertyLoader.installationRoot, + PropertyLoader.dbConnectURL, + PropertyLoader.dbDriverName, + PropertyLoader.dbUserid, + PropertyLoader.dbPasswordFile, + PropertyLoader.userTimezone, + PropertyLoader.mongodbHostInternal, + PropertyLoader.mongodbPortInternal, + PropertyLoader.mongodbDatabase, + PropertyLoader.jmsIntHostInternal, + PropertyLoader.jmsIntPortInternal, + PropertyLoader.jmsSimHostInternal, + PropertyLoader.jmsSimPortInternal, + PropertyLoader.jmsUser, + PropertyLoader.jmsPasswordFile, + PropertyLoader.htcUser, + PropertyLoader.jmsBlobMessageUseMongo, + PropertyLoader.maxJobsPerScan, + PropertyLoader.maxOdeJobsPerUser, + PropertyLoader.maxPdeJobsPerUser, + PropertyLoader.slurm_partition + }; + +} From 41a41940fc7636df81909fc3990bcd25297656d9 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Fri, 30 Aug 2024 15:12:01 -0400 Subject: [PATCH 13/24] Initial Simulation Dispatcher Test --- .../dispatcher/DispatcherTestUtils.java | 148 ++++++++++++++++++ .../dispatcher/MockMessagingService.java | 51 ++++++ .../server/dispatcher/MockSimulationDB.java | 52 +++++- .../dispatcher/SimulationDispatcherTest.java | 51 ++++++ .../SimulationStateMachineTest.java | 135 ++++------------ 5 files changed, 325 insertions(+), 112 deletions(-) create mode 100644 vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/DispatcherTestUtils.java create mode 100644 vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockMessagingService.java create mode 100644 vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/DispatcherTestUtils.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/DispatcherTestUtils.java new file mode 100644 index 0000000000..a3b14262d5 --- /dev/null +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/DispatcherTestUtils.java @@ -0,0 +1,148 @@ +package cbit.vcell.message.server.dispatcher; + +import cbit.vcell.geometry.Geometry; +import cbit.vcell.mapping.MathSymbolMapping; +import cbit.vcell.math.*; +import cbit.vcell.mathmodel.MathModel; +import cbit.vcell.parser.ExpressionBindingException; +import cbit.vcell.resource.PropertyLoader; +import cbit.vcell.server.HtcJobID; +import cbit.vcell.server.SimulationExecutionStatus; +import cbit.vcell.server.SimulationJobStatus; +import cbit.vcell.solver.MeshSpecification; +import cbit.vcell.solver.Simulation; +import cbit.vcell.solver.VCSimulationIdentifier; +import cbit.vcell.solver.server.SimulationMessage; +import org.vcell.util.DataAccessException; +import org.vcell.util.ISize; +import org.vcell.util.document.*; + +import java.beans.PropertyVetoException; +import java.sql.SQLException; +import java.time.Instant; +import java.util.Date; + +public class DispatcherTestUtils { + private static String previousServerID = ""; + private static String previousHtcMax = ""; + private static String previousHtcMin = ""; + private static String previousHtcPowerFloor = ""; + private static String previousMongoBlob = ""; + private static String previousJMSIntHostProperty = ""; + private static String previousJMSIntPortProperty = ""; + private static String previousSimJMSIntHostProperty = ""; + private static String previousSimJMSIntPortProperty = ""; + private static String previousHTCHost = ""; + private static String previousHTCUser = ""; + private static String previousHTCUserKeyFile = ""; + private static String previousMaxJobsPerScan = ""; + + public static final VCellServerID testVCellServerID = VCellServerID.getServerID("test"); + public static final MockVCMessageSession testMessageSession = new MockVCMessageSession(); + public static final int jobIndex = 0; + public static final int taskID = 0; + public static final KeyValue simKey = new KeyValue("0"); + public static User alice = new User("Alice", new KeyValue("0")); + public static User bob = new User("Bob", new KeyValue("1")); + public static final VCSimulationIdentifier simID = new VCSimulationIdentifier(simKey, alice); + + public static void setRequiredProperties(){ + previousServerID = PropertyLoader.getProperty(PropertyLoader.vcellServerIDProperty, ""); + PropertyLoader.setProperty(PropertyLoader.vcellServerIDProperty, ""); + + previousHtcMax = PropertyLoader.getProperty(PropertyLoader.htcMaxMemoryMB, ""); + PropertyLoader.setProperty(PropertyLoader.htcMaxMemoryMB, "4096"); + + previousHtcMin = PropertyLoader.getProperty(PropertyLoader.htcMinMemoryMB, ""); + PropertyLoader.setProperty(PropertyLoader.htcMinMemoryMB, "1024"); + + previousHtcPowerFloor = PropertyLoader.getProperty(PropertyLoader.htcPowerUserMemoryFloorMB, ""); + PropertyLoader.setProperty(PropertyLoader.htcPowerUserMemoryFloorMB, "51200"); + + previousMongoBlob = PropertyLoader.getProperty(PropertyLoader.jmsBlobMessageUseMongo, ""); + PropertyLoader.setProperty(PropertyLoader.jmsBlobMessageUseMongo, ""); + + previousJMSIntHostProperty = PropertyLoader.getProperty(PropertyLoader.jmsIntHostInternal, ""); + PropertyLoader.setProperty(PropertyLoader.jmsIntHostInternal, "host"); + + previousJMSIntPortProperty = PropertyLoader.getProperty(PropertyLoader.jmsIntPortInternal, ""); + PropertyLoader.setProperty(PropertyLoader.jmsIntPortInternal, "80"); + + previousSimJMSIntHostProperty = PropertyLoader.getProperty(PropertyLoader.jmsSimHostInternal, ""); + PropertyLoader.setProperty(PropertyLoader.jmsSimHostInternal, "host"); + + previousSimJMSIntPortProperty = PropertyLoader.getProperty(PropertyLoader.jmsSimPortInternal, ""); + PropertyLoader.setProperty(PropertyLoader.jmsSimPortInternal, "80"); + + previousHTCHost = PropertyLoader.getProperty(PropertyLoader.htcHosts, ""); + PropertyLoader.setProperty(PropertyLoader.htcHosts, "host"); + + previousHTCUser = PropertyLoader.getProperty(PropertyLoader.htcUser, ""); + PropertyLoader.setProperty(PropertyLoader.htcUser, "user"); + + previousHTCUserKeyFile = PropertyLoader.getProperty(PropertyLoader.htcUserKeyFile, ""); + PropertyLoader.setProperty(PropertyLoader.htcUserKeyFile, "keyFile"); + + previousMaxJobsPerScan = PropertyLoader.getProperty(PropertyLoader.maxJobsPerScan, ""); + PropertyLoader.setProperty(PropertyLoader.maxJobsPerScan, "100"); + } + + public static void restoreRequiredProperties(){ + PropertyLoader.setProperty(PropertyLoader.vcellServerIDProperty, previousServerID); + PropertyLoader.setProperty(PropertyLoader.htcMaxMemoryMB, previousHtcMax); + PropertyLoader.setProperty(PropertyLoader.htcMinMemoryMB, previousHtcMin); + PropertyLoader.setProperty(PropertyLoader.htcPowerUserMemoryFloorMB, previousHtcPowerFloor); + PropertyLoader.setProperty(PropertyLoader.jmsBlobMessageUseMongo, previousMongoBlob); + PropertyLoader.setProperty(PropertyLoader.jmsIntPortInternal, previousJMSIntPortProperty); + PropertyLoader.setProperty(PropertyLoader.jmsIntHostInternal, previousJMSIntHostProperty); + PropertyLoader.setProperty(PropertyLoader.jmsSimPortInternal, previousSimJMSIntPortProperty); + PropertyLoader.setProperty(PropertyLoader.jmsSimHostInternal, previousSimJMSIntHostProperty); + PropertyLoader.setProperty(PropertyLoader.htcHosts, previousHTCHost); + PropertyLoader.setProperty(PropertyLoader.htcUser, previousHTCUser); + PropertyLoader.setProperty(PropertyLoader.htcUserKeyFile, previousHTCUserKeyFile); + PropertyLoader.setProperty(PropertyLoader.maxJobsPerScan, previousMaxJobsPerScan); + } + + public static Simulation createMockSimulation(int iSizeX, int iSizeY, int iSizeZ) throws PropertyVetoException, MathException, ExpressionBindingException { + VolVariable volVariable = new VolVariable("t", new Variable.Domain(new CompartmentSubDomain("t", 1))); + VolVariable volVariable2 = new VolVariable("b", new Variable.Domain(new CompartmentSubDomain("b", 2))); + MathSymbolMapping mathSymbolMapping = new MathSymbolMapping(); + Geometry geometry = new Geometry("T", 3); + MathModel mathModel = new MathModel(new Version("Test", alice)); + MathDescription mathDescription = new MathDescription("Test", mathSymbolMapping); + mathDescription.setGeometry(new Geometry("T", 3)); + Simulation simulation = new Simulation(SimulationVersion.createTempSimulationVersion(), + mathDescription, mathModel); + MeshSpecification meshSpecification = new MeshSpecification(geometry); + meshSpecification.setSamplingSize(new ISize(iSizeX, iSizeY, iSizeZ)); + simulation.setMeshSpecification(meshSpecification); + mathDescription.setAllVariables(new Variable[]{volVariable, volVariable2}); + return simulation; + } + + public static void insertOrUpdateStatus(KeyValue simKey, int jobIndex, int taskID, User user, SimulationJobStatus.SchedulerStatus status, SimulationDatabase simulationDB) throws SQLException, DataAccessException { + SimulationJobStatus jobStatus = simulationDB.getLatestSimulationJobStatus(simKey, jobIndex); + VCSimulationIdentifier simID = new VCSimulationIdentifier(simKey, user); + SimulationJobStatus simulationJobStatus = new SimulationJobStatus(testVCellServerID, simID, jobIndex, Date.from(Instant.now()), status, taskID, + SimulationMessage.workerAccepted("accepted"), null, + new SimulationExecutionStatus(Date.from(Instant.now()), "", + Date.from(Instant.now()), Date.from(Instant.now()), false, new HtcJobID("2", HtcJobID.BatchSystemType.SLURM))); + if (jobStatus == null){ + simulationDB.insertSimulationJobStatus(simulationJobStatus); + } else { + simulationDB.updateSimulationJobStatus(simulationJobStatus); + } + } + + public static void insertOrUpdateStatus(KeyValue simKey, int jobIndex, int taskID, User user, SimulationDatabase simulationDB) throws SQLException, DataAccessException { + insertOrUpdateStatus(simKey, jobIndex, taskID, user, SimulationJobStatus.SchedulerStatus.RUNNING, simulationDB); + } + + /** + Defaults to a running status. + */ + public static void insertOrUpdateStatus(SimulationDatabase simulationDatabase) throws SQLException, DataAccessException { + insertOrUpdateStatus(simKey, jobIndex, taskID, alice, simulationDatabase); + } + +} diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockMessagingService.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockMessagingService.java new file mode 100644 index 0000000000..82b0283713 --- /dev/null +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockMessagingService.java @@ -0,0 +1,51 @@ +package cbit.vcell.message.server.dispatcher; + +import cbit.vcell.message.*; + +import java.util.ArrayList; +import java.util.List; + +public class MockMessagingService implements VCMessagingService { + + public ArrayList messagingConsumers = new ArrayList<>(); + + @Override + public VCMessageSession createProducerSession() { + return new MockVCMessageSession(); + } + + @Override + public void addMessageConsumer(VCMessagingConsumer vcMessagingConsumer) { + messagingConsumers.add(vcMessagingConsumer); + } + + @Override + public void removeMessageConsumer(VCMessagingConsumer vcMessagingConsumer) { + + } + + @Override + public List getMessageConsumers() { + return List.of(); + } + + @Override + public void close() throws VCMessagingException { + + } + + @Override + public VCMessageSelector createSelector(String clientMessageFilter) { + return null; + } + + @Override + public VCMessagingDelegate getDelegate() { + return null; + } + + @Override + public void setConfiguration(VCMessagingDelegate delegate, String jmshost, int jmsport) { + + } +} diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java index 121e58a3e9..af407a0492 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java @@ -7,16 +7,22 @@ import cbit.vcell.solver.SimulationInfo; import org.vcell.util.DataAccessException; import org.vcell.util.ObjectNotFoundException; -import org.vcell.util.document.KeyValue; -import org.vcell.util.document.User; -import org.vcell.util.document.VCellServerID; +import org.vcell.util.document.*; +import java.math.BigDecimal; import java.sql.SQLException; +import java.time.Instant; import java.util.*; public class MockSimulationDB implements SimulationDatabase{ - private final HashMap> dbTable = new HashMap<>(); + private HashMap> dbTable = new HashMap<>(); + + public static User.SpecialUser specialAdmin = new User.SpecialUser("Tom", new KeyValue("999"), new User.SPECIAL_CLAIM[User.SPECIAL_CLAIM.admins.ordinal()]); + + private final HashMap users = new HashMap<>(){ + {put(specialAdmin.getName(), specialAdmin); put(DispatcherTestUtils.alice.getName(), DispatcherTestUtils.alice);} + }; @Override public SimulationJobStatus getLatestSimulationJobStatus(KeyValue simKey, int jobIndex) throws DataAccessException, SQLException { @@ -47,7 +53,15 @@ public void insertSimulationJobStatus(SimulationJobStatus simulationJobStatus) t @Override public SimulationJobStatus[] getActiveJobs(VCellServerID vcellServerID) throws DataAccessException, SQLException { - throw new SQLException(); + ArrayList allActiveJobs = new ArrayList<>(); + for (ArrayList jobStatuses : dbTable.values()){ + for (SimulationJobStatus jobStatus: jobStatuses){ + if (jobStatus.getSchedulerStatus().isActive()){ + allActiveJobs.add(jobStatus); + } + } + } + return allActiveJobs.toArray(new SimulationJobStatus[]{}); } @Override @@ -101,7 +115,12 @@ public Set getUnreferencedSimulations() throws SQLException { @Override public User.SpecialUser getUser(String username) throws DataAccessException, SQLException { - return null; + User user = users.get(username); + if (user instanceof User.SpecialUser){ + return (User.SpecialUser) user; + } + User.SpecialUser specialUser = new User.SpecialUser(user.getName(), user.getID(), new User.SPECIAL_CLAIM[]{}); + return specialUser; } @Override @@ -111,7 +130,7 @@ public TreeMap> getSpecialUsers() thro @Override public SimulationInfo getSimulationInfo(User user, KeyValue simKey) throws ObjectNotFoundException, DataAccessException { - return null; + return mockSimulationInfo(user, simKey); } @Override @@ -121,6 +140,7 @@ public SimulationStatus[] getSimulationStatus(KeyValue[] simKeys) throws ObjectN @Override public SimulationStatus getSimulationStatus(KeyValue simulationKey) throws ObjectNotFoundException, DataAccessException { +// dbTable.get(simulationKey.toString()).get(0); return null; } @@ -128,4 +148,22 @@ public SimulationStatus getSimulationStatus(KeyValue simulationKey) throws Objec public SimpleJobStatus[] getSimpleJobStatus(User user, SimpleJobStatusQuerySpec simStatusQuerySpec) throws ObjectNotFoundException, DataAccessException { return new SimpleJobStatus[0]; } + + + private SimulationInfo mockSimulationInfo(User user, KeyValue simKey){ + KeyValue versionKey = new KeyValue("22"); + KeyValue versionBranchPoint = new KeyValue("23"); + VersionFlag versionFlag = VersionFlag.fromInt(0); + KeyValue parentSimulationRef = new KeyValue("24"); + SimulationVersion simulationVersion = new SimulationVersion(versionKey, "Mock Sim Info", user, null, + versionBranchPoint, new BigDecimal(22), Date.from(Instant.now()), versionFlag, "Version annot", + parentSimulationRef); + SimulationInfo simulationInfo = new SimulationInfo(simKey, simulationVersion, VCellSoftwareVersion.fromString("50")); + return simulationInfo; + } + + public void resetDataBase(){ + dbTable = new HashMap<>(); + } + } diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java new file mode 100644 index 0000000000..ed78c82efe --- /dev/null +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java @@ -0,0 +1,51 @@ +package cbit.vcell.message.server.dispatcher; + +import cbit.vcell.server.SimulationJobStatus; +import cbit.vcell.server.SimulationStatus; +import org.junit.jupiter.api.*; +import org.vcell.util.DataAccessException; +import org.vcell.util.document.User; + +import java.sql.SQLException; + +@Tag("Fast") +public class SimulationDispatcherTest { + private final static User testUser = DispatcherTestUtils.alice; + private MockSimulationDB mockSimulationDB = new MockSimulationDB(); + private MockMessagingService mockMessagingServiceInternal = new MockMessagingService(); + private MockMessagingService mockMessagingServiceSim = new MockMessagingService(); + + @BeforeAll + public static void setSystemProperties(){ + DispatcherTestUtils.setRequiredProperties(); + } + + @AfterAll + public static void restoreSystemProperties(){ + DispatcherTestUtils.restoreRequiredProperties(); + } + + @BeforeEach + public void beforeEach(){ + mockSimulationDB = new MockSimulationDB(); + } + + + @Test + public void onStartRequestTest() throws DataAccessException, SQLException { + SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(mockSimulationDB, mockMessagingServiceInternal, mockMessagingServiceSim); + SimulationStatus simStatus = simulationDispatcher.simServiceImpl.startSimulation(testUser, DispatcherTestUtils.simID, 1); + SimulationJobStatus jobStatus = mockSimulationDB.getLatestSimulationJobStatus(DispatcherTestUtils.simKey, 0); + Assertions.assertTrue(jobStatus.getSchedulerStatus().isWaiting()); + } + + @Test + public void onStopRequestTest() throws DataAccessException, SQLException { + DispatcherTestUtils.insertOrUpdateStatus(mockSimulationDB); + SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(mockSimulationDB, mockMessagingServiceInternal, mockMessagingServiceSim); + SimulationStatus simStatus = simulationDispatcher.simServiceImpl.stopSimulation(testUser, DispatcherTestUtils.simID); + SimulationJobStatus jobStatus = mockSimulationDB.getLatestSimulationJobStatus(DispatcherTestUtils.simKey, 0); + Assertions.assertTrue(jobStatus.getSchedulerStatus().isStopped()); + } + +} diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java index 1307c36115..c181b0762a 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationStateMachineTest.java @@ -1,79 +1,46 @@ package cbit.vcell.message.server.dispatcher; import cbit.rmi.event.WorkerEvent; -import cbit.vcell.geometry.Geometry; -import cbit.vcell.mapping.MathSymbolMapping; -import cbit.vcell.math.*; -import cbit.vcell.mathmodel.MathModel; +import cbit.vcell.math.MathException; import cbit.vcell.message.VCMessagingException; import cbit.vcell.message.VCellTopic; import cbit.vcell.message.messages.StatusMessage; import cbit.vcell.parser.ExpressionBindingException; -import cbit.vcell.resource.PropertyLoader; -import cbit.vcell.server.HtcJobID; -import cbit.vcell.server.SimulationExecutionStatus; import cbit.vcell.server.SimulationJobStatus; -import cbit.vcell.solver.MeshSpecification; import cbit.vcell.solver.Simulation; import cbit.vcell.solver.VCSimulationIdentifier; import cbit.vcell.solver.server.SimulationMessage; import org.junit.jupiter.api.*; import org.vcell.util.DataAccessException; -import org.vcell.util.ISize; -import org.vcell.util.document.*; +import org.vcell.util.document.KeyValue; +import org.vcell.util.document.User; +import org.vcell.util.document.VCellServerID; import java.beans.PropertyVetoException; import java.sql.SQLException; -import java.time.Instant; import java.util.ArrayList; -import java.util.Date; import java.util.NoSuchElementException; @Tag("Fast") public class SimulationStateMachineTest { - - private static final VCellServerID testVCellServerID = VCellServerID.getServerID("test"); - private static final User testUser = new User("Alice", new KeyValue("0")); + private static final User testUser = DispatcherTestUtils.alice; private static final MockVCMessageSession testMessageSession = new MockVCMessageSession(); - private static final int jobIndex = 0; - private static final int taskID = 0; - private static final KeyValue simKey = new KeyValue("1"); - private static final VCSimulationIdentifier simID = new VCSimulationIdentifier(simKey, testUser); - - public static String previousServerID = ""; - public static String previousHtcMax = ""; - public static String previousHtcMin = ""; - public static String previousHtcPowerFloor = ""; - public static String previousMongoBlob = ""; + private static final int jobIndex = DispatcherTestUtils.jobIndex; + private static final int taskID = DispatcherTestUtils.taskID; + private static final KeyValue simKey = DispatcherTestUtils.simKey; + private static final VCSimulationIdentifier simID = DispatcherTestUtils.simID; private MockSimulationDB simulationDB; private SimulationStateMachine stateMachine; @BeforeAll public static void setSystemProperties(){ - previousServerID = PropertyLoader.getProperty(PropertyLoader.vcellServerIDProperty, ""); - PropertyLoader.setProperty(PropertyLoader.vcellServerIDProperty, ""); - - previousHtcMax = PropertyLoader.getProperty(PropertyLoader.htcMaxMemoryMB, ""); - PropertyLoader.setProperty(PropertyLoader.htcMaxMemoryMB, "4096"); - - previousHtcMin = PropertyLoader.getProperty(PropertyLoader.htcMinMemoryMB, ""); - PropertyLoader.setProperty(PropertyLoader.htcMinMemoryMB, "1024"); - - previousHtcPowerFloor = PropertyLoader.getProperty(PropertyLoader.htcPowerUserMemoryFloorMB, ""); - PropertyLoader.setProperty(PropertyLoader.htcPowerUserMemoryFloorMB, "51200"); - - previousMongoBlob = PropertyLoader.getProperty(PropertyLoader.jmsBlobMessageUseMongo, ""); - PropertyLoader.setProperty(PropertyLoader.jmsBlobMessageUseMongo, ""); + DispatcherTestUtils.setRequiredProperties(); } @AfterAll public static void restoreSystemProperties(){ - PropertyLoader.setProperty(PropertyLoader.vcellServerIDProperty, previousServerID); - PropertyLoader.setProperty(PropertyLoader.htcMaxMemoryMB, previousHtcMax); - PropertyLoader.setProperty(PropertyLoader.htcMinMemoryMB, previousHtcMin); - PropertyLoader.setProperty(PropertyLoader.htcPowerUserMemoryFloorMB, previousHtcPowerFloor); - PropertyLoader.setProperty(PropertyLoader.jmsBlobMessageUseMongo, previousMongoBlob); + DispatcherTestUtils.restoreRequiredProperties(); } @BeforeEach @@ -98,48 +65,6 @@ private WorkerEvent createWorkerEvent(ChangedStateValues w){ acceptedSimulationMessage); } - private Simulation createMockSimulation(int iSizeX, int iSizeY, int iSizeZ) throws PropertyVetoException, MathException, ExpressionBindingException { - VolVariable volVariable = new VolVariable("t", new Variable.Domain(new CompartmentSubDomain("t", 1))); - VolVariable volVariable2 = new VolVariable("b", new Variable.Domain(new CompartmentSubDomain("b", 2))); - MathSymbolMapping mathSymbolMapping = new MathSymbolMapping(); - Geometry geometry = new Geometry("T", 3); - MathModel mathModel = new MathModel(new Version("Test", testUser)); - MathDescription mathDescription = new MathDescription("Test", mathSymbolMapping); - mathDescription.setGeometry(new Geometry("T", 3)); - Simulation simulation = new Simulation(SimulationVersion.createTempSimulationVersion(), - mathDescription, mathModel); - MeshSpecification meshSpecification = new MeshSpecification(geometry); - meshSpecification.setSamplingSize(new ISize(iSizeX, iSizeY, iSizeZ)); - simulation.setMeshSpecification(meshSpecification); - mathDescription.setAllVariables(new Variable[]{volVariable, volVariable2}); - return simulation; - } - - private void insertOrUpdateStatus(KeyValue simKey, int jobIndex, int taskID, User user, SimulationJobStatus.SchedulerStatus status) throws SQLException, DataAccessException { - SimulationJobStatus jobStatus = simulationDB.getLatestSimulationJobStatus(simKey, jobIndex); - VCSimulationIdentifier simID = new VCSimulationIdentifier(simKey, user); - SimulationJobStatus simulationJobStatus = new SimulationJobStatus(testVCellServerID, simID, jobIndex, Date.from(Instant.now()), status, taskID, - SimulationMessage.workerAccepted("accepted"), null, - new SimulationExecutionStatus(Date.from(Instant.now()), "", - Date.from(Instant.now()), Date.from(Instant.now()), false, new HtcJobID("2", HtcJobID.BatchSystemType.SLURM))); - if (jobStatus == null){ - simulationDB.insertSimulationJobStatus(simulationJobStatus); - } else { - simulationDB.updateSimulationJobStatus(simulationJobStatus); - } - } - - private void insertOrUpdateStatus(KeyValue simKey, int jobIndex, int taskID, User user) throws SQLException, DataAccessException { - insertOrUpdateStatus(simKey, jobIndex, taskID, user, SimulationJobStatus.SchedulerStatus.RUNNING); - } - - /** - Defaults to a running status. - */ - private void insertOrUpdateStatus() throws SQLException, DataAccessException { - insertOrUpdateStatus(simKey, jobIndex, taskID, testUser); - } - private SimulationJobStatus getLatestJobSubmission() throws SQLException, DataAccessException { return simulationDB.getLatestSimulationJobStatus(simKey, jobIndex); } @@ -161,7 +86,7 @@ public void workerEventRejectionsTest() throws SQLException, DataAccessException for (int i = 0; i < changedValues.size(); i++){ ChangedStateValues workerEventChangedValues = changedValues.get(i); if (i > 1) { - insertOrUpdateStatus(simKey, jobIndex, taskID, testUser, workerEventChangedValues.schedulerStatus); + DispatcherTestUtils.insertOrUpdateStatus(simKey, jobIndex, taskID, testUser, workerEventChangedValues.schedulerStatus, simulationDB); } WorkerEvent workerEvent = createWorkerEvent(workerEventChangedValues); Assertions.assertFalse(stateMachine.isWorkerEventOkay(workerEvent, simulationDB), workerEventChangedValues.changesResult); @@ -172,7 +97,7 @@ public void workerEventRejectionsTest() throws SQLException, DataAccessException for (SimulationJobStatus.SchedulerStatus passingStatus: SimulationJobStatus.SchedulerStatus.values()){ if (!passingStatus.isDone()){ - insertOrUpdateStatus(simKey, jobIndex, taskID, testUser, passingStatus); + DispatcherTestUtils.insertOrUpdateStatus(simKey, jobIndex, taskID, testUser, passingStatus, simulationDB); Assertions.assertTrue(stateMachine.isWorkerEventOkay(passingWorkerEvent, simulationDB)); } } @@ -187,7 +112,7 @@ public void stateShouldTransitionToFailure() throws SQLException, DataAccessExce }}; for (ChangedStateValues changedValue : changedValues){ - insertOrUpdateStatus(); + DispatcherTestUtils.insertOrUpdateStatus(simulationDB); stateMachine.onWorkerEvent(createWorkerEvent(changedValue), simulationDB, testMessageSession); SimulationJobStatus result = getLatestJobSubmission(); Assertions.assertTrue(result.getSchedulerStatus().isFailed(), changedValue.changesResult); @@ -195,21 +120,21 @@ public void stateShouldTransitionToFailure() throws SQLException, DataAccessExce } simulationDB = new MockSimulationDB(); - StatusMessage statusMessage = stateMachine.onStartRequest(new User("Bob", new KeyValue("1")), simID, simulationDB, testMessageSession); + StatusMessage statusMessage = stateMachine.onStartRequest(DispatcherTestUtils.bob, simID, simulationDB, testMessageSession); Assertions.assertTrue(statusMessage.getSimulationJobStatus().getSchedulerStatus().isFailed(), "Different from initial user that owns the simulation"); SimulationJobStatus jobStatus = getLatestJobSubmission(); Assertions.assertNull(jobStatus, "If it fails on start request, there should be nothing in the DB."); Assertions.assertTrue(getClientTopicMessage().getSchedulerStatus().isFailed(), "Only the client receives start request failure status."); - insertOrUpdateStatus(); + DispatcherTestUtils.insertOrUpdateStatus(simulationDB); Assertions.assertThrows(RuntimeException.class, () -> {stateMachine.onStartRequest(testUser, simID, simulationDB, testMessageSession);}, "Can't start simulation job unless previous is done."); Assertions.assertThrows(NoSuchElementException.class,() -> getClientTopicMessage().getSchedulerStatus().isFailed(), "No message sent to client."); - insertOrUpdateStatus(); + DispatcherTestUtils.insertOrUpdateStatus(simulationDB); jobStatus = getLatestJobSubmission(); stateMachine.onSystemAbort(jobStatus, "Test Abort", simulationDB, testMessageSession); jobStatus = getLatestJobSubmission(); @@ -217,22 +142,22 @@ public void stateShouldTransitionToFailure() throws SQLException, DataAccessExce Assertions.assertTrue(getClientTopicMessage().getSchedulerStatus().isFailed(), "On abort client gets failed status."); // - Simulation memoryIntensiveSimulation = createMockSimulation(900, 900, 900); + Simulation memoryIntensiveSimulation = DispatcherTestUtils.createMockSimulation(900, 900, 900); - insertOrUpdateStatus(); + DispatcherTestUtils.insertOrUpdateStatus(simulationDB); Assertions.assertThrows(RuntimeException.class, () -> {stateMachine.onDispatch(memoryIntensiveSimulation, getLatestJobSubmission(), simulationDB, testMessageSession);}, "Can't dispatch simulation that is already running."); Assertions.assertThrows(NoSuchElementException.class, () -> getClientTopicMessage().getSchedulerStatus().isFailed(), "Client receives failure because simulation is already running."); - insertOrUpdateStatus(simKey, jobIndex, taskID, testUser, SimulationJobStatus.SchedulerStatus.WAITING); + DispatcherTestUtils.insertOrUpdateStatus(simKey, jobIndex, taskID, testUser, SimulationJobStatus.SchedulerStatus.WAITING, simulationDB); stateMachine.onDispatch(memoryIntensiveSimulation, getLatestJobSubmission(), simulationDB, testMessageSession); jobStatus = getLatestJobSubmission(); - Assertions.assertTrue(jobStatus.getSchedulerStatus().isFailed(), "Memory size to large"); + Assertions.assertTrue(jobStatus.getSchedulerStatus().isFailed(), "Memory size too large"); Assertions.assertTrue(getClientTopicMessage().getSchedulerStatus().isFailed(), "Failed because of memory size."); - insertOrUpdateStatus(); - statusMessage = stateMachine.onStopRequest(new User("Bob", new KeyValue("2")), getLatestJobSubmission(), simulationDB, testMessageSession); + DispatcherTestUtils.insertOrUpdateStatus(simulationDB); + statusMessage = stateMachine.onStopRequest(DispatcherTestUtils.bob, getLatestJobSubmission(), simulationDB, testMessageSession); Assertions.assertTrue(statusMessage.getSimulationJobStatus().getSchedulerStatus().isFailed(), "Stopping as another user."); Assertions.assertTrue(getClientTopicMessage().getSchedulerStatus().isFailed(), "Can't stop as another user."); } @@ -247,21 +172,21 @@ public void stateShouldTransitionToWaiting() throws SQLException, VCMessagingExc @Test public void stateShouldTransitionToDispatched() throws SQLException, DataAccessException, VCMessagingException, PropertyVetoException, MathException, ExpressionBindingException { - insertOrUpdateStatus(simKey, jobIndex, taskID, testUser, SimulationJobStatus.SchedulerStatus.WAITING); + DispatcherTestUtils.insertOrUpdateStatus(simKey, jobIndex, taskID, testUser, SimulationJobStatus.SchedulerStatus.WAITING, simulationDB); WorkerEvent acceptedWorker = createWorkerEvent(new ChangedStateValues(simID, null, WorkerEvent.JOB_ACCEPTED, taskID, "Worker just got accepted")); stateMachine.onWorkerEvent(acceptedWorker, simulationDB, testMessageSession); SimulationJobStatus jobStatus = getLatestJobSubmission(); Assertions.assertTrue(jobStatus.getSchedulerStatus().isDispatched(), "Job recently got accepted, only works if previous state was waiting."); Assertions.assertTrue(getClientTopicMessage().getSchedulerStatus().isDispatched()); - insertOrUpdateStatus(); + DispatcherTestUtils.insertOrUpdateStatus(simulationDB); stateMachine.onWorkerEvent(acceptedWorker, simulationDB, testMessageSession); jobStatus = getLatestJobSubmission(); Assertions.assertTrue(jobStatus.getSchedulerStatus().isRunning(), "The state has not changed from running, because something that is running can not be dispatched."); - insertOrUpdateStatus(simKey, jobIndex, taskID, testUser, SimulationJobStatus.SchedulerStatus.WAITING); - Simulation simulation = createMockSimulation(50, 50, 50); + DispatcherTestUtils.insertOrUpdateStatus(simKey, jobIndex, taskID, testUser, SimulationJobStatus.SchedulerStatus.WAITING, simulationDB); + Simulation simulation = DispatcherTestUtils.createMockSimulation(50, 50, 50); stateMachine.onDispatch(simulation, getLatestJobSubmission(), simulationDB, testMessageSession); jobStatus = getLatestJobSubmission(); Assertions.assertTrue(jobStatus.getSchedulerStatus().isDispatched()); @@ -272,7 +197,7 @@ public void stateShouldTransitionToDispatched() throws SQLException, DataAccessE public void stateShouldTransitionToRunning() throws SQLException, DataAccessException, VCMessagingException { for (int workerStatus: WorkerEvent.ALL_JOB_EVENTS){ WorkerEvent workerEvent = createWorkerEvent(new ChangedStateValues(simID, null, workerStatus, taskID, "")); - insertOrUpdateStatus(simKey, jobIndex, taskID, testUser, SimulationJobStatus.SchedulerStatus.WAITING); + DispatcherTestUtils.insertOrUpdateStatus(simKey, jobIndex, taskID, testUser, SimulationJobStatus.SchedulerStatus.WAITING, simulationDB); stateMachine.onWorkerEvent(workerEvent, simulationDB, testMessageSession); SimulationJobStatus jobStatus = getLatestJobSubmission(); if (workerEvent.isProgressEvent() || workerEvent.isNewDataEvent() || workerEvent.isStartingEvent() || workerEvent.isWorkerAliveEvent()){ @@ -291,7 +216,7 @@ public void stateShouldTransitionToRunning() throws SQLException, DataAccessExce public void stateShouldTransitionToCompleted() throws SQLException, VCMessagingException, DataAccessException { for (int workerStatus : WorkerEvent.ALL_JOB_EVENTS){ WorkerEvent workerEvent = createWorkerEvent(new ChangedStateValues(simID, SimulationJobStatus.SchedulerStatus.RUNNING, workerStatus, taskID, "")); - insertOrUpdateStatus(); + DispatcherTestUtils.insertOrUpdateStatus(simulationDB); stateMachine.onWorkerEvent(workerEvent, simulationDB, testMessageSession); SimulationJobStatus jobStatus = getLatestJobSubmission(); if (workerEvent.isCompletedEvent()){ @@ -310,7 +235,7 @@ public void stateShouldTransitionToCompleted() throws SQLException, VCMessagingE public void stateShouldTransitionToStopped() throws SQLException, DataAccessException, VCMessagingException { for (SimulationJobStatus.SchedulerStatus status : SimulationJobStatus.SchedulerStatus.values()){ - insertOrUpdateStatus(simKey,jobIndex, taskID,testUser, status); + DispatcherTestUtils.insertOrUpdateStatus(simKey,jobIndex, taskID,testUser, status, simulationDB); if (status.isActive()){ stateMachine.onStopRequest(testUser, getLatestJobSubmission(), simulationDB, testMessageSession); Assertions.assertTrue(getLatestJobSubmission().getSchedulerStatus().isStopped(), ""); From 1b7a824fe6e57d2e70ad58d8fb1e4888bd4c9b54 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Tue, 3 Sep 2024 08:15:03 -0400 Subject: [PATCH 14/24] Reintroduce Server Topic Message --- .../dispatcher/SimulationStateMachine.java | 21 +++++++++++++------ .../dispatcher/MockMessagingService.java | 3 ++- .../dispatcher/MockVCMessageSession.java | 19 ++++++----------- .../dispatcher/SimulationDispatcherTest.java | 16 ++++++++++++-- 4 files changed, 37 insertions(+), 22 deletions(-) diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java index 30a056e40d..989e973378 100644 --- a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java @@ -2,10 +2,7 @@ import cbit.rmi.event.WorkerEvent; import cbit.vcell.field.FieldDataIdentifierSpec; -import cbit.vcell.message.VCMessageSession; -import cbit.vcell.message.VCMessagingConstants; -import cbit.vcell.message.VCMessagingException; -import cbit.vcell.message.VCellTopic; +import cbit.vcell.message.*; import cbit.vcell.message.messages.MessageConstants; import cbit.vcell.message.messages.SimulationTaskMessage; import cbit.vcell.message.messages.StatusMessage; @@ -345,13 +342,11 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba simulationDatabase.updateSimulationJobStatus(newJobStatus,runningStateInfo); StatusMessage msgForClient = new StatusMessage(newJobStatus, userName, progress, timepoint); - // TODO: Implement messaging to client msgForClient.sendToClient(session); if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); } else { simulationDatabase.updateSimulationJobStatus(newJobStatus); StatusMessage msgForClient = new StatusMessage(newJobStatus, userName, null, null); - // TODO: Implement messaging to client msgForClient.sendToClient(session); if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); } @@ -541,6 +536,20 @@ null, new VCSimulationIdentifier(simKey, user), jobIndex, simJobStatus.getSubmit SimulationJobStatus.SchedulerStatus.STOPPED, taskID, simJobStatus.getSimulationMessage(), simQueueEntryStatus, simExeStatus ); + // + // send stopSimulation to serviceControl topic + // + VCMessage msg = session.createMessage(); + msg.setStringProperty(VCMessagingConstants.MESSAGE_TYPE_PROPERTY, MessageConstants.MESSAGE_TYPE_STOPSIMULATION_VALUE); + msg.setLongProperty(MessageConstants.SIMKEY_PROPERTY, Long.parseLong(simKey + "")); + msg.setIntProperty(MessageConstants.JOBINDEX_PROPERTY, jobIndex); + msg.setIntProperty(MessageConstants.TASKID_PROPERTY, taskID); + msg.setStringProperty(VCMessagingConstants.USERNAME_PROPERTY, user.getName()); + if (simExeStatus.getHtcJobID()!=null){ + msg.setStringProperty(MessageConstants.HTCJOBID_PROPERTY, simExeStatus.getHtcJobID().toDatabase()); + } + session.sendTopicMessage(VCellTopic.ServiceControlTopic, msg); + simulationDatabase.updateSimulationJobStatus(newJobStatus); statusMessage = new StatusMessage(simulationJobStatusRecord, user.getName(), null, null); statusMessage.sendToClient(session); diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockMessagingService.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockMessagingService.java index 82b0283713..b2a634c996 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockMessagingService.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockMessagingService.java @@ -8,10 +8,11 @@ public class MockMessagingService implements VCMessagingService { public ArrayList messagingConsumers = new ArrayList<>(); + public final MockVCMessageSession mockVCMessageSession = new MockVCMessageSession(); @Override public VCMessageSession createProducerSession() { - return new MockVCMessageSession(); + return mockVCMessageSession; } @Override diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockVCMessageSession.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockVCMessageSession.java index 092022373e..8a35e48096 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockVCMessageSession.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockVCMessageSession.java @@ -2,30 +2,23 @@ import cbit.vcell.message.*; import cbit.vcell.message.jms.VCMessageJms; +import org.apache.activemq.command.ActiveMQMessage; import org.apache.activemq.command.ActiveMQObjectMessage; import org.apache.activemq.command.ActiveMQTextMessage; import org.vcell.util.document.UserLoginInfo; -import javax.jms.*; +import javax.jms.JMSException; +import javax.jms.ObjectMessage; +import javax.jms.TextMessage; import java.io.Serializable; -import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedList; import java.util.Queue; public class MockVCMessageSession implements VCMessageSession { - private Connection connection; - private Session session; - public MockVCMessageSession(){ - try { -// connection = ActiveMQConnection.makeConnection(""); -// session = connection.createSession(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } + public MockVCMessageSession(){ } private final HashMap> topics = new HashMap<>(){{ put(VCellTopic.ClientStatusTopic, new LinkedList<>()); @@ -77,7 +70,7 @@ public VCMessage createTextMessage(String text) { @Override public VCMessage createMessage() { - return null; + return new VCMessageJms(new ActiveMQMessage(), null, null); } @Override diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java index ed78c82efe..345abd816f 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java @@ -1,11 +1,18 @@ package cbit.vcell.message.server.dispatcher; +import cbit.vcell.math.MathException; +import cbit.vcell.message.VCMessagingConstants; +import cbit.vcell.message.VCellTopic; +import cbit.vcell.message.messages.MessageConstants; +import cbit.vcell.parser.ExpressionBindingException; import cbit.vcell.server.SimulationJobStatus; import cbit.vcell.server.SimulationStatus; +import cbit.vcell.solver.Simulation; import org.junit.jupiter.api.*; import org.vcell.util.DataAccessException; import org.vcell.util.document.User; +import java.beans.PropertyVetoException; import java.sql.SQLException; @Tag("Fast") @@ -30,10 +37,12 @@ public void beforeEach(){ mockSimulationDB = new MockSimulationDB(); } + //################# Test Simulation Service Impl ####################### + // All the get functions withing SimulationDispatcher seem to be exercising the DB and not simulation control, so not tested @Test public void onStartRequestTest() throws DataAccessException, SQLException { - SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(mockSimulationDB, mockMessagingServiceInternal, mockMessagingServiceSim); + SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(mockSimulationDB, mockMessagingServiceInternal, mockMessagingServiceSim, mockHtcProxy); SimulationStatus simStatus = simulationDispatcher.simServiceImpl.startSimulation(testUser, DispatcherTestUtils.simID, 1); SimulationJobStatus jobStatus = mockSimulationDB.getLatestSimulationJobStatus(DispatcherTestUtils.simKey, 0); Assertions.assertTrue(jobStatus.getSchedulerStatus().isWaiting()); @@ -42,10 +51,13 @@ public void onStartRequestTest() throws DataAccessException, SQLException { @Test public void onStopRequestTest() throws DataAccessException, SQLException { DispatcherTestUtils.insertOrUpdateStatus(mockSimulationDB); - SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(mockSimulationDB, mockMessagingServiceInternal, mockMessagingServiceSim); + SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(mockSimulationDB, mockMessagingServiceInternal, mockMessagingServiceSim, mockHtcProxy); SimulationStatus simStatus = simulationDispatcher.simServiceImpl.stopSimulation(testUser, DispatcherTestUtils.simID); SimulationJobStatus jobStatus = mockSimulationDB.getLatestSimulationJobStatus(DispatcherTestUtils.simKey, 0); Assertions.assertTrue(jobStatus.getSchedulerStatus().isStopped()); + + String s = mockMessagingServiceInternal.mockVCMessageSession.getTopicMessage(VCellTopic.ServiceControlTopic).getStringProperty(VCMessagingConstants.MESSAGE_TYPE_PROPERTY); + Assertions.assertEquals(MessageConstants.MESSAGE_TYPE_STOPSIMULATION_VALUE, s); } } From 2aeaf916673124e6e3b5d3c170f8ad21df9b2c45 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Tue, 3 Sep 2024 08:51:53 -0400 Subject: [PATCH 15/24] Simulation Dispatcher Thread Tests --- .../dispatcher/SimulationDispatcher.java | 15 ++-- .../dispatcher/DispatcherTestUtils.java | 14 +++- .../server/dispatcher/MockHtcProxy.java | 74 +++++++++++++++++++ .../server/dispatcher/MockSimulationDB.java | 8 +- .../dispatcher/SimulationDispatcherTest.java | 42 +++++++++++ 5 files changed, 144 insertions(+), 9 deletions(-) create mode 100644 vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockHtcProxy.java diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java index 786a7a99b6..01f8022c85 100644 --- a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java @@ -87,7 +87,7 @@ public class SimulationDispatcher { private final SimulationDispatcherEngine simDispatcherEngine = new SimulationDispatcherEngine(); - private final DispatchThread dispatchThread; + protected final DispatchThread dispatchThread; private final SimulationMonitor simMonitor; private final VCMessageSession dispatcherQueueSession_int; private final VCMessageSession clientStatusTopicSession_int; @@ -235,7 +235,7 @@ private void reloadSpecialUsers() { } public class DispatchThread extends Thread { - Object notifyObject = new Object(); + final Object notifyObject = new Object(); public DispatchThread() { super(); @@ -545,8 +545,8 @@ public Thread newThread(Runnable r) { } public static SimulationDispatcher simulationDispatcherCreator(SimulationDatabase simulationDatabase, VCMessagingService messagingServiceInternal, - VCMessagingService messagingServiceSim){ - return new SimulationDispatcher(simulationDatabase, messagingServiceInternal, messagingServiceSim); + VCMessagingService messagingServiceSim, HtcProxy htcProxy){ + return new SimulationDispatcher(simulationDatabase, messagingServiceInternal, messagingServiceSim, htcProxy); } public static SimulationDispatcher simulationDispatcherCreator() throws SQLException, DataAccessException { @@ -566,11 +566,12 @@ public static SimulationDispatcher simulationDispatcherCreator() throws SQLExcep int jmsport_sim = Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.jmsSimPortInternal)); vcMessagingServiceSim.setConfiguration(new ServerMessagingDelegate(), jmshost_sim, jmsport_sim); - return SimulationDispatcher.simulationDispatcherCreator(simulationDatabase, vcMessagingServiceInternal, vcMessagingServiceSim); + return SimulationDispatcher.simulationDispatcherCreator(simulationDatabase, + vcMessagingServiceInternal, vcMessagingServiceSim, SlurmProxy.createRemoteProxy()); } private SimulationDispatcher(SimulationDatabase simulationDatabase, VCMessagingService messagingServiceInternal, - VCMessagingService messagingServiceSim){ + VCMessagingService messagingServiceSim, HtcProxy htcProxy){ this.simulationDatabase = simulationDatabase; this.vcMessagingService_int = messagingServiceInternal; this.vcMessagingService_sim = messagingServiceSim; @@ -605,7 +606,7 @@ public void onQueueMessage(VCMessage vcMessage, VCMessageSession session) throws this.simMonitorThreadSession_sim = this.vcMessagingService_sim.createProducerSession(); this.simMonitor = new SimulationMonitor(); - this.htcProxy = SlurmProxy.createRemoteProxy(); + this.htcProxy = htcProxy; } diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/DispatcherTestUtils.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/DispatcherTestUtils.java index a3b14262d5..7d8e70427f 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/DispatcherTestUtils.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/DispatcherTestUtils.java @@ -36,6 +36,8 @@ public class DispatcherTestUtils { private static String previousHTCUser = ""; private static String previousHTCUserKeyFile = ""; private static String previousMaxJobsPerScan = ""; + private static String previousOdeJobsPerUser = ""; + private static String previousPdeJobsPerUser = ""; public static final VCellServerID testVCellServerID = VCellServerID.getServerID("test"); public static final MockVCMessageSession testMessageSession = new MockVCMessageSession(); @@ -48,7 +50,7 @@ public class DispatcherTestUtils { public static void setRequiredProperties(){ previousServerID = PropertyLoader.getProperty(PropertyLoader.vcellServerIDProperty, ""); - PropertyLoader.setProperty(PropertyLoader.vcellServerIDProperty, ""); + PropertyLoader.setProperty(PropertyLoader.vcellServerIDProperty, testVCellServerID.toString()); previousHtcMax = PropertyLoader.getProperty(PropertyLoader.htcMaxMemoryMB, ""); PropertyLoader.setProperty(PropertyLoader.htcMaxMemoryMB, "4096"); @@ -85,6 +87,12 @@ public static void setRequiredProperties(){ previousMaxJobsPerScan = PropertyLoader.getProperty(PropertyLoader.maxJobsPerScan, ""); PropertyLoader.setProperty(PropertyLoader.maxJobsPerScan, "100"); + + previousPdeJobsPerUser = PropertyLoader.getProperty(PropertyLoader.maxPdeJobsPerUser, ""); + PropertyLoader.setProperty(PropertyLoader.maxPdeJobsPerUser, "100"); + + previousOdeJobsPerUser = PropertyLoader.getProperty(PropertyLoader.maxOdeJobsPerUser, ""); + PropertyLoader.setProperty(PropertyLoader.maxOdeJobsPerUser, "100"); } public static void restoreRequiredProperties(){ @@ -138,6 +146,10 @@ public static void insertOrUpdateStatus(KeyValue simKey, int jobIndex, int taskI insertOrUpdateStatus(simKey, jobIndex, taskID, user, SimulationJobStatus.SchedulerStatus.RUNNING, simulationDB); } + public static void insertOrUpdateStatus(SimulationDatabase simulationDatabase, SimulationJobStatus.SchedulerStatus status) throws SQLException, DataAccessException { + insertOrUpdateStatus(simKey, jobIndex, taskID, alice, status, simulationDatabase); + } + /** Defaults to a running status. */ diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockHtcProxy.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockHtcProxy.java new file mode 100644 index 0000000000..904e892c1b --- /dev/null +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockHtcProxy.java @@ -0,0 +1,74 @@ +package cbit.vcell.message.server.dispatcher; + +import cbit.vcell.message.server.cmd.CommandService; +import cbit.vcell.message.server.htc.HtcException; +import cbit.vcell.message.server.htc.HtcJobNotFoundException; +import cbit.vcell.message.server.htc.HtcJobStatus; +import cbit.vcell.message.server.htc.HtcProxy; +import cbit.vcell.messaging.server.SimulationTask; +import cbit.vcell.server.HtcJobID; +import cbit.vcell.simdata.PortableCommand; +import cbit.vcell.solvers.ExecutableCommand; +import org.vcell.util.exe.ExecutableException; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +public class MockHtcProxy extends HtcProxy { + public MockHtcProxy(CommandService commandService, String htcUser) { + super(commandService, htcUser); + } + + @Override + public void killJobSafe(HtcJobInfo htcJobInfo) throws ExecutableException, HtcJobNotFoundException, HtcException { + + } + + @Override + public void killJobUnsafe(HtcJobID htcJobId) throws ExecutableException, HtcJobNotFoundException, HtcException { + + } + + @Override + public void killJobs(String htcJobSubstring) throws ExecutableException, HtcJobNotFoundException, HtcException { + + } + + @Override + public Map getJobStatus(List requestedHtcJobInfos) throws ExecutableException, IOException { + return Map.of(); + } + + @Override + public HtcJobID submitJob(String jobName, File sub_file_internal, File sub_file_external, ExecutableCommand.Container commandSet, int ncpus, double memSize, Collection postProcessingCommands, SimulationTask simTask, File primaryUserDirExternal) throws ExecutableException { + return null; + } + + @Override + public HtcJobID submitOptimizationJob(String jobName, File sub_file_internal, File sub_file_external, File optProblemInputFile, File optProblemOutputFile, File optReportFile) throws ExecutableException { + return null; + } + + @Override + public HtcProxy cloneThreadsafe() { + return null; + } + + @Override + public Map getRunningJobs() throws ExecutableException, IOException { + return Map.of(); + } + + @Override + public PartitionStatistics getPartitionStatistics() throws HtcException, ExecutableException, IOException { + return new PartitionStatistics(1, 20, 100); + } + + @Override + public String getSubmissionFileExtension() { + return ""; + } +} diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java index af407a0492..b5c345f88e 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java @@ -24,6 +24,8 @@ public class MockSimulationDB implements SimulationDatabase{ {put(specialAdmin.getName(), specialAdmin); put(DispatcherTestUtils.alice.getName(), DispatcherTestUtils.alice);} }; + private final HashMap simulations = new HashMap<>(); + @Override public SimulationJobStatus getLatestSimulationJobStatus(KeyValue simKey, int jobIndex) throws DataAccessException, SQLException { ArrayList simList = dbTable.get(simKey.toString()); @@ -100,7 +102,7 @@ public KeyValue[] getSimulationKeysFromBiomodel(KeyValue biomodelKey) throws SQL @Override public Simulation getSimulation(User user, KeyValue simKey) throws DataAccessException { - throw new DataAccessException(); + return simulations.get(simKey.toString() + user.getName()); } @Override @@ -166,4 +168,8 @@ public void resetDataBase(){ dbTable = new HashMap<>(); } + public void insertSimulation(User user, Simulation sim){ + simulations.put(sim.getKey().toString() + user.getName(), sim); + } + } diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java index 345abd816f..4cf058a751 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java @@ -21,6 +21,7 @@ public class SimulationDispatcherTest { private MockSimulationDB mockSimulationDB = new MockSimulationDB(); private MockMessagingService mockMessagingServiceInternal = new MockMessagingService(); private MockMessagingService mockMessagingServiceSim = new MockMessagingService(); + private MockHtcProxy mockHtcProxy = new MockHtcProxy(null, null); @BeforeAll public static void setSystemProperties(){ @@ -60,4 +61,45 @@ public void onStopRequestTest() throws DataAccessException, SQLException { Assertions.assertEquals(MessageConstants.MESSAGE_TYPE_STOPSIMULATION_VALUE, s); } + + //###################### Test Dispatcher Thread ########################### + @Test + public void dispatcherThreadTest() throws SQLException, DataAccessException, InterruptedException, PropertyVetoException, MathException, ExpressionBindingException { + DispatcherTestUtils.insertOrUpdateStatus(mockSimulationDB, SimulationJobStatus.SchedulerStatus.WAITING); + SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(mockSimulationDB, mockMessagingServiceInternal, mockMessagingServiceSim, mockHtcProxy); + SimulationDispatcher.DispatchThread thread = simulationDispatcher.dispatchThread; + synchronized (thread.notifyObject){ + thread.notifyObject.notify(); + } + SimulationJobStatus jobStatus = mockSimulationDB.getLatestSimulationJobStatus(DispatcherTestUtils.simKey, 0); + Assertions.assertTrue(jobStatus.getSchedulerStatus().isWaiting(), "Still waiting."); + + // needs time for the dispatcher thread to fully update + Thread.sleep(1000); + + jobStatus = mockSimulationDB.getLatestSimulationJobStatus(DispatcherTestUtils.simKey, 0); + Assertions.assertTrue(jobStatus.getSchedulerStatus().isFailed(), "Simulation gets aborted since theres no simulation in DB."); + + Simulation mockSimulation = DispatcherTestUtils.createMockSimulation(20, 20, 20); + mockSimulationDB.insertSimulation(DispatcherTestUtils.alice, mockSimulation); + DispatcherTestUtils.insertOrUpdateStatus(mockSimulation.getKey(), DispatcherTestUtils.jobIndex, DispatcherTestUtils.taskID, DispatcherTestUtils.alice, + SimulationJobStatus.SchedulerStatus.WAITING, mockSimulationDB); + synchronized (thread.notifyObject){ + thread.notifyObject.notify(); + } + Thread.sleep(1000); + + jobStatus = mockSimulationDB.getLatestSimulationJobStatus(mockSimulation.getKey(), 0); + Assertions.assertTrue(jobStatus.getSchedulerStatus().isDispatched(), "Dispatches"); + } + + + + //###################### Test Simulation Monitor + + + + + + } From 1fba4e570170d6522a9c5690832336526b1cd168 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Wed, 4 Sep 2024 07:39:43 -0400 Subject: [PATCH 16/24] Zombie Thread Tests --- .../dispatcher/SimulationDispatcher.java | 30 +++++----- .../dispatcher/DispatcherTestUtils.java | 5 +- .../server/dispatcher/MockHtcProxy.java | 31 +++++++++-- .../server/dispatcher/MockSimulationDB.java | 28 +++++++++- .../dispatcher/SimulationDispatcherTest.java | 55 +++++++++++++++++-- 5 files changed, 123 insertions(+), 26 deletions(-) diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java index 01f8022c85..f31b4e79f2 100644 --- a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java @@ -88,7 +88,7 @@ public class SimulationDispatcher { private final SimulationDispatcherEngine simDispatcherEngine = new SimulationDispatcherEngine(); protected final DispatchThread dispatchThread; - private final SimulationMonitor simMonitor; + protected final SimulationMonitor simMonitor; private final VCMessageSession dispatcherQueueSession_int; private final VCMessageSession clientStatusTopicSession_int; private final VCMessageSession simMonitorThreadSession_sim; @@ -233,7 +233,7 @@ private void reloadSpecialUsers() { } lastSpecialUserCheck = System.currentTimeMillis(); } - public class DispatchThread extends Thread { + protected class DispatchThread extends Thread { final Object notifyObject = new Object(); @@ -350,9 +350,11 @@ public void run() { } } - class SimulationMonitor implements ThreadFactory, RejectedExecutionHandler { - private ScheduledThreadPoolExecutor executor; + protected class SimulationMonitor implements ThreadFactory, RejectedExecutionHandler { + protected final ScheduledThreadPoolExecutor executor; private int threadCount; + protected ZombieKiller initialZombieKiller = new ZombieKiller(); + protected QueueFlusher initialQueueFlusher = new QueueFlusher(); /** * synchronizes {@link SimulationDispatcher#onWorkerEventMessage(VCMessage, VCMessageSession)} and * {@link QueueFlusher#flushWorkerEventQueue()} @@ -362,14 +364,17 @@ class SimulationMonitor implements ThreadFactory, RejectedExecutionHandler { public SimulationMonitor( ) { threadCount = 1; executor = new ScheduledThreadPoolExecutor(2,this,this); - executor.scheduleAtFixedRate(new ZombieKiller( ), 0, ZOMBIE_MINUTES, TimeUnit.MINUTES); - executor.scheduleAtFixedRate(new QueueFlusher( ), 1,FLUSH_QUEUE_MINUTES,TimeUnit.MINUTES); + executor.scheduleAtFixedRate(initialZombieKiller, 0, ZOMBIE_MINUTES, TimeUnit.MINUTES); + executor.scheduleAtFixedRate(initialQueueFlusher, 1,FLUSH_QUEUE_MINUTES,TimeUnit.MINUTES); } /** * find and kill zombie processes */ - class ZombieKiller implements Runnable { + protected class ZombieKiller implements Runnable { + public static final String noJob = "no jobStatus found in database for running htc job"; + public static final String newJobFound = "newer task found in database for running htc job"; + public static final String jobIsAlreadyDone = "jobStatus Done in database for running htc job"; @Override public void run() { try { @@ -387,13 +392,13 @@ public void run() { String failureMessage = null; boolean killJob = false; if (simJobStatus==null){ - failureMessage = "no jobStatus found in database for running htc job"; + failureMessage = noJob; killJob = true; }else if (simTaskInfo.taskId < simJobStatus.getTaskID()){ - failureMessage = "newer task found in database for running htc job"; + failureMessage = newJobFound; killJob = true; }else if (simJobStatus.getSchedulerStatus().isDone()){ - failureMessage = "jobStatus Done in database for running htc job"; + failureMessage = jobIsAlreadyDone; if (simJobStatus.getSimulationExecutionStatus()==null){ killJob = true; }else{ @@ -406,9 +411,8 @@ public void run() { } if (killJob && HtcProxy.isMySimulationJob(htcJobInfo)){ if (lg.isWarnEnabled()) { - lg.warn("killing " + htcJobInfo + ", " + failureMessage); + lg.warn("killing {}; {}; {}", htcJobInfo, failureMessage, simJobStatus); } - VCMongoMessage.sendZombieJob(simJobStatus,failureMessage,htcJobInfo.getHtcJobID()); htcProxy.killJobSafe(htcJobInfo); } }catch (Exception e){ @@ -428,7 +432,7 @@ public void run() { /** * flush message queue */ - class QueueFlusher implements Runnable { + protected class QueueFlusher implements Runnable { public void run() { try { traceThread(this); diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/DispatcherTestUtils.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/DispatcherTestUtils.java index 7d8e70427f..7004768334 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/DispatcherTestUtils.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/DispatcherTestUtils.java @@ -47,6 +47,7 @@ public class DispatcherTestUtils { public static User alice = new User("Alice", new KeyValue("0")); public static User bob = new User("Bob", new KeyValue("1")); public static final VCSimulationIdentifier simID = new VCSimulationIdentifier(simKey, alice); + public static final HtcJobID htcJobID = new HtcJobID("2", HtcJobID.BatchSystemType.SLURM); public static void setRequiredProperties(){ previousServerID = PropertyLoader.getProperty(PropertyLoader.vcellServerIDProperty, ""); @@ -109,6 +110,8 @@ public static void restoreRequiredProperties(){ PropertyLoader.setProperty(PropertyLoader.htcUser, previousHTCUser); PropertyLoader.setProperty(PropertyLoader.htcUserKeyFile, previousHTCUserKeyFile); PropertyLoader.setProperty(PropertyLoader.maxJobsPerScan, previousMaxJobsPerScan); + PropertyLoader.setProperty(PropertyLoader.maxOdeJobsPerUser, previousOdeJobsPerUser); + PropertyLoader.setProperty(PropertyLoader.maxPdeJobsPerUser, previousPdeJobsPerUser); } public static Simulation createMockSimulation(int iSizeX, int iSizeY, int iSizeZ) throws PropertyVetoException, MathException, ExpressionBindingException { @@ -134,7 +137,7 @@ public static void insertOrUpdateStatus(KeyValue simKey, int jobIndex, int taskI SimulationJobStatus simulationJobStatus = new SimulationJobStatus(testVCellServerID, simID, jobIndex, Date.from(Instant.now()), status, taskID, SimulationMessage.workerAccepted("accepted"), null, new SimulationExecutionStatus(Date.from(Instant.now()), "", - Date.from(Instant.now()), Date.from(Instant.now()), false, new HtcJobID("2", HtcJobID.BatchSystemType.SLURM))); + Date.from(Instant.now()), Date.from(Instant.now()), false, htcJobID)); if (jobStatus == null){ simulationDB.insertSimulationJobStatus(simulationJobStatus); } else { diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockHtcProxy.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockHtcProxy.java index 904e892c1b..fa188df76d 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockHtcProxy.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockHtcProxy.java @@ -5,26 +5,31 @@ import cbit.vcell.message.server.htc.HtcJobNotFoundException; import cbit.vcell.message.server.htc.HtcJobStatus; import cbit.vcell.message.server.htc.HtcProxy; +import cbit.vcell.message.server.htc.slurm.SlurmJobStatus; import cbit.vcell.messaging.server.SimulationTask; import cbit.vcell.server.HtcJobID; +import cbit.vcell.server.SimulationJobStatus; import cbit.vcell.simdata.PortableCommand; import cbit.vcell.solvers.ExecutableCommand; +import org.vcell.util.DataAccessException; import org.vcell.util.exe.ExecutableException; import java.io.File; import java.io.IOException; -import java.util.Collection; -import java.util.List; -import java.util.Map; +import java.sql.SQLException; +import java.util.*; public class MockHtcProxy extends HtcProxy { - public MockHtcProxy(CommandService commandService, String htcUser) { + private final MockSimulationDB mockSimulationDB; + public MockHtcProxy(CommandService commandService, String htcUser, MockSimulationDB mockSimulationDB) { super(commandService, htcUser); + this.mockSimulationDB = mockSimulationDB; } + public final ArrayList jobsKilledSafely = new ArrayList<>(); @Override public void killJobSafe(HtcJobInfo htcJobInfo) throws ExecutableException, HtcJobNotFoundException, HtcException { - + jobsKilledSafely.add(htcJobInfo); } @Override @@ -59,7 +64,21 @@ public HtcProxy cloneThreadsafe() { @Override public Map getRunningJobs() throws ExecutableException, IOException { - return Map.of(); + HashMap map = new HashMap<>(); + SimulationJobStatus[] statuses; + try { + statuses = mockSimulationDB.getActiveJobs(DispatcherTestUtils.testVCellServerID); + } catch (DataAccessException | SQLException e) { + throw new RuntimeException(e); + } + for (SimulationJobStatus status : statuses){ + if (status.getSchedulerStatus().isRunning()){ + HtcJobInfo jobInfo = new HtcJobInfo(DispatcherTestUtils.htcJobID, HtcProxy.createHtcSimJobName(new SimTaskInfo(status.getVCSimulationIdentifier().getSimulationKey(), status.getJobIndex(), status.getTaskID()))); + HtcJobStatus jobStatus = new HtcJobStatus(SlurmJobStatus.RUNNING); + map.put(jobInfo, jobStatus); + } + } + return map; } @Override diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java index b5c345f88e..5777b2886b 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java @@ -26,6 +26,17 @@ public class MockSimulationDB implements SimulationDatabase{ private final HashMap simulations = new HashMap<>(); + // Return a latest simulation that differs in one of these ways + public enum BadLatestSimulation{ + HIGHER_TASK_ID, + RETURN_NULL, + IS_DONE, + DO_NOTHING + } + + public BadLatestSimulation badLatestSimulation = BadLatestSimulation.DO_NOTHING; + + @Override public SimulationJobStatus getLatestSimulationJobStatus(KeyValue simKey, int jobIndex) throws DataAccessException, SQLException { ArrayList simList = dbTable.get(simKey.toString()); @@ -40,7 +51,21 @@ public SimulationJobStatus getLatestSimulationJobStatus(KeyValue simKey, int job latestSim = jobStatus; } } - return latestSim; + switch (badLatestSimulation){ + case RETURN_NULL -> { + return null; + } case HIGHER_TASK_ID -> { + SimulationJobStatus simulationJobStatus = new SimulationJobStatus(latestSim.getServerID(), latestSim.getVCSimulationIdentifier(), latestSim.getJobIndex(), + latestSim.getSubmitDate(), latestSim.getSchedulerStatus(), latestSim.getTaskID() + 1, latestSim.getSimulationMessage(), latestSim.getSimulationQueueEntryStatus(), latestSim.getSimulationExecutionStatus()); + return simulationJobStatus; + } case IS_DONE -> { + return new SimulationJobStatus(latestSim.getServerID(), latestSim.getVCSimulationIdentifier(), latestSim.getJobIndex(), latestSim.getSubmitDate(), SimulationJobStatus.SchedulerStatus.COMPLETED, + latestSim.getTaskID(), latestSim.getSimulationMessage(), latestSim.getSimulationQueueEntryStatus(), null); + }default -> { + return latestSim; + } + } + } @Override @@ -166,6 +191,7 @@ versionBranchPoint, new BigDecimal(22), Date.from(Instant.now()), versionFlag, " public void resetDataBase(){ dbTable = new HashMap<>(); + badLatestSimulation = BadLatestSimulation.DO_NOTHING; } public void insertSimulation(User user, Simulation sim){ diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java index 4cf058a751..13ef50b9dd 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java @@ -8,20 +8,29 @@ import cbit.vcell.server.SimulationJobStatus; import cbit.vcell.server.SimulationStatus; import cbit.vcell.solver.Simulation; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.Logger; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.WriterAppender; +import org.apache.logging.log4j.core.config.Configurator; +import org.apache.logging.log4j.spi.ExtendedLogger; import org.junit.jupiter.api.*; import org.vcell.util.DataAccessException; import org.vcell.util.document.User; import java.beans.PropertyVetoException; +import java.io.IOException; +import java.io.StringWriter; import java.sql.SQLException; @Tag("Fast") public class SimulationDispatcherTest { + public static ExtendedLogger lg = LoggerContext.getContext().getLogger(SimulationDispatcher.class); private final static User testUser = DispatcherTestUtils.alice; private MockSimulationDB mockSimulationDB = new MockSimulationDB(); - private MockMessagingService mockMessagingServiceInternal = new MockMessagingService(); - private MockMessagingService mockMessagingServiceSim = new MockMessagingService(); - private MockHtcProxy mockHtcProxy = new MockHtcProxy(null, null); + private final MockMessagingService mockMessagingServiceInternal = new MockMessagingService(); + private final MockMessagingService mockMessagingServiceSim = new MockMessagingService(); + private final MockHtcProxy mockHtcProxy = new MockHtcProxy(null, null, mockSimulationDB); @BeforeAll public static void setSystemProperties(){ @@ -35,7 +44,7 @@ public static void restoreSystemProperties(){ @BeforeEach public void beforeEach(){ - mockSimulationDB = new MockSimulationDB(); + mockSimulationDB.resetDataBase(); } //################# Test Simulation Service Impl ####################### @@ -95,8 +104,44 @@ public void dispatcherThreadTest() throws SQLException, DataAccessException, Int - //###################### Test Simulation Monitor + //###################### Test Simulation Monitor ########################## + @Test + public void zombieKillerTest() throws SQLException, DataAccessException, InterruptedException, IOException { + Level pastLevel = lg.getLevel(); + Configurator.setLevel(lg, Level.WARN); + StringWriter logOutput = new StringWriter(); + WriterAppender appender = WriterAppender.newBuilder().setTarget(logOutput).setName("test").build(); + LoggerContext.getContext(false).getConfiguration().addLoggerAppender((Logger) lg, appender); + appender.start(); + SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(mockSimulationDB, mockMessagingServiceInternal, mockMessagingServiceSim, mockHtcProxy); + DispatcherTestUtils.insertOrUpdateStatus(mockSimulationDB); + + mockSimulationDB.badLatestSimulation = MockSimulationDB.BadLatestSimulation.HIGHER_TASK_ID; + SimulationDispatcher.SimulationMonitor.ZombieKiller zombieKiller = simulationDispatcher.simMonitor.initialZombieKiller; + zombieKiller.run(); + Assertions.assertTrue(logOutput.toString().contains(SimulationDispatcher.SimulationMonitor.ZombieKiller.newJobFound)); + Assertions.assertEquals(1, mockHtcProxy.jobsKilledSafely.size()); + + mockSimulationDB.badLatestSimulation = MockSimulationDB.BadLatestSimulation.RETURN_NULL; + zombieKiller.run(); + Assertions.assertTrue(logOutput.toString().contains(SimulationDispatcher.SimulationMonitor.ZombieKiller.noJob)); + Assertions.assertEquals(2, mockHtcProxy.jobsKilledSafely.size()); + + mockSimulationDB.badLatestSimulation = MockSimulationDB.BadLatestSimulation.IS_DONE; + zombieKiller.run(); + Assertions.assertTrue(logOutput.toString().contains(SimulationDispatcher.SimulationMonitor.ZombieKiller.jobIsAlreadyDone)); + Assertions.assertEquals(3, mockHtcProxy.jobsKilledSafely.size()); + + appender.stop(); + logOutput.close(); + Configurator.setLevel(lg, pastLevel); + } + + + public void queueFlusherTest(){ + + } From a1c13e9910a6144c81b8f812c90eb411b92292fb Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Thu, 5 Sep 2024 10:59:17 -0400 Subject: [PATCH 17/24] Queue Flusher Test And Test Stability Improvement --- .../java/org/vcell/util/document/User.java | 8 +- .../dispatcher/SimulationDispatcher.java | 66 +++++---- .../server/dispatcher/MockHtcProxy.java | 3 +- .../server/dispatcher/MockSimulationDB.java | 33 +++-- .../dispatcher/SimulationDispatcherTest.java | 127 +++++++++++++----- 5 files changed, 168 insertions(+), 69 deletions(-) diff --git a/vcell-core/src/main/java/org/vcell/util/document/User.java b/vcell-core/src/main/java/org/vcell/util/document/User.java index 38ccdd7ecf..ca1cf79dbb 100644 --- a/vcell-core/src/main/java/org/vcell/util/document/User.java +++ b/vcell-core/src/main/java/org/vcell/util/document/User.java @@ -20,11 +20,17 @@ * This type was created in VisualAge. */ @SuppressWarnings("serial") -public class User implements java.io.Serializable, Matchable, Immutable { +public class User implements java.io.Serializable, Matchable, Immutable, Comparable { private final static String PREVIOUS_DATABASE_VALUE_ADMIN = "special0"; private final static String PREVIOUS_DATABASE_VALUE_POWERUSER = "special1"; private final static String PREVIOUS_DATABASE_VALUE_PUBLICATION = "publication"; + @Override + public int compareTo(Object o) { + User user = (User) o; + return this.getID().compareTo(user.key); + } + public enum SPECIAL_CLAIM { admins/*special0*/, powerUsers/*special1*/, diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java index f31b4e79f2..a4778057fb 100644 --- a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java @@ -75,7 +75,7 @@ public class SimulationDispatcher { /** * queue flush wait time */ - public static final long QUEUE_FLUSH_WAITIME = MessageConstants.MINUTE_IN_MS*5; + public final static long QUEUE_FLUSH_WAITIME = MessageConstants.MINUTE_IN_MS*5; private final VCMessagingService vcMessagingService_int; private final VCMessagingService vcMessagingService_sim; @@ -85,7 +85,7 @@ public class SimulationDispatcher { private final VCQueueConsumer simRequestConsumer_int; private final VCRpcMessageHandler rpcMessageHandler_int; - private final SimulationDispatcherEngine simDispatcherEngine = new SimulationDispatcherEngine(); + protected final SimulationDispatcherEngine simDispatcherEngine = new SimulationDispatcherEngine(); protected final DispatchThread dispatchThread; protected final SimulationMonitor simMonitor; @@ -141,8 +141,8 @@ public SimulationStatus startSimulation(User user, VCSimulationIdentifier vcSimu // wake up dispatcher thread if (dispatchThread!=null){ try { - synchronized (dispatchThread.notifyObject){ - dispatchThread.notifyObject.notify(); + synchronized (dispatchThread.dispatcherNotifyObject){ + dispatchThread.dispatcherNotifyObject.notify(); } }catch (IllegalMonitorStateException e){ lg.error("failed to notify dispatchThread",e); @@ -235,7 +235,8 @@ private void reloadSpecialUsers() { } protected class DispatchThread extends Thread { - final Object notifyObject = new Object(); + final Object dispatcherNotifyObject = new Object(); + final Object finishListener = new Object(); public DispatchThread() { super(); @@ -313,7 +314,7 @@ public void run() { tempSimulationMap.put(simKey, sim); } if (lg.isDebugEnabled()) { - lg.debug("dispatching simKey="+vcSimID+", jobId="+jobStatus.getJobIndex()+", taskId="+jobStatus.getTaskID()); + lg.debug("dispatching simKey={}, jobId={}, taskId={}", vcSimID, jobStatus.getJobIndex(), jobStatus.getTaskID()); } simDispatcherEngine.onDispatch(sim, jobStatus, simulationDatabase, dispatcherQueueSession_int); bDispatchedAnyJobs = true; @@ -328,14 +329,19 @@ public void run() { } catch (Exception ex) { lg.error(ex.getMessage(), ex); } + finally { + synchronized (finishListener){ + finishListener.notify(); + } + } // if there are no messages or no qualified jobs or exceptions, sleep for a few seconds while // this will be interrupted if there is a start request. if (!bDispatchedAnyJobs){ - synchronized (notifyObject) { + synchronized (dispatcherNotifyObject) { try { long waitTime = 5 * MessageConstants.SECOND_IN_MS; - notifyObject.wait(waitTime); + dispatcherNotifyObject.wait(waitTime); } catch (InterruptedException ex) { lg.debug("Dispatch thread wait interrupted", ex); } @@ -359,7 +365,7 @@ protected class SimulationMonitor implements ThreadFactory, RejectedExecutionHan * synchronizes {@link SimulationDispatcher#onWorkerEventMessage(VCMessage, VCMessageSession)} and * {@link QueueFlusher#flushWorkerEventQueue()} */ - Object notifyObject = new Object(); + final Object monitorNotifyObject = new Object(); public SimulationMonitor( ) { threadCount = 1; @@ -433,6 +439,9 @@ public void run() { * flush message queue */ protected class QueueFlusher implements Runnable { + protected final static String timeOutFailure = "failed: timed out"; + protected final static String unreferencedFailure = "failed: unreferenced simulation"; + protected final Object finishListener = new Object(); public void run() { try { traceThread(this); @@ -450,20 +459,24 @@ public void run() { abortStalledOrUnreferencedSimulationTasks(messageFlushTimeMS); } catch (Exception e1) { lg.error(e1.getMessage(), e1); + } finally { + synchronized (finishListener){ + finishListener.notify(); + } } } private void flushWorkerEventQueue() throws VCMessagingException{ VCMessage message = simMonitorThreadSession_sim.createObjectMessage(VCMongoMessage.getServiceStartupTime()); message.setStringProperty(VCMessagingConstants.MESSAGE_TYPE_PROPERTY,MessageConstants.MESSAGE_TYPE_FLUSH_VALUE); - synchronized (notifyObject) { + synchronized (monitorNotifyObject) { simMonitorThreadSession_sim.sendQueueMessage(VCellQueue.WorkerEventQueue, message, false, MessageConstants.MINUTE_IN_MS*5L); try { long startWaitTime = System.currentTimeMillis(); - notifyObject.wait(QUEUE_FLUSH_WAITIME); + monitorNotifyObject.wait(QUEUE_FLUSH_WAITIME); long endWaitTime = System.currentTimeMillis(); long elapsedFlushTime = endWaitTime-startWaitTime; - VCMongoMessage.sendInfo("flushed worker event queue: elapsedTime="+(elapsedFlushTime/1000.0)+" s"); + lg.info("flushed worker event queue: elapsedTime={} s", elapsedFlushTime / 1000.0); if (elapsedFlushTime >= QUEUE_FLUSH_WAITIME){ throw new VCMessagingException("worker event queue flush timed out (>"+QUEUE_FLUSH_WAITIME+" s), considerable message backlog?"); } @@ -515,11 +528,11 @@ private void abortStalledOrUnreferencedSimulationTasks(long messageFlushTimeMS) boolean bUnreferencedSimulation = unreferencedSimKeys.contains(activeJobStatus.getVCSimulationIdentifier().getSimulationKey()); if (bTimedOutSimulation || bUnreferencedSimulation){ - String failureMessage = (bTimedOutSimulation) ? ("failed: timed out") : ("failed: unreferenced simulation"); - lg.info("obsolete job detected at timestampMS="+currentTimeMS+", status=(" + activeJobStatus + ")"); + String failureMessage = (bTimedOutSimulation) ? timeOutFailure : unreferencedFailure; + lg.info("obsolete job detected at timestampMS={}, status={}", currentTimeMS, activeJobStatus); //SimulationStateMachine simStateMachine = simDispatcherEngine.getSimulationStateMachine(activeJobStatus.getVCSimulationIdentifier().getSimulationKey(), activeJobStatus.getJobIndex()); // lg.debug(simStateMachine.show()); - VCMongoMessage.sendObsoleteJob(activeJobStatus,failureMessage); + lg.warn("{} {}", activeJobStatus, failureMessage); simDispatcherEngine.onSystemAbort(activeJobStatus, failureMessage, simulationDatabase, clientStatusTopicSession_int); if (activeJobStatus.getSimulationExecutionStatus()!=null && activeJobStatus.getSimulationExecutionStatus().getHtcJobID()!=null){ HtcJobID htcJobId = activeJobStatus.getSimulationExecutionStatus().getHtcJobID(); @@ -549,8 +562,8 @@ public Thread newThread(Runnable r) { } public static SimulationDispatcher simulationDispatcherCreator(SimulationDatabase simulationDatabase, VCMessagingService messagingServiceInternal, - VCMessagingService messagingServiceSim, HtcProxy htcProxy){ - return new SimulationDispatcher(simulationDatabase, messagingServiceInternal, messagingServiceSim, htcProxy); + VCMessagingService messagingServiceSim, HtcProxy htcProxy, boolean startDispatcher){ + return new SimulationDispatcher(simulationDatabase, messagingServiceInternal, messagingServiceSim, htcProxy, startDispatcher); } public static SimulationDispatcher simulationDispatcherCreator() throws SQLException, DataAccessException { @@ -571,11 +584,11 @@ public static SimulationDispatcher simulationDispatcherCreator() throws SQLExcep vcMessagingServiceSim.setConfiguration(new ServerMessagingDelegate(), jmshost_sim, jmsport_sim); return SimulationDispatcher.simulationDispatcherCreator(simulationDatabase, - vcMessagingServiceInternal, vcMessagingServiceSim, SlurmProxy.createRemoteProxy()); + vcMessagingServiceInternal, vcMessagingServiceSim, SlurmProxy.createRemoteProxy(), true); } private SimulationDispatcher(SimulationDatabase simulationDatabase, VCMessagingService messagingServiceInternal, - VCMessagingService messagingServiceSim, HtcProxy htcProxy){ + VCMessagingService messagingServiceSim, HtcProxy htcProxy, boolean startDispatcher){ this.simulationDatabase = simulationDatabase; this.vcMessagingService_int = messagingServiceInternal; this.vcMessagingService_sim = messagingServiceSim; @@ -605,12 +618,17 @@ public void onQueueMessage(VCMessage vcMessage, VCMessageSession session) throws this.dispatcherQueueSession_int = this.vcMessagingService_int.createProducerSession(); this.clientStatusTopicSession_int = this.vcMessagingService_int.createProducerSession(); - this.dispatchThread = new DispatchThread(); - this.dispatchThread.start(); this.simMonitorThreadSession_sim = this.vcMessagingService_sim.createProducerSession(); - this.simMonitor = new SimulationMonitor(); this.htcProxy = htcProxy; + + // Wait until all resources are created to start separate threads + + this.simMonitor = new SimulationMonitor(); + this.dispatchThread = new DispatchThread(); + if (startDispatcher){ + this.dispatchThread.start(); + } } @@ -626,8 +644,8 @@ private void onWorkerEventMessage(VCMessage vcMessage, VCMessageSession session) if (vcMessage.propertyExists(VCMessagingConstants.MESSAGE_TYPE_PROPERTY) && vcMessage.getStringProperty(VCMessagingConstants.MESSAGE_TYPE_PROPERTY).equals(MessageConstants.MESSAGE_TYPE_FLUSH_VALUE)){ if (simMonitor!=null){ try { - synchronized (simMonitor.notifyObject){ - simMonitor.notifyObject.notify(); + synchronized (simMonitor.monitorNotifyObject){ + simMonitor.monitorNotifyObject.notify(); } }catch (IllegalMonitorStateException e){ lg.warn(e); diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockHtcProxy.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockHtcProxy.java index fa188df76d..10a419ab69 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockHtcProxy.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockHtcProxy.java @@ -26,6 +26,7 @@ public MockHtcProxy(CommandService commandService, String htcUser, MockSimulatio this.mockSimulationDB = mockSimulationDB; } public final ArrayList jobsKilledSafely = new ArrayList<>(); + public final ArrayList jobsKilledUnsafely = new ArrayList<>(); @Override public void killJobSafe(HtcJobInfo htcJobInfo) throws ExecutableException, HtcJobNotFoundException, HtcException { @@ -34,7 +35,7 @@ public void killJobSafe(HtcJobInfo htcJobInfo) throws ExecutableException, HtcJo @Override public void killJobUnsafe(HtcJobID htcJobId) throws ExecutableException, HtcJobNotFoundException, HtcException { - + jobsKilledUnsafely.add(htcJobId); } @Override diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java index 5777b2886b..aa222cc0e1 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/MockSimulationDB.java @@ -26,6 +26,8 @@ public class MockSimulationDB implements SimulationDatabase{ private final HashMap simulations = new HashMap<>(); + private final Set unreferencedSimKeys = new HashSet<>(); + // Return a latest simulation that differs in one of these ways public enum BadLatestSimulation{ HIGHER_TASK_ID, @@ -93,12 +95,16 @@ public SimulationJobStatus[] getActiveJobs(VCellServerID vcellServerID) throws D @Override public SimulationJobStatus[] queryJobs(SimpleJobStatusQuerySpec simStatusQuerySpec) throws ObjectNotFoundException, DataAccessException { - return new SimulationJobStatus[0]; + throw new ObjectNotFoundException(""); } @Override public Map getSimulationRequirements(Collection simKeys) throws SQLException { - return Map.of(); + HashMap map = new HashMap<>(); + for (KeyValue simKey : simKeys){ + map.put(simKey, new SimulationRequirements(simKey, 3)); + } + return map; } @Override @@ -132,12 +138,12 @@ public Simulation getSimulation(User user, KeyValue simKey) throws DataAccessExc @Override public FieldDataIdentifierSpec[] getFieldDataIdentifierSpecs(Simulation sim) throws DataAccessException { - return null; + return new FieldDataIdentifierSpec[0]; } @Override public Set getUnreferencedSimulations() throws SQLException { - return Set.of(); + return unreferencedSimKeys; } @Override @@ -152,7 +158,11 @@ public User.SpecialUser getUser(String username) throws DataAccessException, SQL @Override public TreeMap> getSpecialUsers() throws DataAccessException, SQLException { - return null; + TreeMap> map = new TreeMap<>(); + TreeMap subMap = new TreeMap<>(); + subMap.put(specialAdmin, "f"); + map.put(User.SPECIAL_CLAIM.admins, subMap); + return map; } @Override @@ -167,13 +177,14 @@ public SimulationStatus[] getSimulationStatus(KeyValue[] simKeys) throws ObjectN @Override public SimulationStatus getSimulationStatus(KeyValue simulationKey) throws ObjectNotFoundException, DataAccessException { -// dbTable.get(simulationKey.toString()).get(0); - return null; + SimulationJobStatus status = dbTable.get(simulationKey.toString()).get(0); + SimulationStatus simulationStatus = new SimulationStatus(new SimulationJobStatus[]{status}); + return simulationStatus; } @Override public SimpleJobStatus[] getSimpleJobStatus(User user, SimpleJobStatusQuerySpec simStatusQuerySpec) throws ObjectNotFoundException, DataAccessException { - return new SimpleJobStatus[0]; + throw new ObjectNotFoundException(""); } @@ -192,10 +203,16 @@ versionBranchPoint, new BigDecimal(22), Date.from(Instant.now()), versionFlag, " public void resetDataBase(){ dbTable = new HashMap<>(); badLatestSimulation = BadLatestSimulation.DO_NOTHING; + unreferencedSimKeys.clear(); + simulations.clear(); } public void insertSimulation(User user, Simulation sim){ simulations.put(sim.getKey().toString() + user.getName(), sim); } + public void insertUnreferencedSimKey(KeyValue k){ + unreferencedSimKeys.add(k); + } + } diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java index 13ef50b9dd..d63b4aa7f2 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java @@ -30,21 +30,29 @@ public class SimulationDispatcherTest { private MockSimulationDB mockSimulationDB = new MockSimulationDB(); private final MockMessagingService mockMessagingServiceInternal = new MockMessagingService(); private final MockMessagingService mockMessagingServiceSim = new MockMessagingService(); - private final MockHtcProxy mockHtcProxy = new MockHtcProxy(null, null, mockSimulationDB); + private final MockHtcProxy mockHtcProxy = new MockHtcProxy(null, "htcUser", mockSimulationDB); + private static StringWriter logOutPut; + private static WriterAppender appender; + private static Level pastLevel; @BeforeAll public static void setSystemProperties(){ DispatcherTestUtils.setRequiredProperties(); + + pastLevel = lg.getLevel(); + Configurator.setLevel(lg, Level.WARN); + logOutPut = new StringWriter(); + appender = WriterAppender.newBuilder().setTarget(logOutPut).setName("Simulation Dispatcher Test").build(); + LoggerContext.getContext(false).getConfiguration().addLoggerAppender((Logger) lg, appender); + appender.start(); } @AfterAll - public static void restoreSystemProperties(){ + public static void restoreSystemProperties() throws IOException { DispatcherTestUtils.restoreRequiredProperties(); - } - - @BeforeEach - public void beforeEach(){ - mockSimulationDB.resetDataBase(); + appender.stop(); + logOutPut.close(); + Configurator.setLevel(lg, pastLevel); } //################# Test Simulation Service Impl ####################### @@ -52,7 +60,8 @@ public void beforeEach(){ @Test public void onStartRequestTest() throws DataAccessException, SQLException { - SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(mockSimulationDB, mockMessagingServiceInternal, mockMessagingServiceSim, mockHtcProxy); + SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(mockSimulationDB, mockMessagingServiceInternal, + mockMessagingServiceSim, mockHtcProxy, false); SimulationStatus simStatus = simulationDispatcher.simServiceImpl.startSimulation(testUser, DispatcherTestUtils.simID, 1); SimulationJobStatus jobStatus = mockSimulationDB.getLatestSimulationJobStatus(DispatcherTestUtils.simKey, 0); Assertions.assertTrue(jobStatus.getSchedulerStatus().isWaiting()); @@ -61,7 +70,8 @@ public void onStartRequestTest() throws DataAccessException, SQLException { @Test public void onStopRequestTest() throws DataAccessException, SQLException { DispatcherTestUtils.insertOrUpdateStatus(mockSimulationDB); - SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(mockSimulationDB, mockMessagingServiceInternal, mockMessagingServiceSim, mockHtcProxy); + SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(mockSimulationDB, mockMessagingServiceInternal, + mockMessagingServiceSim, mockHtcProxy, false); SimulationStatus simStatus = simulationDispatcher.simServiceImpl.stopSimulation(testUser, DispatcherTestUtils.simID); SimulationJobStatus jobStatus = mockSimulationDB.getLatestSimulationJobStatus(DispatcherTestUtils.simKey, 0); Assertions.assertTrue(jobStatus.getSchedulerStatus().isStopped()); @@ -75,16 +85,18 @@ public void onStopRequestTest() throws DataAccessException, SQLException { @Test public void dispatcherThreadTest() throws SQLException, DataAccessException, InterruptedException, PropertyVetoException, MathException, ExpressionBindingException { DispatcherTestUtils.insertOrUpdateStatus(mockSimulationDB, SimulationJobStatus.SchedulerStatus.WAITING); - SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(mockSimulationDB, mockMessagingServiceInternal, mockMessagingServiceSim, mockHtcProxy); + SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(mockSimulationDB, mockMessagingServiceInternal, + mockMessagingServiceSim, mockHtcProxy, true); SimulationDispatcher.DispatchThread thread = simulationDispatcher.dispatchThread; - synchronized (thread.notifyObject){ - thread.notifyObject.notify(); + synchronized (thread.dispatcherNotifyObject){ + thread.dispatcherNotifyObject.notify(); } SimulationJobStatus jobStatus = mockSimulationDB.getLatestSimulationJobStatus(DispatcherTestUtils.simKey, 0); Assertions.assertTrue(jobStatus.getSchedulerStatus().isWaiting(), "Still waiting."); - // needs time for the dispatcher thread to fully update - Thread.sleep(1000); + synchronized (thread.finishListener){ + thread.finishListener.wait(); + } jobStatus = mockSimulationDB.getLatestSimulationJobStatus(DispatcherTestUtils.simKey, 0); Assertions.assertTrue(jobStatus.getSchedulerStatus().isFailed(), "Simulation gets aborted since theres no simulation in DB."); @@ -93,10 +105,12 @@ public void dispatcherThreadTest() throws SQLException, DataAccessException, Int mockSimulationDB.insertSimulation(DispatcherTestUtils.alice, mockSimulation); DispatcherTestUtils.insertOrUpdateStatus(mockSimulation.getKey(), DispatcherTestUtils.jobIndex, DispatcherTestUtils.taskID, DispatcherTestUtils.alice, SimulationJobStatus.SchedulerStatus.WAITING, mockSimulationDB); - synchronized (thread.notifyObject){ - thread.notifyObject.notify(); + synchronized (thread.dispatcherNotifyObject){ + thread.dispatcherNotifyObject.notify(); + } + synchronized (thread.finishListener){ + thread.finishListener.wait(); } - Thread.sleep(1000); jobStatus = mockSimulationDB.getLatestSimulationJobStatus(mockSimulation.getKey(), 0); Assertions.assertTrue(jobStatus.getSchedulerStatus().isDispatched(), "Dispatches"); @@ -108,43 +122,86 @@ public void dispatcherThreadTest() throws SQLException, DataAccessException, Int @Test public void zombieKillerTest() throws SQLException, DataAccessException, InterruptedException, IOException { - Level pastLevel = lg.getLevel(); - Configurator.setLevel(lg, Level.WARN); - StringWriter logOutput = new StringWriter(); - WriterAppender appender = WriterAppender.newBuilder().setTarget(logOutput).setName("test").build(); - LoggerContext.getContext(false).getConfiguration().addLoggerAppender((Logger) lg, appender); - appender.start(); - SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(mockSimulationDB, mockMessagingServiceInternal, mockMessagingServiceSim, mockHtcProxy); + SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(mockSimulationDB, mockMessagingServiceInternal, + mockMessagingServiceSim, mockHtcProxy, false); DispatcherTestUtils.insertOrUpdateStatus(mockSimulationDB); mockSimulationDB.badLatestSimulation = MockSimulationDB.BadLatestSimulation.HIGHER_TASK_ID; SimulationDispatcher.SimulationMonitor.ZombieKiller zombieKiller = simulationDispatcher.simMonitor.initialZombieKiller; zombieKiller.run(); - Assertions.assertTrue(logOutput.toString().contains(SimulationDispatcher.SimulationMonitor.ZombieKiller.newJobFound)); + Assertions.assertTrue(logOutPut.toString().contains(SimulationDispatcher.SimulationMonitor.ZombieKiller.newJobFound)); Assertions.assertEquals(1, mockHtcProxy.jobsKilledSafely.size()); mockSimulationDB.badLatestSimulation = MockSimulationDB.BadLatestSimulation.RETURN_NULL; zombieKiller.run(); - Assertions.assertTrue(logOutput.toString().contains(SimulationDispatcher.SimulationMonitor.ZombieKiller.noJob)); + Assertions.assertTrue(logOutPut.toString().contains(SimulationDispatcher.SimulationMonitor.ZombieKiller.noJob)); Assertions.assertEquals(2, mockHtcProxy.jobsKilledSafely.size()); mockSimulationDB.badLatestSimulation = MockSimulationDB.BadLatestSimulation.IS_DONE; zombieKiller.run(); - Assertions.assertTrue(logOutput.toString().contains(SimulationDispatcher.SimulationMonitor.ZombieKiller.jobIsAlreadyDone)); + Assertions.assertTrue(logOutPut.toString().contains(SimulationDispatcher.SimulationMonitor.ZombieKiller.jobIsAlreadyDone)); Assertions.assertEquals(3, mockHtcProxy.jobsKilledSafely.size()); - - appender.stop(); - logOutput.close(); - Configurator.setLevel(lg, pastLevel); } + @Test + public void queueFlusherTest() throws SQLException, DataAccessException, InterruptedException { + SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(mockSimulationDB, mockMessagingServiceInternal, + mockMessagingServiceSim, mockHtcProxy, false); + DispatcherTestUtils.insertOrUpdateStatus(mockSimulationDB); - public void queueFlusherTest(){ - - } - + SimulationDispatcher.SimulationMonitor simMonitor = simulationDispatcher.simMonitor; + SimulationDispatcher.SimulationMonitor.QueueFlusher queueFlusher = simMonitor.initialQueueFlusher; + SimulationStateMachine sm = simulationDispatcher.simDispatcherEngine.getSimulationStateMachine(DispatcherTestUtils.simKey, DispatcherTestUtils.jobIndex); + sm.setSolverProcessTimestamp(0); + Thread queueThread = new Thread(queueFlusher); + queueThread.start(); + int retries = 0; + while (queueThread.getState() != Thread.State.TIMED_WAITING){ + if (retries == 10){ + break; + } + Thread.sleep(500); + retries += 1; + } + synchronized (simMonitor.monitorNotifyObject){ + simMonitor.monitorNotifyObject.notify(); + } + synchronized (queueFlusher.finishListener){ + queueFlusher.finishListener.wait(); + } + SimulationJobStatus status = mockSimulationDB.getLatestSimulationJobStatus(DispatcherTestUtils.simKey, DispatcherTestUtils.jobIndex); + Assertions.assertTrue(status.getSchedulerStatus().isFailed()); + Assertions.assertTrue(mockHtcProxy.jobsKilledUnsafely.contains(status.getSimulationExecutionStatus().getHtcJobID())); + Assertions.assertTrue(logOutPut.toString().contains(SimulationDispatcher.SimulationMonitor.QueueFlusher.timeOutFailure)); + // reset for next test + simulationDispatcher.simDispatcherEngine.resetTimeStamps(); + mockHtcProxy.jobsKilledUnsafely.clear(); + mockSimulationDB.resetDataBase(); + mockSimulationDB.insertUnreferencedSimKey(DispatcherTestUtils.simKey); + DispatcherTestUtils.insertOrUpdateStatus(mockSimulationDB); + queueThread = new Thread(queueFlusher); + queueThread.start(); + retries = 0; + while (queueThread.getState() != Thread.State.TIMED_WAITING){ + if (retries == 10){ + break; + } + Thread.sleep(500); + retries += 1; + } + synchronized (simMonitor.monitorNotifyObject){ + simMonitor.monitorNotifyObject.notify(); + } + synchronized (queueFlusher.finishListener){ + queueFlusher.finishListener.wait(); + } + status = mockSimulationDB.getLatestSimulationJobStatus(DispatcherTestUtils.simKey, DispatcherTestUtils.jobIndex); + Assertions.assertTrue(status.getSchedulerStatus().isFailed()); + Assertions.assertTrue(mockHtcProxy.jobsKilledUnsafely.contains(status.getSimulationExecutionStatus().getHtcJobID())); + Assertions.assertTrue(logOutPut.toString().contains(SimulationDispatcher.SimulationMonitor.QueueFlusher.unreferencedFailure)); + } } From b562501e457852403bd0ef355c1a4ba031f3a8e2 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Thu, 5 Sep 2024 12:19:42 -0400 Subject: [PATCH 18/24] Add HTC Memory Limit Env Variables --- docker/build/Dockerfile-sched-dev | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docker/build/Dockerfile-sched-dev b/docker/build/Dockerfile-sched-dev index cc9827cb94..e1ae4ec969 100644 --- a/docker/build/Dockerfile-sched-dev +++ b/docker/build/Dockerfile-sched-dev @@ -68,7 +68,10 @@ ENV softwareVersion=SOFTWARE-VERSION-NOT-SET \ maxOdeJobsPerUser="max-ode-jobs-per-user-not-set" \ vcell_ssh_cmd_cmdtimeout="cmdSrvcSshCmdTimeoutMS-not-set" \ vcell_ssh_cmd_restoretimeout="cmdSrvcSshCmdRestoreTimeoutFactor-not-set" \ - maxPdeJobsPerUser="max-pde-jobs-per-user-not-set" + maxPdeJobsPerUser="max-pde-jobs-per-user-not-set" \ + htcMinMemoryMB="htc-min-memory-not-set" \ + htcMaxMemoryMB="htc-max-memory-not-set" \ + htcPowerUserMemoryFloorMB = "htc-power-user-memory-floor-not-set" ENV dbpswdfile=/run/secrets/dbpswd \ jmspswdfile=/run/secrets/jmspswd \ @@ -119,4 +122,7 @@ ENTRYPOINT java \ -Dvcell.server.maxPdeJobsPerUser=${maxPdeJobsPerUser} \ -Dvcell.ssh.cmd.cmdtimeout=${vcell_ssh_cmd_cmdtimeout} \ -Dvcell.ssh.cmd.restoretimeout=${vcell_ssh_cmd_restoretimeout} \ + -Dvcell.htc.memory.min.mb=${htcMinMemoryMB} \ + -Dvcell.htc.memory.max.mb=${htcMaxMemoryMB} \ + -Dvcell.htc.memory.pu.floor.mb=${htcPowerUserMemoryFloorMB} -cp "./lib/*" cbit.vcell.message.server.dispatcher.SimulationDispatcher From 0ca4ac89114f3844dd64ce2467e0c18efea44467 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Thu, 5 Sep 2024 12:50:17 -0400 Subject: [PATCH 19/24] Add Memory Env Variables To Slurm Test --- .../cbit/vcell/message/server/htc/slurm/SlurmProxyTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/htc/slurm/SlurmProxyTest.java b/vcell-server/src/test/java/cbit/vcell/message/server/htc/slurm/SlurmProxyTest.java index 9cf9f78d67..5a7dd61acd 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/htc/slurm/SlurmProxyTest.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/htc/slurm/SlurmProxyTest.java @@ -73,7 +73,11 @@ public void setup() setProperty(PropertyLoader.primarySimDataDirInternalProperty, "/share/apps/vcell3/users"); setProperty(PropertyLoader.vcellopt_singularity_image, "/state/partition1/singularityImages/ghcr.io_virtualcell_vcell-opt_d6825f4.img"); - } + + setProperty(PropertyLoader.htcPowerUserMemoryFloorMB, "51200"); + setProperty(PropertyLoader.htcMinMemoryMB, "4096"); + setProperty(PropertyLoader.htcMaxMemoryMB, "81920"); + } @AfterEach public void teardown() { From 206f4601197ada7650009fbe40486588903f71e3 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Thu, 5 Sep 2024 13:37:45 -0400 Subject: [PATCH 20/24] Change Expected Source of Memory Limit --- .../slurm_fixtures/adams_moulton/V_REL_274633859_0_0.slurm.sub | 2 +- .../slurm_fixtures/cvode/V_REL_274630682_0_0.slurm.sub | 2 +- .../slurm_fixtures/finite_volume/V_REL_274514696_0_0.slurm.sub | 2 +- .../slurm_fixtures/gibson/V_REL_274635122_0_0.slurm.sub | 2 +- .../gibson_milstein/V_REL_274641698_0_0.slurm.sub | 2 +- .../slurm_fixtures/langevin/V_REL_274672135_0_0.slurm.sub | 2 +- .../moving_boundary/V_REL_274641196_0_0.slurm.sub | 2 +- .../slurm_fixtures/nfsim/V_REL_274642453_0_0.slurm.sub | 2 +- .../runge_kutta_fehlberg/V_REL_274631114_0_0.slurm.sub | 2 +- .../slurm_fixtures/smoldyn/V_REL_274630052_0_0.slurm.sub | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/vcell-server/src/test/resources/slurm_fixtures/adams_moulton/V_REL_274633859_0_0.slurm.sub b/vcell-server/src/test/resources/slurm_fixtures/adams_moulton/V_REL_274633859_0_0.slurm.sub index 453e68274b..9273398495 100644 --- a/vcell-server/src/test/resources/slurm_fixtures/adams_moulton/V_REL_274633859_0_0.slurm.sub +++ b/vcell-server/src/test/resources/slurm_fixtures/adams_moulton/V_REL_274633859_0_0.slurm.sub @@ -8,7 +8,7 @@ #SBATCH --mem=4096M #SBATCH --no-kill #SBATCH --no-requeue -# VCell SlurmProxy memory limit source=Exception NoSuchFileException used FALLBACK_MEM_LIMIT_MB +# VCell SlurmProxy memory limit source=default memory limit #BEGIN---------SlurmProxy.generateScript():slurmInitSingularity---------- diff --git a/vcell-server/src/test/resources/slurm_fixtures/cvode/V_REL_274630682_0_0.slurm.sub b/vcell-server/src/test/resources/slurm_fixtures/cvode/V_REL_274630682_0_0.slurm.sub index 5c7b4d8d99..e519eb078a 100644 --- a/vcell-server/src/test/resources/slurm_fixtures/cvode/V_REL_274630682_0_0.slurm.sub +++ b/vcell-server/src/test/resources/slurm_fixtures/cvode/V_REL_274630682_0_0.slurm.sub @@ -8,7 +8,7 @@ #SBATCH --mem=4096M #SBATCH --no-kill #SBATCH --no-requeue -# VCell SlurmProxy memory limit source=Exception NoSuchFileException used FALLBACK_MEM_LIMIT_MB +# VCell SlurmProxy memory limit source=default memory limit #BEGIN---------SlurmProxy.generateScript():slurmInitSingularity---------- diff --git a/vcell-server/src/test/resources/slurm_fixtures/finite_volume/V_REL_274514696_0_0.slurm.sub b/vcell-server/src/test/resources/slurm_fixtures/finite_volume/V_REL_274514696_0_0.slurm.sub index 11882bea7e..c6d66f94f2 100644 --- a/vcell-server/src/test/resources/slurm_fixtures/finite_volume/V_REL_274514696_0_0.slurm.sub +++ b/vcell-server/src/test/resources/slurm_fixtures/finite_volume/V_REL_274514696_0_0.slurm.sub @@ -8,7 +8,7 @@ #SBATCH --mem=4096M #SBATCH --no-kill #SBATCH --no-requeue -# VCell SlurmProxy memory limit source=Exception NoSuchFileException used FALLBACK_MEM_LIMIT_MB +# VCell SlurmProxy memory limit source=default memory limit #BEGIN---------SlurmProxy.generateScript():slurmInitSingularity---------- diff --git a/vcell-server/src/test/resources/slurm_fixtures/gibson/V_REL_274635122_0_0.slurm.sub b/vcell-server/src/test/resources/slurm_fixtures/gibson/V_REL_274635122_0_0.slurm.sub index cc4a0f3f7b..3af7ec6b11 100644 --- a/vcell-server/src/test/resources/slurm_fixtures/gibson/V_REL_274635122_0_0.slurm.sub +++ b/vcell-server/src/test/resources/slurm_fixtures/gibson/V_REL_274635122_0_0.slurm.sub @@ -8,7 +8,7 @@ #SBATCH --mem=4096M #SBATCH --no-kill #SBATCH --no-requeue -# VCell SlurmProxy memory limit source=Exception NoSuchFileException used FALLBACK_MEM_LIMIT_MB +# VCell SlurmProxy memory limit source=default memory limit #BEGIN---------SlurmProxy.generateScript():slurmInitSingularity---------- diff --git a/vcell-server/src/test/resources/slurm_fixtures/gibson_milstein/V_REL_274641698_0_0.slurm.sub b/vcell-server/src/test/resources/slurm_fixtures/gibson_milstein/V_REL_274641698_0_0.slurm.sub index d512bfad88..94419e41f3 100644 --- a/vcell-server/src/test/resources/slurm_fixtures/gibson_milstein/V_REL_274641698_0_0.slurm.sub +++ b/vcell-server/src/test/resources/slurm_fixtures/gibson_milstein/V_REL_274641698_0_0.slurm.sub @@ -8,7 +8,7 @@ #SBATCH --mem=4096M #SBATCH --no-kill #SBATCH --no-requeue -# VCell SlurmProxy memory limit source=Exception NoSuchFileException used FALLBACK_MEM_LIMIT_MB +# VCell SlurmProxy memory limit source=default memory limit #BEGIN---------SlurmProxy.generateScript():slurmInitSingularity---------- diff --git a/vcell-server/src/test/resources/slurm_fixtures/langevin/V_REL_274672135_0_0.slurm.sub b/vcell-server/src/test/resources/slurm_fixtures/langevin/V_REL_274672135_0_0.slurm.sub index 73efd9a977..fff6531d9a 100644 --- a/vcell-server/src/test/resources/slurm_fixtures/langevin/V_REL_274672135_0_0.slurm.sub +++ b/vcell-server/src/test/resources/slurm_fixtures/langevin/V_REL_274672135_0_0.slurm.sub @@ -8,7 +8,7 @@ #SBATCH --mem=4096M #SBATCH --no-kill #SBATCH --no-requeue -# VCell SlurmProxy memory limit source=Exception NoSuchFileException used FALLBACK_MEM_LIMIT_MB +# VCell SlurmProxy memory limit source=default memory limit #BEGIN---------SlurmProxy.generateScript():slurmInitSingularity---------- diff --git a/vcell-server/src/test/resources/slurm_fixtures/moving_boundary/V_REL_274641196_0_0.slurm.sub b/vcell-server/src/test/resources/slurm_fixtures/moving_boundary/V_REL_274641196_0_0.slurm.sub index 6adc1eacf5..a06b208a87 100644 --- a/vcell-server/src/test/resources/slurm_fixtures/moving_boundary/V_REL_274641196_0_0.slurm.sub +++ b/vcell-server/src/test/resources/slurm_fixtures/moving_boundary/V_REL_274641196_0_0.slurm.sub @@ -8,7 +8,7 @@ #SBATCH --mem=4096M #SBATCH --no-kill #SBATCH --no-requeue -# VCell SlurmProxy memory limit source=Exception NoSuchFileException used FALLBACK_MEM_LIMIT_MB +# VCell SlurmProxy memory limit source=default memory limit #BEGIN---------SlurmProxy.generateScript():slurmInitSingularity---------- diff --git a/vcell-server/src/test/resources/slurm_fixtures/nfsim/V_REL_274642453_0_0.slurm.sub b/vcell-server/src/test/resources/slurm_fixtures/nfsim/V_REL_274642453_0_0.slurm.sub index 70dc6a5a8c..a71e433618 100644 --- a/vcell-server/src/test/resources/slurm_fixtures/nfsim/V_REL_274642453_0_0.slurm.sub +++ b/vcell-server/src/test/resources/slurm_fixtures/nfsim/V_REL_274642453_0_0.slurm.sub @@ -8,7 +8,7 @@ #SBATCH --mem=4096M #SBATCH --no-kill #SBATCH --no-requeue -# VCell SlurmProxy memory limit source=Exception NoSuchFileException used FALLBACK_MEM_LIMIT_MB +# VCell SlurmProxy memory limit source=default memory limit #BEGIN---------SlurmProxy.generateScript():slurmInitSingularity---------- diff --git a/vcell-server/src/test/resources/slurm_fixtures/runge_kutta_fehlberg/V_REL_274631114_0_0.slurm.sub b/vcell-server/src/test/resources/slurm_fixtures/runge_kutta_fehlberg/V_REL_274631114_0_0.slurm.sub index 6a87add1cc..cdc26cc1bb 100644 --- a/vcell-server/src/test/resources/slurm_fixtures/runge_kutta_fehlberg/V_REL_274631114_0_0.slurm.sub +++ b/vcell-server/src/test/resources/slurm_fixtures/runge_kutta_fehlberg/V_REL_274631114_0_0.slurm.sub @@ -8,7 +8,7 @@ #SBATCH --mem=4096M #SBATCH --no-kill #SBATCH --no-requeue -# VCell SlurmProxy memory limit source=Exception NoSuchFileException used FALLBACK_MEM_LIMIT_MB +# VCell SlurmProxy memory limit source=default memory limit #BEGIN---------SlurmProxy.generateScript():slurmInitSingularity---------- diff --git a/vcell-server/src/test/resources/slurm_fixtures/smoldyn/V_REL_274630052_0_0.slurm.sub b/vcell-server/src/test/resources/slurm_fixtures/smoldyn/V_REL_274630052_0_0.slurm.sub index 97ab4c6254..1c312eb1e9 100644 --- a/vcell-server/src/test/resources/slurm_fixtures/smoldyn/V_REL_274630052_0_0.slurm.sub +++ b/vcell-server/src/test/resources/slurm_fixtures/smoldyn/V_REL_274630052_0_0.slurm.sub @@ -8,7 +8,7 @@ #SBATCH --mem=4096M #SBATCH --no-kill #SBATCH --no-requeue -# VCell SlurmProxy memory limit source=Exception NoSuchFileException used FALLBACK_MEM_LIMIT_MB +# VCell SlurmProxy memory limit source=default memory limit #BEGIN---------SlurmProxy.generateScript():slurmInitSingularity---------- From eb0e9bc611fd61239648da974a4cbd534208a3a0 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Fri, 6 Sep 2024 10:42:26 -0400 Subject: [PATCH 21/24] Fix Zombie Test Failing It was failing due to the job prefix not being what is expected within the Zombie killer function. This was resolved by using a function that retrieves the expected job prefix dynamically instead of statically. --- .../dispatcher/SimulationDispatcher.java | 23 +++++++++++-------- .../vcell/message/server/htc/HtcProxy.java | 2 +- .../dispatcher/DispatcherTestUtils.java | 2 ++ .../dispatcher/SimulationDispatcherTest.java | 13 ++++++----- .../src/test/resources/log4j2-test.xml | 21 +++++++++++++++++ 5 files changed, 45 insertions(+), 16 deletions(-) create mode 100644 vcell-server/src/test/resources/log4j2-test.xml diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java index a4778057fb..19916c2d52 100644 --- a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java @@ -35,8 +35,13 @@ import cbit.vcell.solver.Simulation; import cbit.vcell.solver.VCSimulationIdentifier; import com.google.gson.Gson; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.WriterAppender; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; import org.vcell.db.ConnectionFactory; import org.vcell.db.DatabaseService; import org.vcell.db.KeyFactory; @@ -47,6 +52,7 @@ import org.vcell.util.document.VCellServerID; import org.vcell.util.exe.ExecutableException; +import java.io.StringWriter; import java.sql.SQLException; import java.text.SimpleDateFormat; import java.util.*; @@ -233,10 +239,10 @@ private void reloadSpecialUsers() { } lastSpecialUserCheck = System.currentTimeMillis(); } - protected class DispatchThread extends Thread { + public class DispatchThread extends Thread { final Object dispatcherNotifyObject = new Object(); - final Object finishListener = new Object(); + final Object finishListener = new Object(); //used for tests public DispatchThread() { super(); @@ -356,11 +362,11 @@ public void run() { } } - protected class SimulationMonitor implements ThreadFactory, RejectedExecutionHandler { + class SimulationMonitor implements ThreadFactory, RejectedExecutionHandler { protected final ScheduledThreadPoolExecutor executor; private int threadCount; - protected ZombieKiller initialZombieKiller = new ZombieKiller(); - protected QueueFlusher initialQueueFlusher = new QueueFlusher(); + ZombieKiller initialZombieKiller = new ZombieKiller(); + QueueFlusher initialQueueFlusher = new QueueFlusher(); /** * synchronizes {@link SimulationDispatcher#onWorkerEventMessage(VCMessage, VCMessageSession)} and * {@link QueueFlusher#flushWorkerEventQueue()} @@ -377,7 +383,7 @@ public SimulationMonitor( ) { /** * find and kill zombie processes */ - protected class ZombieKiller implements Runnable { + class ZombieKiller implements Runnable { public static final String noJob = "no jobStatus found in database for running htc job"; public static final String newJobFound = "newer task found in database for running htc job"; public static final String jobIsAlreadyDone = "jobStatus Done in database for running htc job"; @@ -385,7 +391,6 @@ protected class ZombieKiller implements Runnable { public void run() { try { traceThread(this); - Map runningJobs = htcProxy.getRunningJobs(); for (HtcJobInfo htcJobInfo : runningJobs.keySet()){ try { @@ -438,10 +443,10 @@ public void run() { /** * flush message queue */ - protected class QueueFlusher implements Runnable { + class QueueFlusher implements Runnable { protected final static String timeOutFailure = "failed: timed out"; protected final static String unreferencedFailure = "failed: unreferenced simulation"; - protected final Object finishListener = new Object(); + protected final Object finishListener = new Object(); //used for tests public void run() { try { traceThread(this); diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/htc/HtcProxy.java b/vcell-server/src/main/java/cbit/vcell/message/server/htc/HtcProxy.java index 0f9bb012ea..3cfb5ca9bd 100644 --- a/vcell-server/src/main/java/cbit/vcell/message/server/htc/HtcProxy.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/htc/HtcProxy.java @@ -219,7 +219,7 @@ public static SimTaskInfo getSimTaskInfoFromSimJobName(String simJobName) throws } public static String createHtcSimJobName(SimTaskInfo simTaskInfo) { - return HTC_SIMULATION_JOB_NAME_PREFIX+simTaskInfo.simId.toString()+"_"+simTaskInfo.jobIndex+"_"+simTaskInfo.taskId; + return simulationJobNamePrefix()+simTaskInfo.simId.toString()+"_"+simTaskInfo.jobIndex+"_"+simTaskInfo.taskId; } public static String toUnixStyleText(String javaString) throws IOException { diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/DispatcherTestUtils.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/DispatcherTestUtils.java index 7004768334..2167e1f5ee 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/DispatcherTestUtils.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/DispatcherTestUtils.java @@ -94,6 +94,8 @@ public static void setRequiredProperties(){ previousOdeJobsPerUser = PropertyLoader.getProperty(PropertyLoader.maxOdeJobsPerUser, ""); PropertyLoader.setProperty(PropertyLoader.maxOdeJobsPerUser, "100"); + + PropertyLoader.setProperty(PropertyLoader.mongodbDatabase, "fakehost"); } public static void restoreRequiredProperties(){ diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java index d63b4aa7f2..67a7cda02f 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java @@ -9,10 +9,13 @@ import cbit.vcell.server.SimulationStatus; import cbit.vcell.solver.Simulation; import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.Logger; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.appender.WriterAppender; +import org.apache.logging.log4j.core.config.Configuration; import org.apache.logging.log4j.core.config.Configurator; +import org.apache.logging.log4j.core.config.LoggerConfig; import org.apache.logging.log4j.spi.ExtendedLogger; import org.junit.jupiter.api.*; import org.vcell.util.DataAccessException; @@ -33,18 +36,16 @@ public class SimulationDispatcherTest { private final MockHtcProxy mockHtcProxy = new MockHtcProxy(null, "htcUser", mockSimulationDB); private static StringWriter logOutPut; private static WriterAppender appender; - private static Level pastLevel; @BeforeAll public static void setSystemProperties(){ DispatcherTestUtils.setRequiredProperties(); - pastLevel = lg.getLevel(); - Configurator.setLevel(lg, Level.WARN); logOutPut = new StringWriter(); appender = WriterAppender.newBuilder().setTarget(logOutPut).setName("Simulation Dispatcher Test").build(); - LoggerContext.getContext(false).getConfiguration().addLoggerAppender((Logger) lg, appender); - appender.start(); + LoggerContext context = LoggerContext.getContext(false); + Configuration configuration = context.getConfiguration(); + configuration.addLoggerAppender((Logger) lg, appender); } @AfterAll @@ -52,7 +53,6 @@ public static void restoreSystemProperties() throws IOException { DispatcherTestUtils.restoreRequiredProperties(); appender.stop(); logOutPut.close(); - Configurator.setLevel(lg, pastLevel); } //################# Test Simulation Service Impl ####################### @@ -125,6 +125,7 @@ public void zombieKillerTest() throws SQLException, DataAccessException, Interru SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(mockSimulationDB, mockMessagingServiceInternal, mockMessagingServiceSim, mockHtcProxy, false); DispatcherTestUtils.insertOrUpdateStatus(mockSimulationDB); + mockHtcProxy.jobsKilledSafely.clear(); mockSimulationDB.badLatestSimulation = MockSimulationDB.BadLatestSimulation.HIGHER_TASK_ID; SimulationDispatcher.SimulationMonitor.ZombieKiller zombieKiller = simulationDispatcher.simMonitor.initialZombieKiller; diff --git a/vcell-server/src/test/resources/log4j2-test.xml b/vcell-server/src/test/resources/log4j2-test.xml new file mode 100644 index 0000000000..7d78860d47 --- /dev/null +++ b/vcell-server/src/test/resources/log4j2-test.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + From c7ad036522570067008ad245dcbe3ace4b201aeb Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Fri, 6 Sep 2024 10:45:39 -0400 Subject: [PATCH 22/24] Revert Unnecessary Changes --- .../SimulationDispatcherEngine.java | 39 +++++----- .../dispatcher/SimulationStateMachine.java | 72 ++++++++++--------- 2 files changed, 60 insertions(+), 51 deletions(-) diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherEngine.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherEngine.java index 8bcde2a22d..f4ac1d817b 100644 --- a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherEngine.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherEngine.java @@ -4,6 +4,8 @@ import cbit.vcell.message.VCMessageSession; import cbit.vcell.message.VCMessagingException; import cbit.vcell.message.messages.StatusMessage; +import cbit.vcell.server.SimulationJobStatus; +import cbit.vcell.server.SimulationJobStatus.SchedulerStatus; import cbit.vcell.server.UpdateSynchronizationException; import cbit.vcell.solver.Simulation; import cbit.vcell.solver.SimulationInfo; @@ -23,7 +25,7 @@ import java.util.List; public class SimulationDispatcherEngine { - public static final Logger lg = LogManager.getLogger(cbit.vcell.message.server.dispatcher.SimulationDispatcherEngine.class); + public static final Logger lg = LogManager.getLogger(SimulationDispatcherEngine.class); private HashMap> simStateMachineHash = new HashMap>(); @@ -58,7 +60,7 @@ public SimulationStateMachine getSimulationStateMachine(KeyValue simulationKey, return newStateMachine; } - public void onDispatch(Simulation simulation, cbit.vcell.server.SimulationJobStatus simJobStatus, SimulationDatabase simulationDatabase, VCMessageSession dispatcherQueueSession) throws VCMessagingException, DataAccessException, SQLException { + public void onDispatch(Simulation simulation, SimulationJobStatus simJobStatus, SimulationDatabase simulationDatabase, VCMessageSession dispatcherQueueSession) throws VCMessagingException, DataAccessException, SQLException { KeyValue simulationKey = simJobStatus.getVCSimulationIdentifier().getSimulationKey(); SimulationStateMachine simStateMachine = getSimulationStateMachine(simulationKey, simJobStatus.getJobIndex()); @@ -72,30 +74,35 @@ public ArrayList onStartRequest(VCSimulationIdentifier vcSimID, U boolean isAdmin = Arrays.asList(myUser.getMySpecials()).contains(User.SPECIAL_CLAIM.admins); SimulationInfo simulationInfo = null; - cbit.vcell.server.SimulationJobStatus simJobStatus = null; + SimulationJobStatus simJobStatus = null; ArrayList status = new ArrayList<>(); try { simulationInfo = simulationDatabase.getSimulationInfo(user, simKey); } catch (DataAccessException ex) { if (lg.isWarnEnabled()) lg.warn("Bad simulation " + vcSimID); - simJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, - cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Failed to dispatch simulation: "+ ex.getMessage()), null, null); + simJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, + SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Failed to dispatch simulation: "+ ex.getMessage()), null, null); status.add(new StatusMessage(simJobStatus, user.getName(), null, null)); return status; } if (simulationInfo == null) { if (lg.isWarnEnabled()) lg.warn("Can't start, simulation [" + vcSimID + "] doesn't exist in database"); - simJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, - cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Can't start, simulation [" + vcSimID + "] doesn't exist"), null, null); - status.add(new StatusMessage(simJobStatus, user.getName(), null, null)); + simJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, + SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Can't start, simulation [" + vcSimID + "] doesn't exist"), null, null); + StatusMessage message = new StatusMessage(simJobStatus, user.getName(), null, null); + message.sendToClient(session); + status.add(message); return status; } if (!isAdmin && simulationScanCount > Integer.parseInt(cbit.vcell.resource.PropertyLoader.getRequiredProperty(cbit.vcell.resource.PropertyLoader.maxJobsPerScan))) { if (lg.isWarnEnabled()) lg.warn("Too many simulations (" + simulationScanCount + ") for parameter scan." + vcSimID); - simJobStatus = new cbit.vcell.server.SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, - cbit.vcell.server.SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Too many simulations (" + simulationScanCount + ") for parameter scan."), null, null); - status.add(new StatusMessage(simJobStatus, user.getName(), null, null)); + simJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, -1, null, + SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("Too many simulations (" + simulationScanCount + ") for parameter scan."), null, null); + StatusMessage message = new StatusMessage(simJobStatus, user.getName(), null, null); + message.sendToClient(session); + status.add(message); + return status; } @@ -114,15 +121,15 @@ public ArrayList onStartRequest(VCSimulationIdentifier vcSimID, U public ArrayList onStopRequest(VCSimulationIdentifier vcSimID, User user, SimulationDatabase simulationDatabase, VCMessageSession session) throws DataAccessException, VCMessagingException, SQLException { KeyValue simKey = vcSimID.getSimulationKey(); - cbit.vcell.server.SimulationJobStatus[] allActiveSimJobStatusArray = simulationDatabase.getActiveJobs(VCellServerID.getSystemServerID()); - ArrayList simJobStatusArray = new ArrayList(); - for (cbit.vcell.server.SimulationJobStatus activeSimJobStatus : allActiveSimJobStatusArray){ + SimulationJobStatus[] allActiveSimJobStatusArray = simulationDatabase.getActiveJobs(VCellServerID.getSystemServerID()); + ArrayList simJobStatusArray = new ArrayList(); + for (SimulationJobStatus activeSimJobStatus : allActiveSimJobStatusArray){ if (activeSimJobStatus.getVCSimulationIdentifier().getSimulationKey().equals(vcSimID.getSimulationKey())){ simJobStatusArray.add(activeSimJobStatus); } } ArrayList stoppedSimulations = new ArrayList<>(); - for (cbit.vcell.server.SimulationJobStatus simJobStatus : simJobStatusArray){ + for (SimulationJobStatus simJobStatus : simJobStatusArray){ SimulationStateMachine simStateMachine = getSimulationStateMachine(simKey, simJobStatus.getJobIndex()); try { stoppedSimulations.add(simStateMachine.onStopRequest(user, simJobStatus, simulationDatabase, session)); @@ -146,7 +153,7 @@ public void onWorkerEvent(WorkerEvent workerEvent, SimulationDatabase simulation } - public void onSystemAbort(cbit.vcell.server.SimulationJobStatus jobStatus, String failureMessage, SimulationDatabase simulationDatabase, VCMessageSession session) { + public void onSystemAbort(SimulationJobStatus jobStatus, String failureMessage, SimulationDatabase simulationDatabase, VCMessageSession session) { try { KeyValue simKey = jobStatus.getVCSimulationIdentifier().getSimulationKey(); int jobIndex = jobStatus.getJobIndex(); diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java index 989e973378..53155b9533 100644 --- a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationStateMachine.java @@ -16,6 +16,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vcell.util.DataAccessException; +import cbit.vcell.server.SimulationJobStatus.SchedulerStatus; +import cbit.vcell.server.SimulationJobStatus.SimulationQueueID; import org.vcell.util.document.KeyValue; import org.vcell.util.document.User; import org.vcell.util.document.VCellServerID; @@ -25,7 +27,7 @@ import java.util.Date; public class SimulationStateMachine { - public static final Logger lg = LogManager.getLogger(cbit.vcell.message.server.dispatcher.SimulationStateMachine.class); + public static final Logger lg = LogManager.getLogger(SimulationStateMachine.class); // bitmapped counter so that allows 3 retries for each request (but preserves ordinal nature) // bits 0-3: retry count @@ -189,7 +191,7 @@ private SimulationJobStatus produceStateFromWorkerEvent( // if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued()) { // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationQueueID.QUEUE_ID_NULL); // new exe status lastUpdateDate = new Date(); @@ -197,7 +199,7 @@ private SimulationJobStatus produceStateFromWorkerEvent( endDate = null; SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.DISPATCHED, + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.DISPATCHED, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); } @@ -205,7 +207,7 @@ private SimulationJobStatus produceStateFromWorkerEvent( // only update database when the job event changes from started to running. The later progress event will not be recorded. if ( oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()) { // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationQueueID.QUEUE_ID_NULL); // new exe status lastUpdateDate = new Date(); @@ -214,17 +216,17 @@ private SimulationJobStatus produceStateFromWorkerEvent( } SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.RUNNING, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); } } else if (workerEvent.isNewDataEvent()) { if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ - if (!oldSchedulerStatus.isRunning() || simQueueID != SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL || hasData==false){ + if (!oldSchedulerStatus.isRunning() || simQueueID != SimulationQueueID.QUEUE_ID_NULL || hasData==false){ // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationQueueID.QUEUE_ID_NULL); // new exe status if (startDate == null){ @@ -233,7 +235,7 @@ private SimulationJobStatus produceStateFromWorkerEvent( hasData = true; SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.RUNNING, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); } } @@ -242,9 +244,9 @@ private SimulationJobStatus produceStateFromWorkerEvent( if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ - if (!oldSchedulerStatus.isRunning() || simQueueID != SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL){ + if (!oldSchedulerStatus.isRunning() || simQueueID != SimulationQueueID.QUEUE_ID_NULL){ // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationQueueID.QUEUE_ID_NULL); // new exe status if (startDate == null){ @@ -252,16 +254,16 @@ private SimulationJobStatus produceStateFromWorkerEvent( } SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.RUNNING, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); }else if (oldSchedulerStatus.isRunning()){ if (oldSimExeStatus != null) { // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationQueueID.QUEUE_ID_NULL); SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.RUNNING, + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.RUNNING, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); } } @@ -270,38 +272,38 @@ private SimulationJobStatus produceStateFromWorkerEvent( } else if (workerEvent.isCompletedEvent()) { if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationQueueID.QUEUE_ID_NULL); // new exe status endDate = new Date(); hasData = true; SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.COMPLETED, + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.COMPLETED, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); } } else if (workerEvent.isFailedEvent()) { if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationQueueID.QUEUE_ID_NULL); // new exe status endDate = new Date(); SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.FAILED, + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.FAILED, taskID, workerEventSimulationMessage, newQueueStatus, newExeStatus); } } else if (workerEvent.isWorkerExitErrorEvent()) { if (oldSchedulerStatus.isWaiting() || oldSchedulerStatus.isQueued() || oldSchedulerStatus.isDispatched() || oldSchedulerStatus.isRunning()){ // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(queueDate, queuePriority, SimulationQueueID.QUEUE_ID_NULL); // new exe status endDate = new Date(); SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(startDate, computeHost, lastUpdateDate, endDate, hasData, htcJobID); SimulationMessage simulationMessage = SimulationMessage.workerFailure("solver stopped unexpectedly, "+workerEventSimulationMessage.getDisplayMessage()); - newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.FAILED, + newJobStatus = new SimulationJobStatus(vcServerID, vcSimDataID.getVcSimID(), jobIndex, submitDate, SchedulerStatus.FAILED, taskID, simulationMessage, newQueueStatus, newExeStatus); } @@ -328,7 +330,7 @@ public synchronized void onWorkerEvent(WorkerEvent workerEvent, SimulationDataba KeyValue simKey = vcSimDataID.getSimulationKey(); SimulationJobStatus oldSimulationJobStatus = simulationDatabase.getLatestSimulationJobStatus(simKey, jobIndex); - SimulationJobStatus.SchedulerStatus oldSchedulerStatus = oldSimulationJobStatus.getSchedulerStatus(); + SchedulerStatus oldSchedulerStatus = oldSimulationJobStatus.getSchedulerStatus(); SimulationJobStatus newJobStatus = produceStateFromWorkerEvent(workerEvent, oldSimulationJobStatus); if (newJobStatus!=null){ @@ -375,7 +377,7 @@ public synchronized StatusMessage onStartRequest(User user, VCSimulationIdentifi if (!user.equals(vcSimID.getOwner())) { lg.error(user + " is not authorized to start simulation (key=" + simKey + ")"); SimulationJobStatus simulationJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(), vcSimID, 0, null, - SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to start this simulation!"), null, null); + SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to start this simulation!"), null, null); statusMessage = new StatusMessage(simulationJobStatus, user.getName(), null, null); VCMongoMessage.sendInfo("onStartRequest("+vcSimID.getID()+") ignoring start simulation request - wrong user): simID="+vcSimID); statusMessage.sendToClient(session); @@ -414,7 +416,7 @@ public static SimulationJobStatus saveSimulationStartRequest(VCSimulationIdentif Date currentDate = new Date(); // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationJobStatus.SimulationQueueID.QUEUE_ID_WAITING); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationQueueID.QUEUE_ID_WAITING); // new exe status Date lastUpdateDate = new Date(); @@ -425,7 +427,7 @@ public static SimulationJobStatus saveSimulationStartRequest(VCSimulationIdentif VCellServerID vcServerID = VCellServerID.getSystemServerID(); Date submitDate = currentDate; - SimulationJobStatus newJobStatus = new SimulationJobStatus(vcServerID, vcSimID, jobIndex, submitDate, SimulationJobStatus.SchedulerStatus.WAITING, + SimulationJobStatus newJobStatus = new SimulationJobStatus(vcServerID, vcSimID, jobIndex, submitDate, SchedulerStatus.WAITING, newTaskID, SimulationMessage.MESSAGE_JOB_WAITING, newQueueStatus, newExeStatus); simulationDatabase.insertSimulationJobStatus(newJobStatus); @@ -467,10 +469,10 @@ public synchronized void onDispatch(Simulation simulation, SimulationJobStatus o // Date currentDate = new Date(); // new queue status - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationQueueID.QUEUE_ID_NULL); SimulationExecutionStatus newSimExeStatus = new SimulationExecutionStatus(null, null, new Date(), null, false, null); newSimJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(),vcSimID,jobIndex, - oldSimulationJobStatus.getSubmitDate(), SimulationJobStatus.SchedulerStatus.FAILED,taskID, + oldSimulationJobStatus.getSubmitDate(), SchedulerStatus.FAILED,taskID, SimulationMessage.jobFailed("simulation required "+estimatedMemMB+"MB of memory, only "+htcMaxMemoryMB+"MB allowed"), newQueueStatus,newSimExeStatus); @@ -484,10 +486,10 @@ public synchronized void onDispatch(Simulation simulation, SimulationJobStatus o // dispatch the simulation, new queue status // Date currentDate = new Date(); - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationJobStatus.SimulationQueueID.QUEUE_ID_SIMULATIONJOB); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentDate, PRIORITY_DEFAULT, SimulationQueueID.QUEUE_ID_SIMULATIONJOB); SimulationExecutionStatus newSimExeStatus = new SimulationExecutionStatus(null, null, new Date(), null, false, null); newSimJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(),vcSimID,jobIndex, - oldSimulationJobStatus.getSubmitDate(), SimulationJobStatus.SchedulerStatus.DISPATCHED,taskID, + oldSimulationJobStatus.getSubmitDate(), SchedulerStatus.DISPATCHED,taskID, SimulationMessage.MESSAGE_JOB_DISPATCHED, newQueueStatus,newSimExeStatus); @@ -496,7 +498,7 @@ public synchronized void onDispatch(Simulation simulation, SimulationJobStatus o simulationDatabase.updateSimulationJobStatus(newSimJobStatus); - cbit.vcell.message.messages.StatusMessage message = new cbit.vcell.message.messages.StatusMessage(newSimJobStatus, simulation.getVersion().getOwner().getName(), null, null); + StatusMessage message = new StatusMessage(newSimJobStatus, simulation.getVersion().getOwner().getName(), null, null); message.sendToClient(session); } @@ -511,7 +513,7 @@ public synchronized StatusMessage onStopRequest(User user, SimulationJobStatus s if (!user.equals(simJobStatus.getVCSimulationIdentifier().getOwner())) { lg.error(user + " is not authorized to stop simulation (key=" + simKey + ")"); SimulationJobStatus simulationJobStatus = new SimulationJobStatus(VCellServerID.getSystemServerID(), simJobStatus.getVCSimulationIdentifier(), 0, null, - SimulationJobStatus.SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to stop this simulation!"), null, null); + SchedulerStatus.FAILED, 0, SimulationMessage.workerFailure("You are not authorized to stop this simulation!"), null, null); VCMongoMessage.sendInfo("onStopRequest("+simJobStatus.getVCSimulationIdentifier()+") ignoring stop simulation request - wrong user)"); statusMessage = new StatusMessage(simulationJobStatus, user.getName(), null, null); @@ -520,20 +522,20 @@ public synchronized StatusMessage onStopRequest(User user, SimulationJobStatus s } // stop latest task if active - SimulationJobStatus.SchedulerStatus schedulerStatus = simJobStatus.getSchedulerStatus(); + SchedulerStatus schedulerStatus = simJobStatus.getSchedulerStatus(); int taskID = simJobStatus.getTaskID(); if (schedulerStatus.isActive()){ SimulationQueueEntryStatus simQueueEntryStatus = simJobStatus.getSimulationQueueEntryStatus(); SimulationExecutionStatus simExeStatus = simJobStatus.getSimulationExecutionStatus(); SimulationJobStatus newJobStatus = new SimulationJobStatus(simJobStatus.getServerID(),simJobStatus.getVCSimulationIdentifier(),jobIndex,simJobStatus.getSubmitDate(), - SimulationJobStatus.SchedulerStatus.STOPPED,taskID,SimulationMessage.solverStopped("simulation stopped by user"),simQueueEntryStatus,simExeStatus); + SchedulerStatus.STOPPED,taskID,SimulationMessage.solverStopped("simulation stopped by user"),simQueueEntryStatus,simExeStatus); if (lg.isTraceEnabled()) lg.trace("send " + MessageConstants.MESSAGE_TYPE_STOPSIMULATION_VALUE + " to " + VCellTopic.ServiceControlTopic.getName() + " topic"); SimulationJobStatus simulationJobStatusRecord = new SimulationJobStatus( null, new VCSimulationIdentifier(simKey, user), jobIndex, simJobStatus.getSubmitDate(), - SimulationJobStatus.SchedulerStatus.STOPPED, taskID, simJobStatus.getSimulationMessage(), simQueueEntryStatus, simExeStatus + SchedulerStatus.STOPPED, taskID, simJobStatus.getSimulationMessage(), simQueueEntryStatus, simExeStatus ); // @@ -569,21 +571,21 @@ public synchronized void onSystemAbort(SimulationJobStatus oldJobStatus, String // CurrentState currentState = new CurrentState(oldJobStatus.getSimulationExecutionStatus(), oldJobStatus.getSimulationQueueEntryStatus(), oldJobStatus); - SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentState.queueDate, currentState.queuePriority, SimulationJobStatus.SimulationQueueID.QUEUE_ID_NULL); + SimulationQueueEntryStatus newQueueStatus = new SimulationQueueEntryStatus(currentState.queueDate, currentState.queuePriority, SimulationQueueID.QUEUE_ID_NULL); Date endDate = new Date(); Date lastUpdateDate = new Date(); SimulationExecutionStatus newExeStatus = new SimulationExecutionStatus(currentState.startDate, currentState.computeHost, lastUpdateDate, endDate, currentState.hasData, currentState.htcJobID); - SimulationJobStatus newJobStatus = new SimulationJobStatus(currentState.vcServerID, oldJobStatus.getVCSimulationIdentifier(), jobIndex, currentState.submitDate, SimulationJobStatus.SchedulerStatus.FAILED, + SimulationJobStatus newJobStatus = new SimulationJobStatus(currentState.vcServerID, oldJobStatus.getVCSimulationIdentifier(), jobIndex, currentState.submitDate, SchedulerStatus.FAILED, taskID, SimulationMessage.jobFailed(failureMessage), newQueueStatus, newExeStatus); simulationDatabase.updateSimulationJobStatus(newJobStatus); // addStateMachineTransition(new StateMachineTransition(new AbortStateMachineEvent(taskID, failureMessage), oldJobStatus, newJobStatus)); String userName = VCMessagingConstants.USERNAME_PROPERTY_VALUE_ALL; - cbit.vcell.message.messages.StatusMessage msgForClient = new cbit.vcell.message.messages.StatusMessage(newJobStatus, userName, null, null); + StatusMessage msgForClient = new StatusMessage(newJobStatus, userName, null, null); msgForClient.sendToClient(session); if (lg.isTraceEnabled()) lg.trace("Send status to client: " + msgForClient); } From b960cceb6c5d551c62c7979e78e3861a62cb1ff1 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Fri, 6 Sep 2024 11:16:54 -0400 Subject: [PATCH 23/24] Working Docker Image --- docker/build/Dockerfile-sched-dev | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/build/Dockerfile-sched-dev b/docker/build/Dockerfile-sched-dev index e1ae4ec969..47f831de56 100644 --- a/docker/build/Dockerfile-sched-dev +++ b/docker/build/Dockerfile-sched-dev @@ -71,7 +71,7 @@ ENV softwareVersion=SOFTWARE-VERSION-NOT-SET \ maxPdeJobsPerUser="max-pde-jobs-per-user-not-set" \ htcMinMemoryMB="htc-min-memory-not-set" \ htcMaxMemoryMB="htc-max-memory-not-set" \ - htcPowerUserMemoryFloorMB = "htc-power-user-memory-floor-not-set" + htcPowerUserMemoryFloorMB="htc-power-user-memory-floor-not-set" ENV dbpswdfile=/run/secrets/dbpswd \ jmspswdfile=/run/secrets/jmspswd \ @@ -124,5 +124,5 @@ ENTRYPOINT java \ -Dvcell.ssh.cmd.restoretimeout=${vcell_ssh_cmd_restoretimeout} \ -Dvcell.htc.memory.min.mb=${htcMinMemoryMB} \ -Dvcell.htc.memory.max.mb=${htcMaxMemoryMB} \ - -Dvcell.htc.memory.pu.floor.mb=${htcPowerUserMemoryFloorMB} - -cp "./lib/*" cbit.vcell.message.server.dispatcher.SimulationDispatcher + -Dvcell.htc.memory.pu.floor.mb=${htcPowerUserMemoryFloorMB} \ + -cp "./lib/*" cbit.vcell.message.server.dispatcher.SimulationDispatcherMain From 0d6b08ea9943261978eb9a05bcd60f483cc23304 Mon Sep 17 00:00:00 2001 From: Ezequiel Valencia Date: Fri, 6 Sep 2024 11:19:23 -0400 Subject: [PATCH 24/24] Delay Automatic Run of Zombie Thread --- .../message/server/dispatcher/SimulationDispatcher.java | 6 ++++-- .../message/server/dispatcher/SimulationDispatcherTest.java | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java index 19916c2d52..d612019a4d 100644 --- a/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java +++ b/vcell-server/src/main/java/cbit/vcell/message/server/dispatcher/SimulationDispatcher.java @@ -73,7 +73,9 @@ public class SimulationDispatcher { /** * minutes between zombie kill runs */ - public static final int ZOMBIE_MINUTES = 1; + public static final int ZOMBIE_MINUTES = 1; + // changed only for testing + static int INITIAL_ZOMBIE_DELAY = 0; /** * minutes between queue flushing */ @@ -376,7 +378,7 @@ class SimulationMonitor implements ThreadFactory, RejectedExecutionHandler { public SimulationMonitor( ) { threadCount = 1; executor = new ScheduledThreadPoolExecutor(2,this,this); - executor.scheduleAtFixedRate(initialZombieKiller, 0, ZOMBIE_MINUTES, TimeUnit.MINUTES); + executor.scheduleAtFixedRate(initialZombieKiller, INITIAL_ZOMBIE_DELAY, ZOMBIE_MINUTES, TimeUnit.MINUTES); executor.scheduleAtFixedRate(initialQueueFlusher, 1,FLUSH_QUEUE_MINUTES,TimeUnit.MINUTES); } diff --git a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java index 67a7cda02f..dc352ad062 100644 --- a/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java +++ b/vcell-server/src/test/java/cbit/vcell/message/server/dispatcher/SimulationDispatcherTest.java @@ -122,6 +122,7 @@ public void dispatcherThreadTest() throws SQLException, DataAccessException, Int @Test public void zombieKillerTest() throws SQLException, DataAccessException, InterruptedException, IOException { + SimulationDispatcher.INITIAL_ZOMBIE_DELAY = 10; SimulationDispatcher simulationDispatcher = SimulationDispatcher.simulationDispatcherCreator(mockSimulationDB, mockMessagingServiceInternal, mockMessagingServiceSim, mockHtcProxy, false); DispatcherTestUtils.insertOrUpdateStatus(mockSimulationDB);