refactor(app,api): Add an "awaiting-recovery" run status (#14651)

api-client/src/maintenance_runs/types.ts

@@ -4,35 +4,16 @@ import type {
   LoadedModule,
   LoadedPipette,
 } from '@opentrons/shared-data'
-import type { RunCommandSummary, LabwareOffsetCreateData } from '../runs'
-
-export const ENGINE_STATUS_IDLE = 'idle' as const
-export const ENGINE_STATUS_RUNNING = 'running' as const
-export const ENGINE_STATUS_PAUSE_REQUESTED = 'pause-requested' as const
-export const ENGINE_STATUS_PAUSED = 'paused'
-export const ENGINE_STATUS_STOP_REQUESTED = 'stop-requested' as const
-export const ENGINE_STATUS_STOPPED = 'stopped' as const
-export const ENGINE_STATUS_FAILED = 'failed' as const
-export const ENGINE_STATUS_FINISHING = 'finishing' as const
-export const ENGINE_STATUS_SUCCEEDED = 'succeeded' as const
-export const ENGINE_STATUS_BLOCKED_BY_OPEN_DOOR = 'blocked-by-open-door' as const
-
-export type EngineStatus =
-  | typeof ENGINE_STATUS_IDLE
-  | typeof ENGINE_STATUS_RUNNING
-  | typeof ENGINE_STATUS_PAUSE_REQUESTED
-  | typeof ENGINE_STATUS_PAUSED
-  | typeof ENGINE_STATUS_STOP_REQUESTED
-  | typeof ENGINE_STATUS_STOPPED
-  | typeof ENGINE_STATUS_FAILED
-  | typeof ENGINE_STATUS_FINISHING
-  | typeof ENGINE_STATUS_SUCCEEDED
-  | typeof ENGINE_STATUS_BLOCKED_BY_OPEN_DOOR
+import type {
+  RunCommandSummary,
+  LabwareOffsetCreateData,
+  RunStatus,
+} from '../runs'
 
 export interface MaintenanceRunData {
   id: string
   createdAt: string
-  status: EngineStatus
+  status: RunStatus
   current: boolean
   actions: MaintenanceRunAction[]
   errors: MaintenanceRunError[]

api-client/src/runs/types.ts

@@ -18,6 +18,7 @@ export const RUN_STATUS_FAILED = 'failed' as const
 export const RUN_STATUS_FINISHING = 'finishing' as const
 export const RUN_STATUS_SUCCEEDED = 'succeeded' as const
 export const RUN_STATUS_BLOCKED_BY_OPEN_DOOR = 'blocked-by-open-door' as const
+export const RUN_STATUS_AWAITING_RECOVERY = 'awaiting-recovery' as const
 
 export type RunStatus =
   | typeof RUN_STATUS_IDLE
@@ -30,6 +31,7 @@ export type RunStatus =
   | typeof RUN_STATUS_FINISHING
   | typeof RUN_STATUS_SUCCEEDED
   | typeof RUN_STATUS_BLOCKED_BY_OPEN_DOOR
+  | typeof RUN_STATUS_AWAITING_RECOVERY
 
 export interface RunData {
   id: string

api/src/opentrons/protocol_engine/state/commands.py

@@ -691,6 +691,13 @@ def validate_action_allowed(
                     "Setup commands are not allowed after run has started."
                 )
 
+        elif self.get_status() == EngineStatus.AWAITING_RECOVERY:
+            # While we're developing error recovery, we'll conservatively disallow
+            # all actions, to avoid putting the engine in weird undefined states.
+            # We'll allow specific actions here as we flesh things out and add support
+            # for them.
+            raise NotImplementedError()
+
         return action
 
     def get_status(self) -> EngineStatus:

api/src/opentrons/protocol_engine/types.py

@@ -35,6 +35,14 @@ class EngineStatus(str, Enum):
     FAILED = "failed"
     SUCCEEDED = "succeeded"
 
+    AWAITING_RECOVERY = "awaiting-recovery"
+    """The engine is waiting for external input to recover from a nonfatal error.
+
+    New fixup commands may be enqueued, which will run immediately.
+    The run can't be paused in this state, but it can be canceled, or resumed from the
+    next protocol command if recovery is complete.
+    """
+
 
 class DeckSlotLocation(BaseModel):
     """The location of something placed in a single deck slot."""

app/src/organisms/Devices/ProtocolRun/ProtocolRunHeader.tsx

@@ -14,7 +14,7 @@ import {
   RUN_STATUS_FINISHING,
   RUN_STATUS_SUCCEEDED,
   RUN_STATUS_BLOCKED_BY_OPEN_DOOR,
-  RunStatus,
+  RUN_STATUS_AWAITING_RECOVERY,
 } from '@opentrons/api-client'
 import {
   useModulesQuery,
@@ -109,7 +109,7 @@ import { useMostRecentCompletedAnalysis } from '../../LabwarePositionCheck/useMo
 import { useMostRecentRunId } from '../../ProtocolUpload/hooks/useMostRecentRunId'
 import { useNotifyRunQuery } from '../../../resources/runs'
 
-import type { Run, RunError } from '@opentrons/api-client'
+import type { Run, RunError, RunStatus } from '@opentrons/api-client'
 import type { State } from '../../../redux/types'
 import type { HeaterShakerModule } from '../../../redux/modules/types'
 import type { PipetteModelSpecs } from '@opentrons/shared-data'
@@ -126,6 +126,7 @@ const CANCELLABLE_STATUSES = [
   RUN_STATUS_PAUSE_REQUESTED,
   RUN_STATUS_BLOCKED_BY_OPEN_DOOR,
   RUN_STATUS_IDLE,
+  RUN_STATUS_AWAITING_RECOVERY,
 ]
 const RUN_OVER_STATUSES: RunStatus[] = [
   RUN_STATUS_FAILED,

app/src/organisms/Devices/hooks/useLastRunCommandKey.ts

@@ -1,6 +1,7 @@
 import { useAllCommandsQuery } from '@opentrons/react-api-client'
 import { useRunStatus } from '../../RunTimeControl/hooks'
 import {
+  RUN_STATUS_AWAITING_RECOVERY,
   RUN_STATUS_BLOCKED_BY_OPEN_DOOR,
   RUN_STATUS_FINISHING,
   RUN_STATUS_IDLE,
@@ -21,6 +22,7 @@ const LIVE_RUN_STATUSES = [
   RUN_STATUS_RUNNING,
   RUN_STATUS_FINISHING,
   RUN_STATUS_BLOCKED_BY_OPEN_DOOR,
+  RUN_STATUS_AWAITING_RECOVERY,
 ]
 const LIVE_RUN_COMMANDS_POLL_MS = 3000
 

app/src/organisms/Devices/hooks/useRunStatuses.ts

@@ -1,4 +1,5 @@
 import {
+  RUN_STATUS_AWAITING_RECOVERY,
   RUN_STATUS_FAILED,
   RUN_STATUS_IDLE,
   RUN_STATUS_PAUSED,
@@ -21,7 +22,12 @@ export function useRunStatuses(): RunStatusesInfo {
   const runStatus = useRunStatus(currentRunId)
   const isRunIdle = runStatus === RUN_STATUS_IDLE
   const isRunRunning =
-    runStatus === RUN_STATUS_PAUSED || runStatus === RUN_STATUS_RUNNING
+    // todo(mm, 2024-03-13): Does this intentionally exclude
+    // RUN_STATUS_FINISHING, RUN_STATUS_STOP_REQUESTED,
+    // and RUN_STATUS_BLOCKED_BY_OPEN_DOOR?
+    runStatus === RUN_STATUS_PAUSED ||
+    runStatus === RUN_STATUS_RUNNING ||
+    runStatus === RUN_STATUS_AWAITING_RECOVERY
   const isRunTerminal =
     runStatus === RUN_STATUS_SUCCEEDED ||
     runStatus === RUN_STATUS_STOPPED ||