Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(api, robot-server): get historical and current command errors #16697

Open
wants to merge 59 commits into
base: edge
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
59 commits
Select commit Hold shift + click to select a range
282c99c
WIP intial schema and orchestrator changes
TamarZanzouri Nov 1, 2024
1bb8d7c
changed schema and imports to 8
TamarZanzouri Nov 1, 2024
77902c6
insert the errors to the DB
TamarZanzouri Nov 4, 2024
15db246
refactor into commands table
TamarZanzouri Nov 4, 2024
b88e722
select from commands
TamarZanzouri Nov 4, 2024
b6d33b6
linting and command error slice done
TamarZanzouri Nov 5, 2024
85eda17
added tests and fixed bugs
TamarZanzouri Nov 5, 2024
5bc0500
changed logic for current command errors
TamarZanzouri Nov 5, 2024
e423f37
removed failed_command_errors from state
TamarZanzouri Nov 5, 2024
e29d95f
refactor get command errors
TamarZanzouri Nov 5, 2024
c275c35
Merge branch 'edge' into EXEC-655-store-commands-error-list-in-db
TamarZanzouri Nov 6, 2024
a950cb8
migration and initialization fixes
TamarZanzouri Nov 6, 2024
55a809a
Update robot-server/robot_server/runs/run_store.py
TamarZanzouri Nov 6, 2024
1ccaba3
Merge branch 'edge' into EXEC-655-store-commands-error-list-in-db
TamarZanzouri Nov 6, 2024
9fa97ba
change get_run_errors to get count
TamarZanzouri Nov 6, 2024
f5fe016
Merge branch 'edge' into EXEC-655-store-commands-error-list-in-db
TamarZanzouri Nov 6, 2024
13c5b87
tables tests fixed
TamarZanzouri Nov 6, 2024
a88e95d
added test for historical run
TamarZanzouri Nov 6, 2024
6fea08a
Merge branch 'edge' into EXEC-655-store-commands-error-list-in-db
TamarZanzouri Nov 6, 2024
1bea28e
fixed persistance path
TamarZanzouri Nov 6, 2024
8f73bc1
fixed import from schema 7
TamarZanzouri Nov 7, 2024
c87dc65
rollback add_column change
TamarZanzouri Nov 7, 2024
5405ef3
append_failed_command_id when setting a failed command
TamarZanzouri Nov 7, 2024
f888364
append_failed_command_id when setting a failed command
TamarZanzouri Nov 7, 2024
e6aa375
moved test to command state
TamarZanzouri Nov 8, 2024
66856ac
changed index and lint
TamarZanzouri Nov 8, 2024
02e6806
WIP - command status
TamarZanzouri Nov 8, 2024
37a4aff
fixed failing tests
TamarZanzouri Nov 8, 2024
6c6257f
insert and get with command_status
TamarZanzouri Nov 8, 2024
56d12c4
translate to CommandStatusSQLEnum
TamarZanzouri Nov 8, 2024
c33745b
nips
TamarZanzouri Nov 12, 2024
635a5af
Update robot-server/tests/runs/test_run_store.py
TamarZanzouri Nov 12, 2024
a16636b
nits
TamarZanzouri Nov 12, 2024
a873369
batch edit
TamarZanzouri Nov 13, 2024
67b6d53
batch update without binding
TamarZanzouri Nov 13, 2024
4d00c99
test table with new index and migration bulk fix
TamarZanzouri Nov 13, 2024
38cd6ea
conditional update
TamarZanzouri Nov 13, 2024
1160174
map command status to sql command status
TamarZanzouri Nov 13, 2024
18ce80c
map command status to sql command status in migration
TamarZanzouri Nov 13, 2024
964fddc
map command status to sql command status in migration
TamarZanzouri Nov 13, 2024
5ad4d0d
add missing indexes
TamarZanzouri Nov 13, 2024
de02268
lint
TamarZanzouri Nov 13, 2024
085d4bb
change conversion into enum
TamarZanzouri Nov 13, 2024
378eec1
Merge branch 'edge' into EXEC-655-store-commands-error-list-in-db
TamarZanzouri Nov 13, 2024
ec64c12
fixing merge conflicts - WIP
TamarZanzouri Nov 14, 2024
b285c99
WIP - fix test_tables failing
TamarZanzouri Nov 14, 2024
3a29bfd
update test table
TamarZanzouri Nov 20, 2024
dca8269
Revert "update test table"
TamarZanzouri Nov 20, 2024
4d91d6f
Fix schema_8.py after merge.
SyntaxColoring Nov 14, 2024
fd78ad3
Fix up test_tables.py after merge.
SyntaxColoring Nov 14, 2024
b59601e
Remove attempts to fix prior migrations, for now.
SyntaxColoring Nov 20, 2024
6496d3c
Make command_status column nullable to match what the migration does.
SyntaxColoring Nov 20, 2024
11aa280
Remove enum constraint, for now.
SyntaxColoring Nov 20, 2024
5af324e
Merge branch 'edge' into EXEC-655-store-commands-error-list-in-db
SyntaxColoring Nov 20, 2024
6669e61
Update robot-server/robot_server/persistence/_migrations/v7_to_v8.py
TamarZanzouri Nov 21, 2024
ce3fbb3
Update robot-server/robot_server/runs/run_store.py
TamarZanzouri Nov 21, 2024
e0aacbd
select subquery and minor fixes
TamarZanzouri Nov 21, 2024
b10e8a2
fixed join logic and added logic to the tests
TamarZanzouri Nov 22, 2024
13fd3a8
fixed logic of cursor
TamarZanzouri Nov 22, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions api/src/opentrons/protocol_engine/state/command_history.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ class CommandHistory:
_all_command_ids: List[str]
"""All command IDs, in insertion order."""

_all_failed_command_ids: List[str]
"""All failed command IDs, in insertion order."""

_all_command_ids_but_fixit_command_ids: List[str]
"""All command IDs besides fixit command intents, in insertion order."""

Expand Down Expand Up @@ -101,6 +104,13 @@ def get_all_commands(self) -> List[Command]:
for command_id in self._all_command_ids
]

def get_all_failed_commands(self) -> List[Command]:
"""Get all failed commands."""
return [
self._commands_by_id[command_id].command
for command_id in self._all_failed_command_ids
]

def get_filtered_command_ids(self, include_fixit_commands: bool) -> List[str]:
"""Get all fixit command IDs."""
if include_fixit_commands:
Expand Down Expand Up @@ -251,6 +261,10 @@ def _add(self, command_id: str, command_entry: CommandEntry) -> None:
self._all_command_ids_but_fixit_command_ids.append(command_id)
self._commands_by_id[command_id] = command_entry

def append_failed_command_id(self, command_id: str) -> None:
"""Create or update a failed command id."""
self._all_failed_command_ids.append(command_id)

TamarZanzouri marked this conversation as resolved.
Show resolved Hide resolved
def _add_to_queue(self, command_id: str) -> None:
"""Add new ID to the queued."""
self._queued_command_ids.add(command_id)
Expand Down
13 changes: 7 additions & 6 deletions api/src/opentrons/protocol_engine/state/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,9 +228,6 @@ class CommandState:
This value can be used to generate future hashes.
"""

failed_command_errors: List[ErrorOccurrence]
"""List of command errors that occurred during run execution."""

has_entered_error_recovery: bool
"""Whether the run has entered error recovery."""

Expand Down Expand Up @@ -269,7 +266,6 @@ def __init__(
run_started_at=None,
latest_protocol_command_hash=None,
stopped_by_estop=False,
failed_command_errors=[],
error_recovery_policy=error_recovery_policy,
has_entered_error_recovery=False,
)
Expand Down Expand Up @@ -366,7 +362,7 @@ def _handle_fail_command_action(self, action: FailCommandAction) -> None:
notes=action.notes,
)
self._state.failed_command = self._state.command_history.get(action.command_id)
self._state.failed_command_errors.append(public_error_occurrence)
self._state.command_history.append_failed_command_id(action.command_id)

if (
prev_entry.command.intent in (CommandIntent.PROTOCOL, None)
Expand Down Expand Up @@ -706,7 +702,12 @@ def get_error(self) -> Optional[ErrorOccurrence]:

def get_all_errors(self) -> List[ErrorOccurrence]:
"""Get the run's full error list, if there was none, returns an empty list."""
return self._state.failed_command_errors
failed_commands = self._state.command_history.get_all_failed_commands()
return [
command_error.error
for command_error in failed_commands
if command_error.error is not None
]

def get_has_entered_recovery_mode(self) -> bool:
"""Get whether the run has entered recovery mode."""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ def test_command_failure(error_recovery_type: ErrorRecoveryType) -> None:
)

assert subject_view.get("command-id") == expected_failed_command
assert subject.state.failed_command_errors == [expected_error_occurrence]
assert subject_view.get_all_errors() == [expected_error_occurrence]


def test_command_failure_clears_queues() -> None:
Expand Down Expand Up @@ -255,7 +255,7 @@ def test_command_failure_clears_queues() -> None:
assert subject_view.get_running_command_id() is None
assert subject_view.get_queue_ids() == OrderedSet()
assert subject_view.get_next_to_execute() is None
assert subject.state.failed_command_errors == [expected_error_occurance]
assert subject_view.get_all_errors() == [expected_error_occurance]


def test_setup_command_failure_only_clears_setup_command_queue() -> None:
Expand Down Expand Up @@ -555,7 +555,7 @@ def test_door_during_error_recovery() -> None:
subject.handle_action(play)
assert subject_view.get_status() == EngineStatus.AWAITING_RECOVERY
assert subject_view.get_next_to_execute() == "command-id-2"
assert subject.state.failed_command_errors == [expected_error_occurance]
assert subject_view.get_all_errors() == [expected_error_occurance]


@pytest.mark.parametrize("close_door_before_queueing", [False, True])
Expand Down Expand Up @@ -732,7 +732,7 @@ def test_error_recovery_type_tracking() -> None:
id="c2-error", createdAt=datetime(year=2023, month=3, day=3), error=exception
)

assert subject.state.failed_command_errors == [
assert view.get_all_errors() == [
error_occurrence_1,
error_occurrence_2,
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,6 @@ def test_command_store_handles_pause_action(pause_source: PauseSource) -> None:
recovery_target=None,
latest_protocol_command_hash=None,
stopped_by_estop=False,
failed_command_errors=[],
error_recovery_policy=matchers.Anything(),
has_entered_error_recovery=False,
)
Expand Down Expand Up @@ -363,7 +362,6 @@ def test_command_store_handles_play_action(pause_source: PauseSource) -> None:
run_started_at=datetime(year=2021, month=1, day=1),
latest_protocol_command_hash=None,
stopped_by_estop=False,
failed_command_errors=[],
error_recovery_policy=matchers.Anything(),
has_entered_error_recovery=False,
)
Expand Down Expand Up @@ -398,7 +396,6 @@ def test_command_store_handles_finish_action() -> None:
run_started_at=datetime(year=2021, month=1, day=1),
latest_protocol_command_hash=None,
stopped_by_estop=False,
failed_command_errors=[],
error_recovery_policy=matchers.Anything(),
has_entered_error_recovery=False,
)
Expand Down Expand Up @@ -453,7 +450,6 @@ def test_command_store_handles_stop_action(
run_started_at=datetime(year=2021, month=1, day=1),
latest_protocol_command_hash=None,
stopped_by_estop=from_estop,
failed_command_errors=[],
error_recovery_policy=matchers.Anything(),
has_entered_error_recovery=False,
)
Expand Down Expand Up @@ -491,7 +487,6 @@ def test_command_store_handles_stop_action_when_awaiting_recovery() -> None:
run_started_at=datetime(year=2021, month=1, day=1),
latest_protocol_command_hash=None,
stopped_by_estop=False,
failed_command_errors=[],
error_recovery_policy=matchers.Anything(),
has_entered_error_recovery=False,
)
Expand Down Expand Up @@ -525,7 +520,6 @@ def test_command_store_cannot_restart_after_should_stop() -> None:
run_started_at=None,
latest_protocol_command_hash=None,
stopped_by_estop=False,
failed_command_errors=[],
error_recovery_policy=matchers.Anything(),
has_entered_error_recovery=False,
)
Expand Down Expand Up @@ -672,7 +666,6 @@ def test_command_store_wraps_unknown_errors() -> None:
recovery_target=None,
latest_protocol_command_hash=None,
stopped_by_estop=False,
failed_command_errors=[],
error_recovery_policy=matchers.Anything(),
has_entered_error_recovery=False,
)
Expand Down Expand Up @@ -742,7 +735,6 @@ def __init__(self, message: str) -> None:
run_started_at=None,
latest_protocol_command_hash=None,
stopped_by_estop=False,
failed_command_errors=[],
error_recovery_policy=matchers.Anything(),
has_entered_error_recovery=False,
)
Expand Down Expand Up @@ -778,7 +770,6 @@ def test_command_store_ignores_stop_after_graceful_finish() -> None:
run_started_at=datetime(year=2021, month=1, day=1),
latest_protocol_command_hash=None,
stopped_by_estop=False,
failed_command_errors=[],
error_recovery_policy=matchers.Anything(),
has_entered_error_recovery=False,
)
Expand Down Expand Up @@ -814,7 +805,6 @@ def test_command_store_ignores_finish_after_non_graceful_stop() -> None:
run_started_at=datetime(year=2021, month=1, day=1),
latest_protocol_command_hash=None,
stopped_by_estop=False,
failed_command_errors=[],
error_recovery_policy=matchers.Anything(),
has_entered_error_recovery=False,
)
Expand Down Expand Up @@ -850,7 +840,6 @@ def test_handles_hardware_stopped() -> None:
run_started_at=None,
latest_protocol_command_hash=None,
stopped_by_estop=False,
failed_command_errors=[],
error_recovery_policy=matchers.Anything(),
has_entered_error_recovery=False,
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,6 @@ def get_command_view( # noqa: C901
finish_error: Optional[errors.ErrorOccurrence] = None,
commands: Sequence[cmd.Command] = (),
latest_command_hash: Optional[str] = None,
failed_command_errors: Optional[List[ErrorOccurrence]] = None,
has_entered_error_recovery: bool = False,
) -> CommandView:
"""Get a command view test subject."""
Expand Down Expand Up @@ -121,7 +120,6 @@ def get_command_view( # noqa: C901
run_started_at=run_started_at,
latest_protocol_command_hash=latest_command_hash,
stopped_by_estop=False,
failed_command_errors=failed_command_errors or [],
has_entered_error_recovery=has_entered_error_recovery,
error_recovery_policy=_placeholder_error_recovery_policy,
)
Expand Down Expand Up @@ -1033,7 +1031,7 @@ def test_get_slice_default_cursor_running() -> None:

def test_get_errors_slice_empty() -> None:
"""It should return a slice from the tail if no current command."""
subject = get_command_view(failed_command_errors=[])
subject = get_command_view()
result = subject.get_errors_slice(cursor=0, length=2)

assert result == CommandErrorSlice(commands_errors=[], cursor=0, total_length=0)
Expand All @@ -1046,8 +1044,15 @@ def test_get_errors_slice() -> None:
error_3 = ErrorOccurrence.construct(id="error-id-3") # type: ignore[call-arg]
error_4 = ErrorOccurrence.construct(id="error-id-4") # type: ignore[call-arg]

command_1 = create_failed_command(command_id="command-id-1", error=error_1)
command_2 = create_failed_command(command_id="command-id-2", error=error_2)
command_3 = create_failed_command(command_id="command-id-3", error=error_3)
command_4 = create_failed_command(command_id="command-id-4", error=error_4)
command_5 = create_running_command(command_id="command-id-5")
command_6 = create_queued_command(command_id="command-id-6")

subject = get_command_view(
failed_command_errors=[error_1, error_2, error_3, error_4]
commands=[command_1, command_2, command_3, command_4, command_5, command_6]
)

result = subject.get_errors_slice(cursor=1, length=3)
Expand Down
2 changes: 1 addition & 1 deletion robot-server/robot_server/persistence/tables/__init__.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""SQL database schemas."""

# Re-export the latest schema.
from .schema_7 import (
from .schema_8 import (
metadata,
protocol_table,
analysis_table,
Expand Down
Loading
Loading