fix(hardware): allow longer for pipettes to boot (#12941)

Pipettes sometimes don't boot fast enough on attach and miss the
InstrumentInfoRequest that happens when the head tells us a new
instrument is present. This didn't used to be an issue because we
wouldn't send the request until the user was finished screwing things
in, so we just didn't notice.

In addition to our previous timeout - or, at a certain level, instead of
it - where we'd wait a long time for the pipette to respond, we'll also
now retry sending the request a couple times too.

Fixes RQA-998

api/src/opentrons/hardware_control/backends/subsystem_manager.py

@@ -405,7 +405,7 @@ async def _tool_detection_task_protected(self) -> None:
                 right=self._tool_if_ok(update.right, NodeId.pipette_right),
                 gripper=self._tool_if_ok(update.gripper, NodeId.gripper),
             )
-            self._present_tools = await self._tool_detector.resolve(to_resolve, 5.0)
+            self._present_tools = await self._tool_detector.resolve(to_resolve, 10.0)
             log.info(f"Present tools are now {self._present_tools}")
             async with self._tool_task_condition:
                 self._tool_task_state = True

api/tests/opentrons/hardware_control/backends/test_ot3_subsystem_manager.py

@@ -268,7 +268,7 @@ async def add_resolution(
             else ToolType.nothing_attached,
         )
 
-        self._decoy.when(await self._tool_detector.resolve(arg, 5.0)).then_return(
+        self._decoy.when(await self._tool_detector.resolve(arg, 10.0)).then_return(
             summary
         )
         return summary

hardware/opentrons_hardware/hardware_control/tools/detector.py

@@ -136,16 +136,17 @@ def _should_query(attach_response: ToolType) -> bool:
     return should_respond
 
 
-async def _resolve_tool_types(
+IntermediateResolution = Tuple[
+    Dict[NodeId, PipetteInformation], Dict[NodeId, GripperInformation]
+]
+
+
+async def _do_tool_resolve(
     messenger: CanMessenger,
     wc: WaitableCallback,
-    attached: ToolDetectionResult,
+    should_respond: Set[NodeId],
     timeout_sec: float,
-) -> ToolSummary:
-
-    should_respond = _need_type_query(attached)
-    if not should_respond:
-        return ToolSummary(left=None, right=None, gripper=None)
+) -> IntermediateResolution:
     await messenger.send(
         node_id=NodeId.broadcast,
         message=message_definitions.InstrumentInfoRequest(),
@@ -170,10 +171,66 @@ async def _resolve_tool_types(
         else:
             gripper[node] = info
 
+    return pipettes, gripper
+
+
+async def _resolve_with_stimulus_retries(
+    messenger: CanMessenger,
+    wc: WaitableCallback,
+    should_respond: Set[NodeId],
+    attempt_timeout_sec: float,
+    output_queue: "asyncio.Queue[IntermediateResolution]",
+) -> None:
+    expected_pipettes = {NodeId.pipette_left, NodeId.pipette_right}.intersection(
+        should_respond
+    )
+    expected_gripper = {NodeId.gripper}.intersection(should_respond)
+
+    while True:
+        pipettes, gripper = await _do_tool_resolve(
+            messenger, wc, should_respond, attempt_timeout_sec
+        )
+        output_queue.put_nowait((pipettes, gripper))
+        seen_pipettes = set([k.application_for() for k in pipettes.keys()])
+        seen_gripper = set([k.application_for() for k in gripper.keys()])
+        if seen_pipettes == expected_pipettes and seen_gripper == expected_gripper:
+            return
+
+
+async def _resolve_tool_types(
+    messenger: CanMessenger,
+    wc: WaitableCallback,
+    attached: ToolDetectionResult,
+    timeout_sec: float,
+) -> ToolSummary:
+
+    should_respond = _need_type_query(attached)
+    if not should_respond:
+        return ToolSummary(left=None, right=None, gripper=None)
+
+    resolve_queue: "asyncio.Queue[IntermediateResolution]" = asyncio.Queue()
+
+    try:
+        await asyncio.wait_for(
+            _resolve_with_stimulus_retries(
+                messenger, wc, should_respond, min(timeout_sec / 10, 0.5), resolve_queue
+            ),
+            timeout_sec,
+        )
+    except asyncio.TimeoutError:
+        log.warning("No response from expected tool")
+
+    last_element: IntermediateResolution = ({}, {})
+    try:
+        while True:
+            last_element = resolve_queue.get_nowait()
+    except asyncio.QueueEmpty:
+        pass
+
     return ToolSummary(
-        left=pipettes.get(NodeId.pipette_left, None),
-        right=pipettes.get(NodeId.pipette_right, None),
-        gripper=gripper.get(NodeId.gripper, None),
+        left=last_element[0].get(NodeId.pipette_left, None),
+        right=last_element[0].get(NodeId.pipette_right, None),
+        gripper=last_element[1].get(NodeId.gripper, None),
     )