fix: restore reliable multi-touch streaming

Author: DjDeveloperr

AGENTS.md

@@ -81,7 +81,7 @@ The current repo uses the private boot path, private display bridge, and private
 CoreSimulator service contexts resolve the active developer directory from `DEVELOPER_DIR`, then `xcode-select -p`, then `/Applications/Xcode.app/Contents/Developer`. The display bridge prefers direct CoreSimulator screen IOSurface callbacks and activates the SimulatorKit offscreen renderable view only if direct callbacks are unavailable.
 Accessibility recovery may use simulator launchctl UIKit application state plus hit-tested translations to recover candidate foreground pids; the returned tree must still be rooted at tokenized `AXPTranslator` application objects, because `translationApplicationObjectForPid:` can omit the bridge delegate token after private display lifecycle changes. Full-tree snapshots merge those recovered roots with the private frontmost application translation. When multiple candidate application roots are discovered, serialize all of them in preferred order: non-extension app roots first, then largest translated roots, with `.appex`/PlugIns processes de-prioritized so SpringBoard and Safari app roots stay primary while widgets and WebContent roots remain debuggable. Widget renderer extension roots may report local frames; normalize those roots and children against matching SpringBoard widget placeholder frames before returning the snapshot.
 Physical chrome button support uses DeviceKit `chrome.json` input geometry for browser hit targets. Volume, action, mute, Apple Watch digital crown, Watch side button, and Watch left-side button dispatch through `IndigoHIDMessageForHIDArbitrary` with consumer/telephony/vendor HID usage pairs from the device chrome metadata; home, lock, and app-switcher remain on the existing SimulatorKit button paths. Apple Watch Digital Crown rotation dispatches through `IndigoHIDMessageForDigitalCrownEvent` when SimulatorKit exposes it, with `IndigoHIDMessageForScrollEvent(..., target=0x34)` as the fallback. tvOS simulators do not support direct screen touch; browser/API tap maps to Enter, swipe maps to arrow keys, and the native bridge rejects tvOS touch packets before they reach guest `SimulatorHID`. watchOS/tvOS skip dynamic pointer/mouse service warm-up because those guest runtimes abort on unsupported virtual services. Apple TV and Apple Watch simulators are fixed-orientation devices, so client and server rotation paths must not expose or dispatch device rotation for those families.
-Two-point multi-touch dispatch prefers the current SimulatorKit/Indigo packet constructor and falls back to SimDeck's manual Indigo packet adapter. The fallback must preserve `began`/`moved`/`ended` mouse event types instead of collapsing movement into down/up packets, because touch recognizers in guest apps otherwise see pinches as ordinary single-finger swipes.
+Two-point multi-touch dispatch prefers the current SimulatorKit/Indigo packet constructor and falls back to SimDeck's manual Indigo packet adapter. On Xcode 26 SimulatorKit, the constructor expects pixel-space points and stable two-finger movement requires sending `LeftMouseDown` for both `began` and `moved`, then `LeftMouseUp` for `ended`/`cancelled`; using `LeftMouseDragged` for multi-touch moves only advances one contact in UIKit. Do not coalesce multi-touch move packets in the WebSocket or WebRTC control paths, because gesture recognizers need the intermediate two-contact samples.
 WebKit inspection uses the simulator `webinspectord` Unix socket named `com.apple.webinspectord_sim.socket` and WebKit's binary-plist Remote Inspector selectors. It lists only WebKit content that the runtime exposes as inspectable. For app-owned `WKWebView` on iOS 16.4 and newer, the app must set `isInspectable = true`.
 
 ## Build and Run

cli/DFPrivateSimulatorDisplayBridge.h

@@ -48,6 +48,8 @@ NS_SWIFT_NAME(PrivateSimulatorDisplayBridge)
 @property (nonatomic, readonly) CGSize displaySize;
 @property (nonatomic, readonly) NSInteger rotationQuarterTurns;
 
+- (void)updateInputDisplaySize:(CGSize)displaySize;
+
 - (nullable CVPixelBufferRef)copyPixelBuffer CF_RETURNS_RETAINED;
 
 - (BOOL)sendTouchAtNormalizedX:(double)normalizedX

cli/DFPrivateSimulatorDisplayBridge.m

@@ -1228,6 +1228,17 @@ static uint32_t DFIndigoMouseEventTypeForPhase(DFPrivateSimulatorTouchPhase phas
     }
 }
 
+static uint32_t DFIndigoMultiTouchMouseEventTypeForPhase(DFPrivateSimulatorTouchPhase phase) {
+    switch (phase) {
+    case DFPrivateSimulatorTouchPhaseBegan:
+    case DFPrivateSimulatorTouchPhaseMoved:
+        return DFIndigoMouseEventDown;
+    case DFPrivateSimulatorTouchPhaseEnded:
+    case DFPrivateSimulatorTouchPhaseCancelled:
+        return DFIndigoMouseEventUp;
+    }
+}
+
 static IndigoHIDMessage *DFCreateIndigoTouchMessageDirect(CGPoint normalizedPoint,
                                                           NSSize displaySize,
                                                           DFPrivateSimulatorTouchPhase phase,
@@ -1238,11 +1249,11 @@ static uint32_t DFIndigoMouseEventTypeForPhase(DFPrivateSimulatorTouchPhase phas
         return NULL;
     }
 
-    CGPoint ratioPoint = CGPointMake(
-        fmax(0.0, fmin(1.0, normalizedPoint.x)),
-        fmax(0.0, fmin(1.0, normalizedPoint.y))
+    CGPoint pixelPoint = CGPointMake(
+        fmax(0.0, fmin(1.0, normalizedPoint.x)) * displaySize.width,
+        fmax(0.0, fmin(1.0, normalizedPoint.y)) * displaySize.height
     );
-    return mouseMessage(&ratioPoint,
+    return mouseMessage(&pixelPoint,
                         NULL,
                         target,
                         (NSEventType)DFIndigoMouseEventTypeForPhase(phase),
@@ -1260,18 +1271,18 @@ static uint32_t DFIndigoMouseEventTypeForPhase(DFPrivateSimulatorTouchPhase phas
         return NULL;
     }
 
-    CGPoint ratioPoint = CGPointMake(
-        fmax(0.0, fmin(1.0, normalizedPoint1.x)),
-        fmax(0.0, fmin(1.0, normalizedPoint1.y))
+    CGPoint pixelPoint = CGPointMake(
+        fmax(0.0, fmin(1.0, normalizedPoint1.x)) * displaySize.width,
+        fmax(0.0, fmin(1.0, normalizedPoint1.y)) * displaySize.height
     );
-    CGPoint secondRatioPoint = CGPointMake(
-        fmax(0.0, fmin(1.0, normalizedPoint2.x)),
-        fmax(0.0, fmin(1.0, normalizedPoint2.y))
+    CGPoint secondPixelPoint = CGPointMake(
+        fmax(0.0, fmin(1.0, normalizedPoint2.x)) * displaySize.width,
+        fmax(0.0, fmin(1.0, normalizedPoint2.y)) * displaySize.height
     );
-    return mouseMessage(&ratioPoint,
-                        &secondRatioPoint,
+    return mouseMessage(&pixelPoint,
+                        &secondPixelPoint,
                         target,
-                        (NSEventType)DFIndigoMouseEventTypeForPhase(phase),
+                        (NSEventType)DFIndigoMultiTouchMouseEventTypeForPhase(phase),
                         displaySize,
                         DFIndigoHIDEdgeNone);
 }
@@ -1449,7 +1460,7 @@ static id DFCreateLegacyHIDClientForDevice(id device, NSError **error) {
         return NULL;
     }
 
-    NSEventType eventType = (NSEventType)DFIndigoMouseEventTypeForPhase(phase);
+    NSEventType eventType = (NSEventType)DFIndigoMultiTouchMouseEventTypeForPhase(phase);
     CGPoint ratioPoint = CGPointMake(
         fmax(0.0, fmin(1.0, normalizedPoint1.x)),
         fmax(0.0, fmin(1.0, normalizedPoint1.y))
@@ -3845,6 +3856,25 @@ - (NSInteger)rotationQuarterTurns {
     return turns;
 }
 
+- (void)updateInputDisplaySize:(CGSize)displaySize {
+    if (displaySize.width <= 0.0 || displaySize.height <= 0.0 ||
+        !isfinite(displaySize.width) || !isfinite(displaySize.height)) {
+        return;
+    }
+
+    dispatch_block_t work = ^{
+        self->_displayPixelSize = displaySize;
+        DFReconcileRotationWithDisplaySize(&self->_deviceRotationDegrees, self->_displayPixelSize);
+        DFLog(@"Configured private HID display size %.0fx%.0f", displaySize.width, displaySize.height);
+    };
+
+    if (dispatch_get_specific(DFPrivateSimulatorCallbackQueueKey) != NULL) {
+        work();
+    } else {
+        dispatch_sync(_callbackQueue, work);
+    }
+}
+
 - (BOOL)sendTouchAtNormalizedX:(double)normalizedX
                    normalizedY:(double)normalizedY
                          phase:(DFPrivateSimulatorTouchPhase)phase
@@ -4101,7 +4131,9 @@ - (BOOL)sendMultiTouchAtNormalizedX1:(double)normalizedX1
                                                                           displaySize,
                                                                           phase,
                                                                           target);
+        NSString *messageSource = @"direct";
         if (message == NULL) {
+            messageSource = @"manual";
             const NSUInteger maxAttempts = phase == DFPrivateSimulatorTouchPhaseMoved ? 12 : 3;
             for (NSUInteger attempt = 0; attempt < maxAttempts; attempt++) {
                 message = (IndigoHIDMessage *)DFCreateIndigoMultiTouchMessage(CGPointMake(x1, y1),
@@ -4130,7 +4162,7 @@ - (BOOL)sendMultiTouchAtNormalizedX1:(double)normalizedX1
         }
 
         if (phase != DFPrivateSimulatorTouchPhaseMoved) {
-            DFLog(@"Sending %@ Indigo HID multi-touch target=0x%x p1=(%.4f, %.4f) p2=(%.4f, %.4f) within %.0fx%.0f", phaseLabel, target, x1, y1, x2, y2, displaySize.width, displaySize.height);
+            DFLog(@"Sending %@ Indigo HID multi-touch (%@) target=0x%x p1=(%.4f, %.4f) p2=(%.4f, %.4f) within %.0fx%.0f", phaseLabel, messageSource, target, x1, y1, x2, y2, displaySize.width, displaySize.height);
         }
         success = YES;
     };

cli/XCWChromeRenderer.h

@@ -20,6 +20,8 @@ NS_ASSUME_NONNULL_BEGIN
                                               error:(NSError * _Nullable * _Nullable)error;
 + (nullable NSDictionary<NSString *, id> *)profileForDeviceName:(NSString *)deviceName
                                                           error:(NSError * _Nullable * _Nullable)error;
++ (CGSize)displayPixelSizeForDeviceName:(NSString *)deviceName
+                                   error:(NSError * _Nullable * _Nullable)error;
 
 @end
 

cli/XCWChromeRenderer.m

@@ -50,6 +50,30 @@ @implementation XCWChromeRenderer
     return [self profileForChromeInfo:chromeInfo error:error];
 }
 
++ (CGSize)displayPixelSizeForDeviceName:(NSString *)deviceName
+                                   error:(NSError * _Nullable __autoreleasing *)error {
+    NSDictionary *chromeInfo = [self chromeInfoForDeviceName:deviceName error:error];
+    if (chromeInfo == nil) {
+        return CGSizeZero;
+    }
+
+    NSDictionary *plist = chromeInfo[@"plist"];
+    CGFloat width = [self numberValue:plist[@"mainScreenWidth"]];
+    CGFloat height = [self numberValue:plist[@"mainScreenHeight"]];
+    if (width <= 0.0 || height <= 0.0) {
+        if (error != NULL) {
+            *error = [NSError errorWithDomain:XCWChromeRendererErrorDomain
+                                         code:16
+                                     userInfo:@{
+                NSLocalizedDescriptionKey: [NSString stringWithFormat:@"The device profile for %@ did not specify a framebuffer size.", deviceName ?: @""],
+            }];
+        }
+        return CGSizeZero;
+    }
+
+    return CGSizeMake(width, height);
+}
+
 + (nullable NSData *)PNGDataForDeviceName:(NSString *)deviceName
                                     error:(NSError * _Nullable __autoreleasing *)error {
     return [self PNGDataForDeviceName:deviceName includeButtons:YES error:error];

cli/native/XCWNativeBridge.m

@@ -648,6 +648,16 @@ static BOOL XCWTouchPhaseFromString(NSString *phase, DFPrivateSimulatorTouchPhas
                                                                                               error:&error];
     if (bridge == nil) {
         XCWSetErrorMessage(errorMessage, error);
+        return nil;
+    }
+
+    NSDictionary *simulator = XCWSimulatorRecordForUDID(udid, NULL);
+    NSString *deviceName = simulator[@"deviceTypeName"] ?: simulator[@"name"] ?: @"";
+    NSError *displaySizeError = nil;
+    CGSize displaySize = [XCWChromeRenderer displayPixelSizeForDeviceName:deviceName
+                                                                    error:&displaySizeError];
+    if (displaySize.width > 0.0 && displaySize.height > 0.0) {
+        [bridge updateInputDisplaySize:displaySize];
     }
     return bridge;
 }

client/src/app/AppShell.tsx

@@ -2050,19 +2050,6 @@ export function AppShell({
     return state;
   }, []);
 
-  useEffect(() => {
-    if (!selectedSimulator?.isBooted) {
-      closeControlSocket();
-      return;
-    }
-    ensureControlSocket(selectedSimulator.udid);
-  }, [
-    closeControlSocket,
-    ensureControlSocket,
-    selectedSimulator?.isBooted,
-    selectedSimulator?.udid,
-  ]);
-
   function sendControl(udid: string, message: ControlMessage): boolean {
     if (isMoveControlMessage(message)) {
       pendingControlMoveRef.current = { message, udid };

client/src/app/controlMessages.test.ts

@@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest";
 import { isMoveControlMessage } from "./controlMessages";
 
 describe("controlMessages", () => {
-  it("marks all continuous touch moves as coalescible", () => {
+  it("marks single-touch moves as coalescible", () => {
     expect(
       isMoveControlMessage({ type: "touch", x: 0.4, y: 0.5, phase: "moved" }),
     ).toBe(true);
@@ -16,6 +16,12 @@ describe("controlMessages", () => {
         edge: "bottom",
       }),
     ).toBe(true);
+  });
+
+  it("does not coalesce multi-touch, gesture boundaries, or discrete controls", () => {
+    expect(
+      isMoveControlMessage({ type: "touch", x: 0.4, y: 0.5, phase: "began" }),
+    ).toBe(false);
     expect(
       isMoveControlMessage({
         type: "multiTouch",
@@ -25,12 +31,6 @@ describe("controlMessages", () => {
         y2: 0.5,
         phase: "moved",
       }),
-    ).toBe(true);
-  });
-
-  it("does not coalesce gesture boundaries or discrete controls", () => {
-    expect(
-      isMoveControlMessage({ type: "touch", x: 0.4, y: 0.5, phase: "began" }),
     ).toBe(false);
     expect(
       isMoveControlMessage({

client/src/app/controlMessages.ts

@@ -2,9 +2,7 @@ import type { ControlMessage } from "../api/controls";
 
 export function isMoveControlMessage(message: ControlMessage): boolean {
   return (
-    (message.type === "touch" ||
-      message.type === "edgeTouch" ||
-      message.type === "multiTouch") &&
+    (message.type === "touch" || message.type === "edgeTouch") &&
     message.phase === "moved"
   );
 }

client/src/features/input/usePointerInput.ts

@@ -43,6 +43,7 @@ type ActiveGesture =
     };
 
 const TWO_FINGER_SPREAD = 0.16;
+const PINCH_MINIMUM_SPREAD = 0.16;
 const BOTTOM_EDGE_GESTURE_START_Y = 0.93;
 
 export function usePointerInput({
@@ -78,8 +79,25 @@ export function usePointerInput({
     };
   }
 
-  function mirrorAroundCenter(point: Point): Point {
-    return clampPoint({ x: 1 - point.x, y: 1 - point.y });
+  function pinchPointsAroundCenter(
+    anchor: Point,
+    previousFirst?: Point,
+  ): [Point, Point] {
+    let dx = anchor.x - 0.5;
+    let dy = anchor.y - 0.5;
+    const halfMinimumSpread = PINCH_MINIMUM_SPREAD / 2;
+    const distance = Math.hypot(dx, dy);
+    if (distance < halfMinimumSpread) {
+      const fallbackDx = previousFirst ? previousFirst.x - 0.5 : 1;
+      const fallbackDy = previousFirst ? previousFirst.y - 0.5 : 0;
+      const fallbackDistance = Math.hypot(fallbackDx, fallbackDy) || 1;
+      dx = (fallbackDx / fallbackDistance) * halfMinimumSpread;
+      dy = (fallbackDy / fallbackDistance) * halfMinimumSpread;
+    }
+    return [
+      clampPoint({ x: 0.5 + dx, y: 0.5 + dy }),
+      clampPoint({ x: 0.5 - dx, y: 0.5 - dy }),
+    ];
   }
 
   function previewMultiTouch(phase: TouchPhase, first: Point, second: Point) {
@@ -193,8 +211,7 @@ export function usePointerInput({
         return;
       }
 
-      const first = clampPoint(coords);
-      const second = mirrorAroundCenter(first);
+      const [first, second] = pinchPointsAroundCenter(coords);
       activeGestureRef.current = {
         kind: "pinch",
         pointerId: event.pointerId,
@@ -226,8 +243,7 @@ export function usePointerInput({
     }
 
     if (active.kind === "pinch") {
-      const first = clampPoint(coords);
-      const second = mirrorAroundCenter(first);
+      const [first, second] = pinchPointsAroundCenter(coords, active.first);
       activeGestureRef.current = { ...active, first, second };
       sendMultiTouch("moved", first, second);
       return;
@@ -280,8 +296,9 @@ export function usePointerInput({
     );
 
     if (active.kind === "pinch") {
-      const first = coords ? clampPoint(coords) : active.first;
-      const second = coords ? mirrorAroundCenter(first) : active.second;
+      const [first, second] = coords
+        ? pinchPointsAroundCenter(coords, active.first)
+        : [active.first, active.second];
       sendMultiTouch(phase, first, second);
       return;
     }