chore(flutter): detect label for clicks from the text

Author: abhaysood

flutter/packages/measure_flutter/lib/src/gestures/layout_snapshot_capture.dart

@@ -20,11 +20,13 @@ class LayoutSnapshotCaptureResult {
   final LayoutSnapshot snapshot;
   final Element? detectedElement;
   final String? detectedElementType;
+  final String? detectedElementLabel;
 
   LayoutSnapshotCaptureResult({
     required this.snapshot,
     this.detectedElement,
     this.detectedElementType,
+    this.detectedElementLabel,
   });
 }
 
@@ -39,6 +41,7 @@ class _CaptureState {
   // Results accumulated during traversal
   Element? detectedElement;
   String? detectedElementType;
+  String? detectedElementLabel;
 
   _CaptureState({
     required this.rootElement,
@@ -81,8 +84,7 @@ class LayoutSnapshotCapture {
       if (rootElement == null) return null;
 
       // Step 2: Find starting point (scaffold or root)
-      final scaffoldElement =
-          developer.Timeline.timeSync('findTopmostScaffold', () {
+      final scaffoldElement = developer.Timeline.timeSync('findTopmostScaffold', () {
         return _findTopmostScaffold(rootElement);
       });
       final startElement = scaffoldElement ?? rootElement;
@@ -118,16 +120,15 @@ class LayoutSnapshotCapture {
         snapshot: tree,
         detectedElement: state.detectedElement,
         detectedElementType: state.detectedElementType,
+        detectedElementLabel: state.detectedElementLabel,
       );
     });
   }
 
   /// Checks if an element has a valid RenderBox with size
   static bool _hasValidRenderBox(Element element) {
     final renderObject = element.renderObject;
-    return renderObject != null &&
-        renderObject is RenderBox &&
-        renderObject.hasSize;
+    return renderObject != null && renderObject is RenderBox && renderObject.hasSize;
   }
 
   /// Checks if an element should be skipped during traversal
@@ -138,8 +139,7 @@ class LayoutSnapshotCapture {
 
   /// Gets the node type for a widget based on the capture state
   static String? _getNodeType(Widget widget, _CaptureState state) {
-    if (state.providedWidgetsTypes != null &&
-        state.providedWidgetsTypes!.isNotEmpty) {
+    if (state.providedWidgetsTypes != null && state.providedWidgetsTypes!.isNotEmpty) {
       return _getUserProvidedWidget(widget, state.providedWidgetsTypes);
     }
     return _getFrameworkWidget(widget);
@@ -177,9 +177,7 @@ class LayoutSnapshotCapture {
   /// Updates the state if a matching element is found
   static void _tryDetectElement(Element element, _CaptureState state) {
     // Skip if already detected or detection not enabled
-    if (state.detectedElement != null ||
-        state.detectionPosition == null ||
-        state.detectionMode == null) {
+    if (state.detectedElement != null || state.detectionPosition == null || state.detectionMode == null) {
       return;
     }
 
@@ -197,10 +195,14 @@ class LayoutSnapshotCapture {
     }
 
     // If it matches and is at the position, record it
-    if (detectedType != null &&
-        _hitTest(element, state.detectionPosition!, state.rootElement)) {
+    if (detectedType != null && _hitTest(element, state.detectionPosition!, state.rootElement)) {
       state.detectedElement = element;
       state.detectedElementType = detectedType;
+
+      // Extract label for click detection mode
+      if (state.detectionMode == DetectionMode.click) {
+        state.detectedElementLabel = _findFirstTextInSubtree(element);
+      }
     }
   }
 
@@ -217,9 +219,7 @@ class LayoutSnapshotCapture {
     // Determine the node type
     final nodeType = _getNodeType(widget, state);
     final isDetected = (state.detectedElement == element);
-    final effectiveNodeType = isDetected && nodeType == null
-        ? state.detectedElementType
-        : nodeType;
+    final effectiveNodeType = isDetected && nodeType == null ? state.detectedElementType : nodeType;
 
     // Determine highlighting (for click detection)
     final isHighlighted = state.detectionMode == DetectionMode.click && isDetected;
@@ -356,8 +356,7 @@ class LayoutSnapshotCapture {
         return;
       }
 
-      if (widget.runtimeType == Scaffold ||
-          widget.runtimeType == CupertinoPageScaffold) {
+      if (widget.runtimeType == Scaffold || widget.runtimeType == CupertinoPageScaffold) {
         topmostScaffold = element;
       }
 
@@ -372,8 +371,7 @@ class LayoutSnapshotCapture {
     if (rootElement == null) return false;
 
     final renderObject = element.renderObject;
-    if (renderObject == null ||
-        (renderObject is RenderBox && !renderObject.hasSize)) {
+    if (renderObject == null || (renderObject is RenderBox && !renderObject.hasSize)) {
       return false;
     }
 
@@ -387,8 +385,7 @@ class LayoutSnapshotCapture {
 
     // Check bounds
     final transform = renderObject.getTransformTo(rootElement.renderObject);
-    final paintBounds =
-        MatrixUtils.transformRect(transform, renderObject.paintBounds);
+    final paintBounds = MatrixUtils.transformRect(transform, renderObject.paintBounds);
     return paintBounds.contains(position);
   }
 
@@ -428,8 +425,7 @@ class LayoutSnapshotCapture {
     return type;
   }
 
-  static String? _getUserProvidedWidget(
-      Widget widget, Map<Type, String>? providedWidgets) {
+  static String? _getUserProvidedWidget(Widget widget, Map<Type, String>? providedWidgets) {
     final type = widget.runtimeType;
     if (providedWidgets?.keys.contains(type) == true) {
       return providedWidgets?[type];
@@ -457,11 +453,7 @@ class LayoutSnapshotCapture {
       ExpansionTile _ => 'ExpansionTile',
       Card _ => 'Card',
       InkWell w when w.onTap != null => 'InkWell',
-      GestureDetector w
-          when w.onTap != null ||
-              w.onDoubleTap != null ||
-              w.onLongPress != null =>
-        'GestureDetector',
+      GestureDetector w when w.onTap != null || w.onDoubleTap != null || w.onLongPress != null => 'GestureDetector',
       InkResponse w when w.onTap != null => 'InkResponse',
       InputChip w when w.onPressed != null => 'InputChip',
       ActionChip w when w.onPressed != null => 'ActionChip',
@@ -494,4 +486,42 @@ class LayoutSnapshotCapture {
       _ => null,
     };
   }
+
+  /// Extracts text or semantic label from a widget
+  /// Returns null if the widget has no text content
+  /// [maxLength] limits the returned text length (default: 32)
+  static String? _extractTextFromWidget(Widget widget, {int maxLength = 32}) {
+    final text = switch (widget) {
+      Text w => w.data ?? w.textSpan?.toPlainText(),
+      RichText w => w.text.toPlainText(),
+      Icon w => w.semanticLabel,
+      _ => null,
+    };
+    return text != null && text.length > maxLength ? text.substring(0, maxLength) : text;
+  }
+
+  /// Finds the first text or semantic label in the subtree of an element
+  /// Returns null if no text is found
+  static String? _findFirstTextInSubtree(Element element) {
+    // Try to extract text from this element first
+    final text = _extractTextFromWidget(element.widget);
+    if (text != null && text.isNotEmpty) {
+      return text;
+    }
+
+    // Traverse children to find text
+    String? foundText;
+    element.visitChildElements((child) {
+      // Skip if we already found text
+      if (foundText != null) return;
+
+      // Skip offstage widgets
+      if (_shouldSkipElement(child)) return;
+
+      // Recursively search in child
+      foundText = _findFirstTextInSubtree(child);
+    });
+
+    return foundText;
+  }
 }

flutter/packages/measure_flutter/lib/src/gestures/msr_gesture_detector.dart

@@ -162,7 +162,7 @@ class MsrGestureDetectorState extends State<MsrGestureDetector> {
           target: result.detectedElementType!,
           x: (position.dx * devicePixelRatio).roundToDouble(),
           y: (position.dy * devicePixelRatio).roundToDouble(),
-          targetId: truncateLabel(label, maxLength: _maxLabelLength),
+          targetId: result.detectedElementLabel,
           touchDownTime: null,
           touchUpTime: null,
         ),