Semantic matching basics #4788

detox/detox.d.ts

@@ -1123,10 +1123,17 @@ declare global {
             label(label: string | RegExp): NativeMatcher;
 
             /**
-             * Find an element by native view type.
-             * @example await element(by.type('RCTImageView'));
+             * Find an element by native view type OR semantic type.
+             * Supports both platform-specific class names and cross-platform semantic types.
+             * @example 
+             * // Platform-specific class names:
+             * await element(by.type('RCTImageView')); // iOS
+             * 
+             * // Cross-platform semantic types:
+             * await element(by.type('image'));
              */
-            type(nativeViewType: string): NativeMatcher;
+            type(typeOrSemanticType: string): NativeMatcher;
+
 
             /**
              * Find an element with an accessibility trait. (iOS only)

detox/src/android/matchers/native.js

@@ -1,6 +1,7 @@
 const DetoxRuntimeError = require('../../errors/DetoxRuntimeError');
 const invoke = require('../../invoke');
 const { isRegExp } = require('../../utils/isRegExp');
+const { getClassNamesForSemanticType, getAvailableSemanticTypes } = require('../../utils/semanticTypes');
 const { NativeMatcher } = require('../core/NativeMatcher');
 const DetoxMatcherApi = require('../espressoapi/DetoxMatcher');
 
@@ -31,7 +32,26 @@ class IdMatcher extends NativeMatcher {
 class TypeMatcher extends NativeMatcher {
   constructor(value) {
     super();
-    this._call = invoke.callDirectly(DetoxMatcherApi.matcherForClass(value));
+    // Check if it's a known semantic type first
+    if (getAvailableSemanticTypes().includes(value)) {
+      // It's a semantic type, create matcher for all class names
+      const classNames = getClassNamesForSemanticType(value, 'android');
+
+      let combinedMatcher = null;
+      for (let i = 0; i < classNames.length; i++) {
+        const matcher = new NativeMatcher(invoke.callDirectly(DetoxMatcherApi.matcherForClass(classNames[i])));
+        combinedMatcher = combinedMatcher ? combinedMatcher.or(matcher) : matcher;
+      }
+
+      if (!combinedMatcher) {
+        throw new DetoxRuntimeError(`No class names found for semantic type: ${value}`);
+      }
+
+      this._call = combinedMatcher._call;
+    } else {
+      // Not a semantic type, treat as regular class name
+      this._call = invoke.callDirectly(DetoxMatcherApi.matcherForClass(value));
+    }
   }
 }
 

detox/src/android/matchers/native.test.js

@@ -0,0 +1,60 @@
+// Mock the semanticTypes module before importing anything that depends on it
+jest.mock('../../utils/semanticTypes', () => ({
+  getAvailableSemanticTypes: jest.fn(),
+  getClassNamesForSemanticType: jest.fn()
+}));
+
+
+const DetoxRuntimeError = require('../../errors/DetoxRuntimeError');
+const semanticTypes = require('../../utils/semanticTypes');
+
+const { TypeMatcher } = require('./native');
+
+describe('Native Matchers', () => {
+  describe('TypeMatcher', () => {
+    beforeEach(() => {
+      jest.clearAllMocks();
+    });
+
+    it('should throw error when semantic type has no class names', () => {
+      // Mock getAvailableSemanticTypes to include a test semantic type
+      semanticTypes.getAvailableSemanticTypes.mockReturnValue(['test-semantic-type']);
+
+      // Mock getClassNamesForSemanticType to return empty array
+      semanticTypes.getClassNamesForSemanticType.mockReturnValue([]);
+
+      expect(() => {
+        new TypeMatcher('test-semantic-type');
+      }).toThrow(DetoxRuntimeError);
+
+      expect(() => {
+        new TypeMatcher('test-semantic-type');
+      }).toThrow('No class names found for semantic type: test-semantic-type');
+    });
+
+    it('should handle regular class names when not a semantic type', () => {
+      // Mock getAvailableSemanticTypes to not include the test class
+      semanticTypes.getAvailableSemanticTypes.mockReturnValue([]);
+
+      // Should not throw and should work with regular class names
+      expect(() => {
+        new TypeMatcher('com.example.CustomView');
+      }).not.toThrow();
+    });
+
+    it('should handle semantic types with valid class names', () => {
+      // Mock getAvailableSemanticTypes to include a test semantic type
+      semanticTypes.getAvailableSemanticTypes.mockReturnValue(['image']);
+
+      // Mock getClassNamesForSemanticType to return valid class names
+      semanticTypes.getClassNamesForSemanticType.mockReturnValue([
+        'android.widget.ImageView',
+        'com.facebook.react.views.image.ReactImageView'
+      ]);
+
+      expect(() => {
+        new TypeMatcher('image');
+      }).not.toThrow();
+    });
+  });
+}); 

detox/src/ios/expectTwo.js

@@ -11,6 +11,7 @@ const { actionDescription, expectDescription } = require('../utils/invocationTra
 const { isRegExp } = require('../utils/isRegExp');
 const log = require('../utils/logger').child({ cat: 'ws-client, ws' });
 const mapLongPressArguments = require('../utils/mapLongPressArguments');
+const { getClassNamesForSemanticType, getAvailableSemanticTypes } = require('../utils/semanticTypes');
 const tempfile = require('../utils/tempfile');
 const traceInvocationCall = require('../utils/traceInvocationCall').bind(null, log);
 
@@ -442,7 +443,17 @@ class Matcher {
 
   type(type) {
     if (typeof type !== 'string') throw new Error('type should be a string, but got ' + (type + (' (' + (typeof type + ')'))));
-    this.predicate = { type: 'type', value: type };
+    // Check if it's a known semantic type first
+    if (getAvailableSemanticTypes().includes(type)) {
+      // It's a semantic type
+      const classNames = getClassNamesForSemanticType(type, 'ios');
+      const predicates = classNames.map(className => ({ type: 'type', value: className }));
+      this.predicate = { type: 'or', predicates };
+    } else {
+      // Not a semantic type, treat as regular class name
+      this.predicate = { type: 'type', value: type };
+    }
+
     return this;
   }
 

detox/src/ios/expectTwo.test.js

@@ -793,6 +793,33 @@ describe('expectTwo', () => {
     });
   });
 
+  describe('semantic types', () => {
+    it(`should parse semantic type 'image' using by.type() to create OR predicate`, async () => {
+      const testCall = await e.element(e.by.type('image')).tap();
+      const jsonOutput = {
+        invocation: {
+          type: 'action',
+          action: 'tap',
+          predicate: {
+            type: 'or',
+            predicates: [
+              {
+                type: 'type',
+                value: 'RCTImageView'
+              },
+              {
+                type: 'type',
+                value: 'UIImageView'
+              }
+            ]
+          }
+        }
+      };
+
+      expect(testCall).toDeepEqual(jsonOutput);
+    });
+  });
+
   describe('web views', () => {
     it(`should parse expect(web(by.id('webViewId').element(web(by.label('tapMe')))).toExist()`, async () => {
       const testCall = await e.expect(e.web(e.by.id('webViewId')).atIndex(1).element(e.by.web.label('tapMe')).atIndex(2)).toExist();

detox/src/utils/semanticTypes.js

@@ -0,0 +1,99 @@
+/**
+ * Semantic type mappings for cross-platform component matching
+ */
+
+const SEMANTIC_TYPE_MAPPINGS = {
+  // Images
+  'image': {
+    ios: ['RCTImageView', 'UIImageView'],
+    android: ['android.widget.ImageView', 'com.facebook.react.views.image.ReactImageView']
+  },
+
+  // Input fields
+  'input-field': {
+    ios: ['RCTTextInput', 'RCTMultilineTextInput', 'UITextField', 'UITextView'],
+    android: ['android.widget.EditText', 'com.facebook.react.views.textinput.ReactEditText']
+  },
+
+  // Text elements
+  'text': {
+    ios: ['RCTText', 'UILabel'],
+    android: ['android.widget.TextView', 'com.facebook.react.views.text.ReactTextView']
+  },
+
+  // Button elements
+  'button': {
+    ios: ['RCTButton', 'UIButton'],
+    android: ['android.widget.Button', 'com.facebook.react.views.view.ReactViewGroup']
+  },
+
+  // Scroll containers
+  'scrollview': {
+    ios: ['RCTScrollView', 'UIScrollView'],
+    android: ['android.widget.ScrollView', 'androidx.core.widget.NestedScrollView', 'com.facebook.react.views.scroll.ReactScrollView']
+  },
+
+  // Lists
+  'list': {
+    ios: ['RCTFlatList', 'UITableView', 'UICollectionView'],
+    android: ['android.widget.ListView', 'androidx.recyclerview.widget.RecyclerView', 'com.facebook.react.views.scroll.ReactScrollView']
+  },
+
+  // Switches/Toggles
+  'switch': {
+    ios: ['RCTSwitch', 'UISwitch'],
+    android: ['android.widget.Switch', 'androidx.appcompat.widget.SwitchCompat', 'com.facebook.react.views.switchview.ReactSwitch']
+  },
+
+  // Sliders
+  'slider': {
+    ios: ['RCTSlider', 'UISlider'],
+    android: ['android.widget.SeekBar', 'com.facebook.react.views.slider.ReactSlider']
+  },
+
+  // Picker/Selector
+  'picker': {
+    ios: ['RCTPicker', 'UIPickerView'],
+    android: ['android.widget.Spinner', 'com.facebook.react.views.picker.ReactPickerManager']
+  },
+
+  // Activity indicators/Progress
+  'activity-indicator': {
+    ios: ['RCTActivityIndicatorView', 'UIActivityIndicatorView'],
+    android: ['android.widget.ProgressBar', 'com.facebook.react.views.progressbar.ReactProgressBarViewManager']
+  }
+};
+
+/**
+ * Get platform-specific class names for a semantic type
+ * @param {string} semanticType - The semantic type (e.g., 'image', 'input-field')
+ * @param {string} platform - The platform ('ios' or 'android')
+ * @returns {string[]} Array of class names for the platform
+ */
+function getClassNamesForSemanticType(semanticType, platform) {
+  const mapping = SEMANTIC_TYPE_MAPPINGS[semanticType];
+  if (!mapping) {
+    throw new Error(`Unknown semantic type: ${semanticType}. Available types: ${Object.keys(SEMANTIC_TYPE_MAPPINGS).join(', ')}`);
+  }
+
+  const classNames = mapping[platform];
+  if (!classNames) {
+    throw new Error(`Platform ${platform} not supported for semantic type ${semanticType}`);
+  }
+
+  return classNames;
+}
+
+/**
+ * Get all available semantic types
+ * @returns {string[]} Array of available semantic type names
+ */
+function getAvailableSemanticTypes() {
+  return Object.keys(SEMANTIC_TYPE_MAPPINGS);
+}
+
+module.exports = {
+  SEMANTIC_TYPE_MAPPINGS,
+  getClassNamesForSemanticType,
+  getAvailableSemanticTypes
+};