Stabilize Android E2E tests for ARM64 APKs (#58140)

Summary:
Stabilizes the Android RNTester E2E jobs after switching their APKs to ARM64 while continuing to run the emulator on an x86_64 host.

- Updates the Android wide-gamut screenshot baseline using the stable ARM64 result from the API 35 emulator. The emulator does not support wide color, so the Display-P3 fixture is converted to solid sRGB red.
- Marks the FlatList `maintainVisibleContentPosition` flows as Android release-only. These flows remain fully covered by the release APK, where they consistently pass, while avoiding debug-only timing failures caused by running the ARM64 debug runtime through native translation.
- Adds generic tag filtering to the Android Maestro runner and unit coverage for it.

Across seven post-migration `main` runs, the release APK passed every FlatList flow, while the debug APK consistently dropped or delayed Maestro interactions and skipped up to 172 frames during startup.

## Changelog:

[INTERNAL] [FIXED] - Stabilize Android RNTester E2E tests when running ARM64 APKs on x86_64 emulators.

Pull Request resolved: #58140

Test Plan:
- `./node_modules/.bin/jest .github/workflow-scripts/__tests__/maestro-android-test.js --runInBand --config='{"testEnvironment":"node","transform":{},"roots":["<rootDir>/.github/workflow-scripts"]}'` — passed (3 tests)
- `maestro 2.6.1 check-syntax` for all 24 tagged FlatList flows — passed
- Prettier check for all changed text files — passed
- `git diff --check` — passed
- Verified the filtered RNTester suite contains 16 debug flows and excludes 24 release-only FlatList MVCP flows
- Compared the new baseline against ARM64 CI screenshots: exact match for release; debug RMSE 0.0024
- Manually exercised the ARM64 release APK on an API 35 ARM64 emulator at 320×640: FlatList offsets progressed as expected (`500 → 544 → 2744 → 4944 → 7144`), and five rapid prepends reached `11500`

Related failing run: https://github.com/react/react-native/actions/runs/32821468795

Reviewed By: Abbondanzo

Differential Revision: D117361459

Pulled By: cortinico

fbshipit-source-id: 4514f8c42f7599c875672efe3c56aae7c9f0395c

Author: cortinico

.github/actions/maestro-android/action.yml

@@ -38,6 +38,10 @@ inputs:
     required: false
     default: /tmp/maestro-android-state/results.json
     description: The path used to persist per-flow test results between retries
+  exclude-tags:
+    required: false
+    default: ''
+    description: Comma-separated flow tags to exclude from the test run
 
 runs:
   using: composite
@@ -77,7 +81,7 @@ runs:
         cores: '4'
         disable-animations: false
         avd-name: e2e_emulator
-        script: node .github/workflow-scripts/maestro-android.js ${{ inputs.app-path }} ${{ inputs.app-id }} ${{ inputs.maestro-flow }} ${{ inputs.flavor }} ${{ inputs.working-directory }} ${{ inputs.test-state-path }}
+        script: node .github/workflow-scripts/maestro-android.js ${{ inputs.app-path }} ${{ inputs.app-id }} ${{ inputs.maestro-flow }} ${{ inputs.flavor }} ${{ inputs.working-directory }} ${{ inputs.test-state-path }} "${{ inputs.exclude-tags }}"
     - name: Normalize APP_ID
       id: normalize-app-id
       shell: bash

.github/workflow-scripts/__tests__/maestro-android-test.js

@@ -14,6 +14,7 @@ const path = require('path');
 const {
   collectFlows,
   executeFlowSuite,
+  filterFlowsByTags,
   loadState,
 } = require('../maestro-android');
 
@@ -45,6 +46,26 @@ describe('Maestro Android runner', () => {
     ]);
   });
 
+  it('excludes flows with matching tags', () => {
+    const releaseOnlyFlow = path.join(
+      temporaryDirectory,
+      'android-release-only.yml',
+    );
+    const regularFlow = path.join(temporaryDirectory, 'regular.yml');
+    fs.writeFileSync(
+      releaseOnlyFlow,
+      'appId: x\ntags:\n  - android-release-only\n---\n- launchApp\n',
+    );
+    fs.writeFileSync(regularFlow, 'appId: x\n---\n- launchApp\n');
+
+    expect(
+      filterFlowsByTags(
+        [releaseOnlyFlow, regularFlow],
+        ['android-release-only'],
+      ),
+    ).toEqual([regularFlow]);
+  });
+
   it('runs every flow and retries only flows that have not passed', () => {
     const flows = ['first.yml', 'second.yml', 'third.yml'].map(file =>
       path.join(temporaryDirectory, file),

.github/workflow-scripts/maestro-android.js

@@ -13,14 +13,15 @@ const path = require('path');
 
 const usage = `
 === Usage ===
-node maestro-android.js <path to app> <app_id> <maestro_flow> <flavor> <working_directory> [test_state_path]
+node maestro-android.js <path to app> <app_id> <maestro_flow> <flavor> <working_directory> [test_state_path] [exclude_tags]
 
 @param {string} appPath - Path to the app APK
 @param {string} appId - App ID that needs to be launched
 @param {string} maestroFlow - Path to the Maestro flow or folder to execute
 @param {string} flavor - Flavor of the app to be launched. Can be 'release' or 'debug'
 @param {string} workingDirectory - Working directory from where to run Metro
 @param {string} testStatePath - File used to persist per-flow results between CI retries
+@param {string} excludeTags - Comma-separated flow tags to exclude
 ==============
 `;
 
@@ -63,6 +64,39 @@ function collectFlows(flowPath) {
   return flows;
 }
 
+function getFlowTags(flow) {
+  const contents = fs.readFileSync(flow, 'utf8');
+  const tags = [];
+  const tagBlockPattern = /^tags:\s*\r?\n((?:^[ \t]+-[ \t]*[^\r\n]+\r?\n?)*)/gm;
+
+  for (const match of contents.matchAll(tagBlockPattern)) {
+    for (const line of match[1].split(/\r?\n/)) {
+      const tag = line.match(/^[ \t]+-[ \t]*(.+?)\s*$/)?.[1];
+      if (tag != null) {
+        tags.push(tag);
+      }
+    }
+  }
+
+  return tags;
+}
+
+function filterFlowsByTags(flows, excludeTags) {
+  if (excludeTags.length === 0) {
+    return flows;
+  }
+
+  return flows.filter(flow => {
+    const tags = getFlowTags(flow);
+    const excludedTag = tags.find(tag => excludeTags.includes(tag));
+    if (excludedTag != null) {
+      console.info(`Skipping flow tagged ${excludedTag}: ${flow}`);
+      return false;
+    }
+    return true;
+  });
+}
+
 function getFlowKey(flow) {
   return path
     .relative(process.cwd(), path.resolve(flow))
@@ -274,7 +308,7 @@ function executeFlowSuite({
 }
 
 async function main(args = process.argv.slice(2)) {
-  if (args.length < 5 || args.length > 6) {
+  if (args.length < 5 || args.length > 7) {
     throw new Error(`Invalid number of arguments.\n${usage}`);
   }
 
@@ -284,6 +318,10 @@ async function main(args = process.argv.slice(2)) {
   const isDebug = args[3] === 'debug';
   const workingDirectory = args[4];
   const statePath = args[5] ?? DEFAULT_STATE_PATH;
+  const excludeTags = (args[6] ?? '')
+    .split(',')
+    .map(tag => tag.trim())
+    .filter(Boolean);
 
   console.info('\n==============================');
   console.info('Running tests for Android with the following parameters:');
@@ -293,6 +331,7 @@ async function main(args = process.argv.slice(2)) {
   console.info(`IS_DEBUG: ${isDebug}`);
   console.info(`WORKING_DIRECTORY: ${workingDirectory}`);
   console.info(`TEST_STATE_PATH: ${statePath}`);
+  console.info(`EXCLUDE_TAGS: ${excludeTags.join(',') || '<none>'}`);
   console.info('==============================\n');
 
   logAndroidAbiConfiguration();
@@ -338,7 +377,7 @@ async function main(args = process.argv.slice(2)) {
 
   let error = null;
   try {
-    const flows = collectFlows(maestroFlow);
+    const flows = filterFlowsByTags(collectFlows(maestroFlow), excludeTags);
     const state = loadState(statePath);
     console.info(`Start testing ${flows.length} flow(s)`);
     executeFlowSuite({flows, appId, state, statePath});
@@ -378,6 +417,7 @@ if (require.main === module) {
 module.exports = {
   collectFlows,
   executeFlowSuite,
+  filterFlowsByTags,
   formatResults,
   loadState,
 };

.github/workflows/e2e-android-rntester.yml

@@ -71,6 +71,10 @@ jobs:
           emulator-api-level: '35'
           emulator-target: google_apis
           emulator-arch: x86_64
+          # These timing-sensitive flows remain covered by the release APK.
+          # Running the debug ARM64 APK through native translation can delay
+          # React commits long enough for Maestro to miss taps and assertions.
+          exclude-tags: ${{ matrix.flavor == 'debug' && 'android-release-only' || '' }}
       - name: Store per-flow test state
         if: always()
         uses: actions/upload-artifact@v6

packages/rn-tester/.maestro/flatlist-append-maintainvisible.yml

@@ -1,6 +1,8 @@
 # Test FlatList maintainVisibleContentPosition with append (baseline)
 # Appending items should NOT affect scroll offset (delta ~0)
 appId: ${APP_ID}
+tags:
+  - android-release-only
 ---
 - launchApp
 # Change to portrait

packages/rn-tester/.maestro/flatlist-complex-mutations-maintainvisible.yml

@@ -1,6 +1,8 @@
 # Test FlatList maintainVisibleContentPosition — complex concurrent mutations
 # Tests prepend + append + delete in sequence
 appId: ${APP_ID}
+tags:
+  - android-release-only
 ---
 - launchApp
 - setOrientation: portrait

packages/rn-tester/.maestro/flatlist-delete-anchor-maintainvisible.yml

@@ -1,6 +1,8 @@
 # Test FlatList maintainVisibleContentPosition — delete anchor item
 # When the anchor item (first visible) is deleted, MVCP should select a new anchor
 appId: ${APP_ID}
+tags:
+  - android-release-only
 ---
 - launchApp
 - setOrientation: portrait

packages/rn-tester/.maestro/flatlist-delete-middle-maintainvisible.yml

@@ -1,6 +1,8 @@
 # Test FlatList maintainVisibleContentPosition — delete from middle
 # When items are deleted from the middle, MVCP should adjust scroll offset
 appId: ${APP_ID}
+tags:
+  - android-release-only
 ---
 - launchApp
 - setOrientation: portrait

packages/rn-tester/.maestro/flatlist-empty-list-maintainvisible.yml

@@ -3,6 +3,8 @@
 # _firstVisibleView.frame on nil returns {0,0}, causing incorrect scroll
 # correction.
 appId: ${APP_ID}
+tags:
+  - android-release-only
 ---
 - launchApp
 # Change to portrait

packages/rn-tester/.maestro/flatlist-first-prepend-maintainvisible.yml

@@ -1,6 +1,8 @@
 # Test FlatList maintainVisibleContentPosition — first prepend only
 # Single prepend with fixed-height items: delta should be ~44px (40px height + 4px margin)
 appId: ${APP_ID}
+tags:
+  - android-release-only
 ---
 - launchApp
 # Change to portrait