Draft: chore: add maestro tests to CI

.github/workflows/e2e.yml

@@ -0,0 +1,217 @@
+name: E2E Tests
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  merge_group:
+    types:
+      - checks_requested
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      android: ${{ steps.filter.outputs.android }}
+      ios: ${{ steps.filter.outputs.ios }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Check file changes
+        uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            android:
+              - 'android/**'
+              - 'apps/example/android/**'
+              - 'apps/example/src/**'
+              - 'src/**'
+              - 'cpp/**'
+              - 'package.json'
+              - 'apps/example/package.json'
+              - 'react-native.config.js'
+              - 'babel.config.js'
+              - '.maestro/**'
+            ios:
+              - 'ios/**'
+              - 'apps/example/ios/**'
+              - 'apps/example/src/**'
+              - 'src/**'
+              - 'cpp/**'
+              - '*.podspec'
+              - 'package.json'
+              - 'apps/example/package.json'
+              - 'react-native.config.js'
+              - 'babel.config.js'
+              - '.maestro/**'
+
+  e2e-ios:
+    needs: [changes]
+    if: needs.changes.outputs.ios == 'true'
+    runs-on: macos-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup
+        uses: ./.github/actions/setup
+
+      - name: Set up Xcode
+        uses: maxim-lobanov/setup-xcode@v1
+        with:
+          xcode-version: latest-stable
+
+      - name: Restore cocoapods
+        id: cocoapods-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            **/ios/Pods
+          key: ${{ runner.os }}-cocoapods-${{ hashFiles('apps/example/ios/Podfile.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cocoapods-
+
+      - name: Install cocoapods
+        if: steps.cocoapods-cache.outputs.cache-hit != 'true'
+        run: |
+          cd apps/example/ios
+          pod install
+        env:
+          NO_FLIPPER: 1
+
+      - name: Cache cocoapods
+        if: steps.cocoapods-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            **/ios/Pods
+          key: ${{ steps.cocoapods-cache.outputs.cache-key }}
+
+      - name: Restore Xcode DerivedData
+        uses: actions/cache/restore@v4
+        id: derived-data-cache
+        with:
+          path: ~/Library/Developer/Xcode/DerivedData
+          key: ${{ runner.os }}-derived-data-${{ hashFiles('ios/**', 'apps/example/ios/**', 'cpp/**', 'src/**') }}
+          restore-keys: |
+            ${{ runner.os }}-derived-data-
+
+      - name: Install Maestro CLI
+        run: |
+          curl -Ls "https://get.maestro.mobile.dev" | bash
+          echo "$HOME/.maestro/bin" >> $GITHUB_PATH
+
+      - name: Start Metro
+        run: yarn example start &
+
+      - name: Run E2E tests
+        run: yarn test:e2e:ios
+
+      - name: Cache Xcode DerivedData
+        if: steps.derived-data-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: ~/Library/Developer/Xcode/DerivedData
+          key: ${{ steps.derived-data-cache.outputs.cache-primary-key }}
+
+      - name: Upload test artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-ios-artifacts
+          path: |
+            .maestro/enrichedInput/screenshots/ios/*_diff.png
+            .maestro/enrichedText/screenshots/ios/*_diff.png
+
+  e2e-android:
+    needs: [changes]
+    if: needs.changes.outputs.android == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup
+        uses: ./.github/actions/setup
+
+      - name: Enable KVM
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+
+      - name: Install JDK
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'zulu'
+          java-version: '17'
+
+      - name: Install Maestro CLI
+        run: |
+          curl -Ls "https://get.maestro.mobile.dev" | bash
+          echo "$HOME/.maestro/bin" >> $GITHUB_PATH
+
+      - name: Cache Gradle
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.gradle/wrapper
+            ~/.gradle/caches
+          key: ${{ runner.os }}-gradle-${{ hashFiles('apps/example/android/gradle/wrapper/gradle-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+
+      - name: AVD cache
+        uses: actions/cache@v4
+        id: avd-cache
+        with:
+          path: |
+            ~/.android/avd/*
+            ~/.android/adb*
+          key: avd-api-36-x86_64
+
+      - name: Create AVD and generate snapshot for caching
+        if: steps.avd-cache.outputs.cache-hit != 'true'
+        uses: reactivecircus/android-emulator-runner@v2
+        with:
+          api-level: 36
+          target: google_apis_playstore
+          arch: x86_64
+          profile: pixel_7
+          force-avd-creation: false
+          emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim
+          disable-animations: false
+          enable-hw-keyboard: true
+          script: echo "Generated AVD snapshot for caching."
+
+      - name: Start Metro
+        run: yarn example start &
+
+      - name: Run E2E tests
+        uses: reactivecircus/android-emulator-runner@v2
+        with:
+          api-level: 36
+          target: google_apis_playstore
+          arch: x86_64
+          profile: pixel_7
+          force-avd-creation: false
+          emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim
+          disable-animations: true
+          disable-spellchecker: true
+          enable-hw-keyboard: true
+          script: EMULATOR_SERIAL=emulator-5554 .maestro/scripts/run-tests-ci-android.sh
+        env:
+          JAVA_OPTS: '-XX:MaxHeapSize=6g'
+
+      - name: Upload test artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-android-artifacts
+          path: |
+            .maestro/enrichedInput/screenshots/android/*_diff.png
+            .maestro/enrichedText/screenshots/android/*_diff.png

.maestro/scripts/run-tests-ci-android.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+# run-tests-ci-android.sh - CI-only script for running Android E2E tests.
+#
+# Designed to run inside reactivecircus/android-emulator-runner's `script` context
+# where the emulator is already booted. Builds the app, installs it, and runs
+# maestro tests.
+#
+# The emulator serial is passed via EMULATOR_SERIAL (defaults to emulator-5554).
+
+set -euo pipefail
+
+MIN_MAESTRO_VERSION="2.3.0"
+
+if ! command -v maestro >/dev/null 2>&1; then
+  echo "Error: maestro CLI not found." >&2
+  exit 1
+fi
+
+MAESTRO_VERSION=$(maestro --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)
+if [ "$(printf '%s\n' "$MIN_MAESTRO_VERSION" "$MAESTRO_VERSION" | sort -V | head -n1)" != "$MIN_MAESTRO_VERSION" ]; then
+  echo "Error: maestro $MAESTRO_VERSION is too old, minimum required is $MIN_MAESTRO_VERSION" >&2
+  exit 1
+fi
+
+SERIAL="${EMULATOR_SERIAL:-emulator-5554}"
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+MAESTRO_ROOT="$REPO_ROOT/.maestro"
+SCREENSHOT_ROOT="$MAESTRO_ROOT"
+
+echo "=== Waiting for emulator to be ready ==="
+adb -s "$SERIAL" wait-for-device
+until adb -s "$SERIAL" shell getprop sys.boot_completed 2>/dev/null | grep -q "^1$"; do
+  sleep 2
+done
+
+adb -s "$SERIAL" shell pm disable-user --user 0 com.google.android.inputmethod.latin 2>/dev/null || true
+adb -s "$SERIAL" shell settings put secure spell_checker_enabled 0
+
+echo "=== Building and installing app ==="
+yarn example android --device "$SERIAL"
+
+set_font_scale() {
+  case "$1" in
+    default) adb -s "$SERIAL" shell settings put system font_scale 1.0 ;;
+    large)   adb -s "$SERIAL" shell settings put system font_scale 1.5 ;;
+  esac
+}
+
+trap 'set_font_scale default' EXIT
+set_font_scale default
+
+FLOWS=".maestro/enrichedInput/flows .maestro/enrichedText/flows"
+ASSETS_DIR="$MAESTRO_ROOT/assets"
+[ -d "$ASSETS_DIR" ] && FLOWS="$ASSETS_DIR $FLOWS"
+
+EXTRA="--env SCREENSHOT_ROOT=$SCREENSHOT_ROOT --exclude-tags ios-only"
+
+run_maestro() {
+  local tmp rc
+  tmp=$(mktemp)
+  local cmd
+  cmd=$(printf '%q ' maestro test "$@")
+  script -qec "$cmd" /dev/null 2>&1 | tee "$tmp"
+  rc=${PIPESTATUS[0]}
+  if [ "$rc" -ne 0 ] && grep -q "did not match any Flows" "$tmp"; then
+    echo "warn: no flows matched the tag filter — treating as success" >&2
+    rc=0
+  fi
+  rm -f "$tmp"
+  return "$rc"
+}
+
+set +e
+
+echo "=== Running maestro tests ==="
+# shellcheck disable=SC2086
+run_maestro --device "$SERIAL" --exclude-tags accessibility $EXTRA $FLOWS
+EXIT_REGULAR=$?
+
+echo "=== Running maestro accessibility tests ==="
+set_font_scale large
+# shellcheck disable=SC2086
+run_maestro --device "$SERIAL" --include-tags accessibility $EXTRA $FLOWS
+EXIT_A11Y=$?
+
+set -e
+
+exit $(( EXIT_REGULAR != 0 || EXIT_A11Y != 0 ))

.maestro/scripts/run-tests.sh

@@ -26,7 +26,7 @@ if ! command -v maestro >/dev/null 2>&1; then
   exit 1
 fi
 
-MAESTRO_VERSION=$(maestro --version)
+MAESTRO_VERSION=$(maestro --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)
 # Compare versions by sorting them; if the minimum sorts after the actual, it's too old.
 if [ "$(printf '%s\n' "$MIN_MAESTRO_VERSION" "$MAESTRO_VERSION" | sort -V | head -n1)" != "$MIN_MAESTRO_VERSION" ]; then
   echo "Error: maestro $MAESTRO_VERSION is too old, minimum required is $MIN_MAESTRO_VERSION" >&2
@@ -61,7 +61,7 @@ case "$PLATFORM" in
   *)       echo "Error: --platform must be ios or android" >&2; exit 1 ;;
 esac
 
-DEVICE_ID=$("$SETUP" | tee /dev/tty | grep "^DEVICE_ID=" | cut -d= -f2)
+DEVICE_ID=$("$SETUP" | tee /dev/stderr | grep "^DEVICE_ID=" | cut -d= -f2)
 
 app_installed() {
   if [ "$PLATFORM" = ios ]; then

.maestro/scripts/setup-android-emulator.sh

@@ -2,7 +2,7 @@
 set -euo pipefail
 
 API_LEVEL="36"
-DEVICE_ID="pixel_9"
+DEVICE_ID="pixel_7"
 ARCH=$(uname -m)
 if [ "$ARCH" = "arm64" ] || [ "$ARCH" = "aarch64" ]; then
   ABI="arm64-v8a"
@@ -11,7 +11,7 @@ else
 fi
 TAG="google_apis_playstore"
 SYSTEM_IMAGE="system-images;android-${API_LEVEL};${TAG};${ABI}"
-AVD_NAME="Pixel9-API${API_LEVEL}-Enriched"
+AVD_NAME="Pixel7-API${API_LEVEL}-Enriched"
 PORT=5570
 SERIAL="emulator-${PORT}"
 
@@ -20,6 +20,11 @@ if [ -z "$ANDROID_HOME" ]; then
   exit 1
 fi
 
+# Ensure avdmanager and emulator use the same AVD directory regardless of
+# what ANDROID_SDK_HOME is set to on the host (e.g. GitHub Actions runners).
+export ANDROID_AVD_HOME="$HOME/.android/avd"
+mkdir -p "$ANDROID_AVD_HOME"
+
 for tool in sdkmanager avdmanager emulator adb; do
   if ! command -v "$tool" &>/dev/null; then
     echo "Error: '$tool' not found. Ensure Android SDK tools are installed and in PATH."
@@ -41,19 +46,20 @@ fi
 
 if ! avdmanager list avd -c | grep -qx "${AVD_NAME}"; then
   echo "Creating AVD '$AVD_NAME'..."
-  echo "no" | avdmanager create avd \
-    --name "$AVD_NAME" \
-    --device "$DEVICE_ID" \
-    --package "$SYSTEM_IMAGE" \
-    --skin "$DEVICE_ID"
+  CREATE_CMD=(avdmanager create avd --name "$AVD_NAME" --device "$DEVICE_ID" --package "$SYSTEM_IMAGE")
+  # Skin is cosmetic (phone frame). Skip it on CI since the runner has no skin files
+  # and the emulator runs headless anyway.
+  [ -z "${CI:-}" ] && CREATE_CMD+=(--skin "$DEVICE_ID")
+  echo "no" | "${CREATE_CMD[@]}"
 fi
 
 AVD_CONFIG="$HOME/.android/avd/${AVD_NAME}.avd/config.ini"
 if [ -f "$AVD_CONFIG" ]; then
-  sed -i '' 's/^hw\.keyboard=.*/hw.keyboard=yes/' "$AVD_CONFIG"
+  sed -i.bak 's/^hw\.keyboard=.*/hw.keyboard=yes/' "$AVD_CONFIG"
   grep -q "^hw.keyboard=" "$AVD_CONFIG" || echo "hw.keyboard=yes" >> "$AVD_CONFIG"
-  sed -i '' 's/^hw\.mainKeys=.*/hw.mainKeys=yes/' "$AVD_CONFIG"
+  sed -i.bak 's/^hw\.mainKeys=.*/hw.mainKeys=yes/' "$AVD_CONFIG"
   grep -q "^hw.mainKeys=" "$AVD_CONFIG" || echo "hw.mainKeys=yes" >> "$AVD_CONFIG"
+  rm -f "$AVD_CONFIG.bak"
 fi
 
 if pgrep -f "emulator.*${AVD_NAME}" > /dev/null 2>&1; then
@@ -63,7 +69,12 @@ if pgrep -f "emulator.*${AVD_NAME}" > /dev/null 2>&1; then
 fi
 
 echo "Starting emulator '$AVD_NAME'..."
-emulator "@${AVD_NAME}" -port "$PORT" > /dev/null 2>&1 &
+EMULATOR_ARGS=("@${AVD_NAME}" -port "$PORT")
+if [ -n "${CI:-}" ]; then
+  EMULATOR_ARGS+=(-no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim)
+fi
+
+emulator "${EMULATOR_ARGS[@]}" > /dev/null 2>&1 &
 
 echo "Waiting for emulator ($SERIAL) to connect to ADB..."
 if ! timeout 120 adb -s "$SERIAL" wait-for-device; then