diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 000000000..7a3020f50 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,217 @@ +name: E2E Tests +on: + push: + branches: + - main + pull_request: + branches: + - main + merge_group: + types: + - checks_requested + +jobs: + changes: + runs-on: ubuntu-latest + outputs: + android: ${{ steps.filter.outputs.android }} + ios: ${{ steps.filter.outputs.ios }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Check file changes + uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + android: + - 'android/**' + - 'apps/example/android/**' + - 'apps/example/src/**' + - 'src/**' + - 'cpp/**' + - 'package.json' + - 'apps/example/package.json' + - 'react-native.config.js' + - 'babel.config.js' + - '.maestro/**' + ios: + - 'ios/**' + - 'apps/example/ios/**' + - 'apps/example/src/**' + - 'src/**' + - 'cpp/**' + - '*.podspec' + - 'package.json' + - 'apps/example/package.json' + - 'react-native.config.js' + - 'babel.config.js' + - '.maestro/**' + + e2e-ios: + needs: [changes] + if: needs.changes.outputs.ios == 'true' + runs-on: macos-latest + timeout-minutes: 60 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup + uses: ./.github/actions/setup + + - name: Set up Xcode + uses: maxim-lobanov/setup-xcode@v1 + with: + xcode-version: latest-stable + + - name: Restore cocoapods + id: cocoapods-cache + uses: actions/cache/restore@v4 + with: + path: | + **/ios/Pods + key: ${{ runner.os }}-cocoapods-${{ hashFiles('apps/example/ios/Podfile.lock') }} + restore-keys: | + ${{ runner.os }}-cocoapods- + + - name: Install cocoapods + if: steps.cocoapods-cache.outputs.cache-hit != 'true' + run: | + cd apps/example/ios + pod install + env: + NO_FLIPPER: 1 + + - name: Cache cocoapods + if: steps.cocoapods-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: | + **/ios/Pods + key: ${{ steps.cocoapods-cache.outputs.cache-key }} + + - name: Restore Xcode DerivedData + uses: actions/cache/restore@v4 + id: derived-data-cache + with: + path: ~/Library/Developer/Xcode/DerivedData + key: ${{ runner.os }}-derived-data-${{ hashFiles('ios/**', 'apps/example/ios/**', 'cpp/**', 'src/**') }} + restore-keys: | + ${{ runner.os }}-derived-data- + + - name: Install Maestro CLI + run: | + curl -Ls "https://get.maestro.mobile.dev" | bash + echo "$HOME/.maestro/bin" >> $GITHUB_PATH + + - name: Start Metro + run: yarn example start & + + - name: Run E2E tests + run: yarn test:e2e:ios + + - name: Cache Xcode DerivedData + if: steps.derived-data-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: ~/Library/Developer/Xcode/DerivedData + key: ${{ steps.derived-data-cache.outputs.cache-primary-key }} + + - name: Upload test artifacts + uses: actions/upload-artifact@v4 + with: + name: e2e-ios-artifacts + path: | + .maestro/enrichedInput/screenshots/ios/*_diff.png + .maestro/enrichedText/screenshots/ios/*_diff.png + + e2e-android: + needs: [changes] + if: needs.changes.outputs.android == 'true' + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup + uses: ./.github/actions/setup + + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + - name: Install JDK + uses: actions/setup-java@v4 + with: + distribution: 'zulu' + java-version: '17' + + - name: Install Maestro CLI + run: | + curl -Ls "https://get.maestro.mobile.dev" | bash + echo "$HOME/.maestro/bin" >> $GITHUB_PATH + + - name: Cache Gradle + uses: actions/cache@v4 + with: + path: | + ~/.gradle/wrapper + ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('apps/example/android/gradle/wrapper/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: AVD cache + uses: actions/cache@v4 + id: avd-cache + with: + path: | + ~/.android/avd/* + ~/.android/adb* + key: avd-api-36-x86_64 + + - name: Create AVD and generate snapshot for caching + if: steps.avd-cache.outputs.cache-hit != 'true' + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 36 + target: google_apis_playstore + arch: x86_64 + profile: pixel_7 + force-avd-creation: false + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim + disable-animations: false + enable-hw-keyboard: true + script: echo "Generated AVD snapshot for caching." + + - name: Start Metro + run: yarn example start & + + - name: Run E2E tests + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 36 + target: google_apis_playstore + arch: x86_64 + profile: pixel_7 + force-avd-creation: false + emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim + disable-animations: true + disable-spellchecker: true + enable-hw-keyboard: true + script: EMULATOR_SERIAL=emulator-5554 .maestro/scripts/run-tests-ci-android.sh + env: + JAVA_OPTS: '-XX:MaxHeapSize=6g' + + - name: Upload test artifacts + uses: actions/upload-artifact@v4 + with: + name: e2e-android-artifacts + path: | + .maestro/enrichedInput/screenshots/android/*_diff.png + .maestro/enrichedText/screenshots/android/*_diff.png diff --git a/.maestro/enrichedInput/screenshots/android/empty_element_parsing.png b/.maestro/enrichedInput/screenshots/android/empty_element_parsing.png index 7e28ad4ea..04053f6aa 100644 Binary files a/.maestro/enrichedInput/screenshots/android/empty_element_parsing.png and b/.maestro/enrichedInput/screenshots/android/empty_element_parsing.png differ diff --git a/.maestro/enrichedInput/screenshots/android/paragraph_styles_no_crash.png b/.maestro/enrichedInput/screenshots/android/paragraph_styles_no_crash.png index 3278d7428..5bf918910 100644 Binary files a/.maestro/enrichedInput/screenshots/android/paragraph_styles_no_crash.png and b/.maestro/enrichedInput/screenshots/android/paragraph_styles_no_crash.png differ diff --git a/.maestro/enrichedInput/screenshots/android/scrolling_paragraph_styles_top.png b/.maestro/enrichedInput/screenshots/android/scrolling_paragraph_styles_top.png index 3b946339f..f46c61242 100644 Binary files a/.maestro/enrichedInput/screenshots/android/scrolling_paragraph_styles_top.png and b/.maestro/enrichedInput/screenshots/android/scrolling_paragraph_styles_top.png differ diff --git a/.maestro/scripts/run-tests-ci-android.sh b/.maestro/scripts/run-tests-ci-android.sh new file mode 100755 index 000000000..cf3162bdd --- /dev/null +++ b/.maestro/scripts/run-tests-ci-android.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# run-tests-ci-android.sh - CI-only script for running Android E2E tests. +# +# Designed to run inside reactivecircus/android-emulator-runner's `script` context +# where the emulator is already booted. Builds the app, installs it, and runs +# maestro tests. +# +# The emulator serial is passed via EMULATOR_SERIAL (defaults to emulator-5554). + +set -euo pipefail + +MIN_MAESTRO_VERSION="2.3.0" + +if ! command -v maestro >/dev/null 2>&1; then + echo "Error: maestro CLI not found." >&2 + exit 1 +fi + +MAESTRO_VERSION=$(maestro --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1) +if [ "$(printf '%s\n' "$MIN_MAESTRO_VERSION" "$MAESTRO_VERSION" | sort -V | head -n1)" != "$MIN_MAESTRO_VERSION" ]; then + echo "Error: maestro $MAESTRO_VERSION is too old, minimum required is $MIN_MAESTRO_VERSION" >&2 + exit 1 +fi + +SERIAL="${EMULATOR_SERIAL:-emulator-5554}" + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +MAESTRO_ROOT="$REPO_ROOT/.maestro" +SCREENSHOT_ROOT="$MAESTRO_ROOT" + +echo "=== Waiting for emulator to be ready ===" +adb -s "$SERIAL" wait-for-device +until adb -s "$SERIAL" shell getprop sys.boot_completed 2>/dev/null | grep -q "^1$"; do + sleep 2 +done + +adb -s "$SERIAL" shell pm disable-user --user 0 com.google.android.inputmethod.latin 2>/dev/null || true +adb -s "$SERIAL" shell settings put secure spell_checker_enabled 0 + +echo "=== Building and installing app ===" +yarn example android --device "$SERIAL" + +set_font_scale() { + case "$1" in + default) adb -s "$SERIAL" shell settings put system font_scale 1.0 ;; + large) adb -s "$SERIAL" shell settings put system font_scale 1.5 ;; + esac +} + +trap 'set_font_scale default' EXIT +set_font_scale default + +FLOWS=".maestro/enrichedInput/flows .maestro/enrichedText/flows" +ASSETS_DIR="$MAESTRO_ROOT/assets" +[ -d "$ASSETS_DIR" ] && FLOWS="$ASSETS_DIR $FLOWS" + +EXTRA="--env SCREENSHOT_ROOT=$SCREENSHOT_ROOT --exclude-tags ios-only" + +run_maestro() { + local tmp rc + tmp=$(mktemp) + local cmd + cmd=$(printf '%q ' maestro test "$@") + script -qec "$cmd" /dev/null 2>&1 | tee "$tmp" + rc=${PIPESTATUS[0]} + if [ "$rc" -ne 0 ] && grep -q "did not match any Flows" "$tmp"; then + echo "warn: no flows matched the tag filter — treating as success" >&2 + rc=0 + fi + rm -f "$tmp" + return "$rc" +} + +set +e + +echo "=== Running maestro tests ===" +# shellcheck disable=SC2086 +run_maestro --device "$SERIAL" --exclude-tags accessibility $EXTRA $FLOWS +EXIT_REGULAR=$? + +echo "=== Running maestro accessibility tests ===" +set_font_scale large +# shellcheck disable=SC2086 +run_maestro --device "$SERIAL" --include-tags accessibility $EXTRA $FLOWS +EXIT_A11Y=$? + +set -e + +exit $(( EXIT_REGULAR != 0 || EXIT_A11Y != 0 )) diff --git a/.maestro/scripts/run-tests.sh b/.maestro/scripts/run-tests.sh index afaf96887..48e9fe296 100755 --- a/.maestro/scripts/run-tests.sh +++ b/.maestro/scripts/run-tests.sh @@ -26,7 +26,7 @@ if ! command -v maestro >/dev/null 2>&1; then exit 1 fi -MAESTRO_VERSION=$(maestro --version) +MAESTRO_VERSION=$(maestro --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1) # Compare versions by sorting them; if the minimum sorts after the actual, it's too old. if [ "$(printf '%s\n' "$MIN_MAESTRO_VERSION" "$MAESTRO_VERSION" | sort -V | head -n1)" != "$MIN_MAESTRO_VERSION" ]; then echo "Error: maestro $MAESTRO_VERSION is too old, minimum required is $MIN_MAESTRO_VERSION" >&2 @@ -61,7 +61,7 @@ case "$PLATFORM" in *) echo "Error: --platform must be ios or android" >&2; exit 1 ;; esac -DEVICE_ID=$("$SETUP" | tee /dev/tty | grep "^DEVICE_ID=" | cut -d= -f2) +DEVICE_ID=$("$SETUP" | tee /dev/stderr | grep "^DEVICE_ID=" | cut -d= -f2) app_installed() { if [ "$PLATFORM" = ios ]; then diff --git a/.maestro/scripts/setup-android-emulator.sh b/.maestro/scripts/setup-android-emulator.sh index 605c8f6bd..e63b0dd84 100755 --- a/.maestro/scripts/setup-android-emulator.sh +++ b/.maestro/scripts/setup-android-emulator.sh @@ -2,7 +2,7 @@ set -euo pipefail API_LEVEL="36" -DEVICE_ID="pixel_9" +DEVICE_ID="pixel_7" ARCH=$(uname -m) if [ "$ARCH" = "arm64" ] || [ "$ARCH" = "aarch64" ]; then ABI="arm64-v8a" @@ -11,7 +11,7 @@ else fi TAG="google_apis_playstore" SYSTEM_IMAGE="system-images;android-${API_LEVEL};${TAG};${ABI}" -AVD_NAME="Pixel9-API${API_LEVEL}-Enriched" +AVD_NAME="Pixel7-API${API_LEVEL}-Enriched" PORT=5570 SERIAL="emulator-${PORT}" @@ -20,6 +20,11 @@ if [ -z "$ANDROID_HOME" ]; then exit 1 fi +# Ensure avdmanager and emulator use the same AVD directory regardless of +# what ANDROID_SDK_HOME is set to on the host (e.g. GitHub Actions runners). +export ANDROID_AVD_HOME="$HOME/.android/avd" +mkdir -p "$ANDROID_AVD_HOME" + for tool in sdkmanager avdmanager emulator adb; do if ! command -v "$tool" &>/dev/null; then echo "Error: '$tool' not found. Ensure Android SDK tools are installed and in PATH." @@ -41,19 +46,20 @@ fi if ! avdmanager list avd -c | grep -qx "${AVD_NAME}"; then echo "Creating AVD '$AVD_NAME'..." - echo "no" | avdmanager create avd \ - --name "$AVD_NAME" \ - --device "$DEVICE_ID" \ - --package "$SYSTEM_IMAGE" \ - --skin "$DEVICE_ID" + CREATE_CMD=(avdmanager create avd --name "$AVD_NAME" --device "$DEVICE_ID" --package "$SYSTEM_IMAGE") + # Skin is cosmetic (phone frame). Skip it on CI since the runner has no skin files + # and the emulator runs headless anyway. + [ -z "${CI:-}" ] && CREATE_CMD+=(--skin "$DEVICE_ID") + echo "no" | "${CREATE_CMD[@]}" fi AVD_CONFIG="$HOME/.android/avd/${AVD_NAME}.avd/config.ini" if [ -f "$AVD_CONFIG" ]; then - sed -i '' 's/^hw\.keyboard=.*/hw.keyboard=yes/' "$AVD_CONFIG" + sed -i.bak 's/^hw\.keyboard=.*/hw.keyboard=yes/' "$AVD_CONFIG" grep -q "^hw.keyboard=" "$AVD_CONFIG" || echo "hw.keyboard=yes" >> "$AVD_CONFIG" - sed -i '' 's/^hw\.mainKeys=.*/hw.mainKeys=yes/' "$AVD_CONFIG" + sed -i.bak 's/^hw\.mainKeys=.*/hw.mainKeys=yes/' "$AVD_CONFIG" grep -q "^hw.mainKeys=" "$AVD_CONFIG" || echo "hw.mainKeys=yes" >> "$AVD_CONFIG" + rm -f "$AVD_CONFIG.bak" fi if pgrep -f "emulator.*${AVD_NAME}" > /dev/null 2>&1; then @@ -63,7 +69,12 @@ if pgrep -f "emulator.*${AVD_NAME}" > /dev/null 2>&1; then fi echo "Starting emulator '$AVD_NAME'..." -emulator "@${AVD_NAME}" -port "$PORT" > /dev/null 2>&1 & +EMULATOR_ARGS=("@${AVD_NAME}" -port "$PORT") +if [ -n "${CI:-}" ]; then + EMULATOR_ARGS+=(-no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim) +fi + +emulator "${EMULATOR_ARGS[@]}" > /dev/null 2>&1 & echo "Waiting for emulator ($SERIAL) to connect to ADB..." if ! timeout 120 adb -s "$SERIAL" wait-for-device; then diff --git a/.maestro/scripts/setup-ios-simulator.sh b/.maestro/scripts/setup-ios-simulator.sh index 5f9d8ea23..16cf16aee 100755 --- a/.maestro/scripts/setup-ios-simulator.sh +++ b/.maestro/scripts/setup-ios-simulator.sh @@ -46,7 +46,9 @@ if [ "$STATE" != "(Booted)" ]; then xcrun simctl boot "$UDID" fi -open -a Simulator +if [ -z "${CI:-}" ]; then + open -a Simulator +fi echo "Simulator ready: $DEVICE_NAME ($UDID)" echo "DEVICE_ID=$UDID" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f16348c2a..df413b7c8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,7 +92,7 @@ The target devices are: | Platform | Device | OS | | -------- | --------- | ----------------------------- | | iOS | iPhone 17 | iOS 26.2 | -| Android | Pixel 9 | API 36 "Baklava" (Android 16) | +| Android | Pixel 7 | API 36 "Baklava" (Android 16) | #### Running E2E tests