diff --git a/.github/workflows/android-appcenter.yml b/.github/workflows/android-browserstack.yml similarity index 62% rename from .github/workflows/android-appcenter.yml rename to .github/workflows/android-browserstack.yml index 608a83be..a10443ce 100644 --- a/.github/workflows/android-appcenter.yml +++ b/.github/workflows/android-browserstack.yml @@ -1,4 +1,4 @@ -name: Android AppCenter Tests +name: Android BrowserStack Tests on: workflow_dispatch: @@ -6,13 +6,13 @@ on: branches: [ main ] paths: - 'binding/android/CobraTestApp/cobra-test-app/build.gradle' - - '.github/workflows/android-appcenter.yml' + - '.github/workflows/android-browserstack.yml' pull_request: branches: [ main, 'v[0-9]+.[0-9]+' ] paths: - 'binding/android/CobraTestApp/cobra-test-app/build.gradle' - - '.github/workflows/android-appcenter.yml' + - '.github/workflows/android-browserstack.yml' defaults: run: @@ -20,19 +20,18 @@ defaults: jobs: build: - name: Run Android Tests on AppCenter + name: Run Android Tests on BrowserStack runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Node.js LTS - uses: actions/setup-node@v3 + - name: Installing Python + uses: actions/setup-python@v5 with: - node-version: lts/* - - - name: Install AppCenter CLI - run: npm install -g appcenter-cli + python-version: '3.10' + - run: + pip3 install requests - name: set up JDK 11 uses: actions/setup-java@v3 @@ -62,30 +61,28 @@ jobs: - name: Build androidTest run: ./gradlew assembleAndroidTest - - name: Run tests on AppCenter - run: appcenter test run espresso - --token ${{secrets.APPCENTERAPITOKEN}} - --app "Picovoice/Cobra-Android-Activity" - --devices "Picovoice/android-min-max" - --app-path cobra-test-app/build/outputs/apk/debug/cobra-test-app-debug.apk - --test-series "cobra-android" - --locale "en_US" - --build-dir cobra-test-app/build/outputs/apk/androidTest/debug + - name: Run tests on BrowserStack + run: python3 ../../../script/automation/browserstack.py + --type espresso + --username "${{secrets.BROWSERSTACK_USERNAME}}" + --access_key "${{secrets.BROWSERSTACK_ACCESS_KEY}}" + --project_name "Cobra-Migration-Test-Android-Build" + --app_path "cobra-test-app/build/outputs/apk/debug/cobra-test-app-debug.apk" + --test_path "cobra-test-app/build/outputs/apk/androidTest/debug/cobra-test-app-debug-androidTest.apk" build-integ: - name: Run Android Integration Tests on AppCenter + name: Run Android Integration Tests on BrowserStack runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Node.js LTS - uses: actions/setup-node@v3 + - name: Installing Python + uses: actions/setup-python@v5 with: - node-version: lts/* - - - name: Install AppCenter CLI - run: npm install -g appcenter-cli + python-version: '3.10' + - run: + pip3 install requests - name: set up JDK 11 uses: actions/setup-java@v3 @@ -118,12 +115,11 @@ jobs: - name: Build androidTest run: ./gradlew assembleReleaseAndroidTest -DtestBuildType=integ - - name: Run tests on AppCenter - run: appcenter test run espresso - --token ${{secrets.APPCENTERAPITOKEN}} - --app "Picovoice/Cobra-Android-Activity" - --devices "Picovoice/android-min-max" - --app-path cobra-test-app/build/outputs/apk/release/cobra-test-app-release.apk - --test-series "cobra-android" - --locale "en_US" - --build-dir cobra-test-app/build/outputs/apk/androidTest/release + - name: Run tests on BrowserStack + run: python3 ../../../script/automation/browserstack.py + --type espresso + --username "${{secrets.BROWSERSTACK_USERNAME}}" + --access_key "${{secrets.BROWSERSTACK_ACCESS_KEY}}" + --project_name "Cobra-Migration-Test-Android-Build-Integ" + --app_path "cobra-test-app/build/outputs/apk/release/cobra-test-app-release.apk" + --test_path "cobra-test-app/build/outputs/apk/androidTest/release/cobra-test-app-release-androidTest.apk" diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml index 85ec23b2..df5d3b61 100644 --- a/.github/workflows/android-perf.yml +++ b/.github/workflows/android-perf.yml @@ -19,7 +19,7 @@ defaults: jobs: build: - name: Run Android Speed Tests on AppCenter + name: Run Android Speed Tests on BrowserStack runs-on: ubuntu-latest strategy: @@ -34,13 +34,12 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Node.js LTS - uses: actions/setup-node@v3 + - name: Installing Python + uses: actions/setup-python@v5 with: - node-version: lts/* - - - name: Install AppCenter CLI - run: npm install -g appcenter-cli + python-version: '3.10' + - run: + pip3 install requests - name: set up JDK 11 uses: actions/setup-java@v3 @@ -79,12 +78,11 @@ jobs: - name: Build androidTest run: ./gradlew assembleAndroidTest -DtestBuildType=perf - - name: Run tests on AppCenter - run: appcenter test run espresso - --token ${{secrets.APPCENTERAPITOKEN}} - --app "Picovoice/Cobra-Android-Activity" - --devices "Picovoice/${{ matrix.device }}" - --app-path cobra-test-app/build/outputs/apk/debug/cobra-test-app-debug.apk - --test-series "cobra-android" - --locale "en_US" - --build-dir cobra-test-app/build/outputs/apk/androidTest/debug + - name: Run tests on BrowserStack + run: python3 ../../../script/automation/browserstack.py + --type espresso + --username "${{secrets.BROWSERSTACK_USERNAME}}" + --access_key "${{secrets.BROWSERSTACK_ACCESS_KEY}}" + --project_name "Cobra-Migration-Test-Android-Perf" + --app_path "cobra-test-app/build/outputs/apk/debug/cobra-test-app-debug.apk" + --test_path "cobra-test-app/build/outputs/apk/androidTest/debug/cobra-test-app-debug-androidTest.apk" diff --git a/.github/workflows/ios-appcenter.yml b/.github/workflows/ios-appcenter.yml deleted file mode 100644 index 651441ad..00000000 --- a/.github/workflows/ios-appcenter.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: iOS AppCenter Tests - -on: - workflow_dispatch: - push: - branches: [ main ] - paths: - - '.github/workflows/ios-appcenter.yml' - - pull_request: - branches: [ main, 'v[0-9]+.[0-9]+' ] - paths: - - '.github/workflows/ios-appcenter.yml' - - -defaults: - run: - working-directory: binding/ios/CobraAppTest - -jobs: - build: - name: Run iOS Tests on AppCenter - runs-on: macos-latest - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Node.js LTS - uses: actions/setup-node@v3 - with: - node-version: lts/* - - - name: Install AppCenter CLI - run: npm install -g appcenter-cli - - - name: Make build dir - run: mkdir ddp - - - name: Inject AccessKey - run: sed -i '.bak' 's:{TESTING_ACCESS_KEY_HERE}:${{secrets.PV_VALID_ACCESS_KEY}}:' - CobraAppTestUITests/CobraAppTestUITests.swift - - - name: XCode Build - run: xcrun xcodebuild build-for-testing - -configuration Debug - -project CobraAppTest.xcodeproj - -sdk iphoneos - -scheme CobraAppTest - -derivedDataPath ddp - CODE_SIGNING_ALLOWED=NO - - - name: Run Tests on AppCenter - run: appcenter test run xcuitest - --token ${{secrets.APPCENTERAPITOKEN}} - --app "Picovoice/Cobra-iOS" - --devices "Picovoice/ios-min-max" - --test-series "cobra-ios" - --locale "en_US" - --build-dir ddp/Build/Products/Debug-iphoneos diff --git a/.github/workflows/ios-browserstack.yml b/.github/workflows/ios-browserstack.yml new file mode 100644 index 00000000..4af942a3 --- /dev/null +++ b/.github/workflows/ios-browserstack.yml @@ -0,0 +1,70 @@ +name: iOS BrowserStack Tests + +on: + workflow_dispatch: + push: + branches: [ main ] + paths: + - '.github/workflows/ios-browserstack.yml' + + pull_request: + branches: [ main, 'v[0-9]+.[0-9]+' ] + paths: + - '.github/workflows/ios-browserstack.yml' + + +defaults: + run: + working-directory: binding/ios/CobraAppTest + +jobs: + build: + name: Run iOS Tests on BrowserStack + runs-on: macos-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Installing Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - run: + pip3 install requests + + - name: Make build dir + run: mkdir ddp + + - name: Inject AccessKey + run: sed -i '.bak' 's:{TESTING_ACCESS_KEY_HERE}:${{secrets.PV_VALID_ACCESS_KEY}}:' + CobraAppTestUITests/CobraAppTestUITests.swift + + - name: XCode Build + run: xcrun xcodebuild build-for-testing + -configuration Debug + -project CobraAppTest.xcodeproj + -sdk iphoneos + -scheme CobraAppTest + -derivedDataPath ddp + CODE_SIGNING_ALLOWED=NO + + - name: Generating ipa + run: cd ddp/Build/Products/Debug-iphoneos/ && + mkdir Payload && + cp -r CobraAppTest.app Payload && + zip --symlinks -r CobraAppTest.ipa Payload && + rm -r Payload + + - name: Zipping Tests + run: cd ddp/Build/Products/Debug-iphoneos/ && + zip --symlinks -r CobraAppTestUITests.zip CobraAppTestUITests-Runner.app + + - name: Run tests on BrowserStack + run: python3 ../../../script/automation/browserstack.py + --type xcuitest + --username "${{secrets.BROWSERSTACK_USERNAME}}" + --access_key "${{secrets.BROWSERSTACK_ACCESS_KEY}}" + --project_name "Cobra-Migration-Test-iOS" + --app_path "ddp/Build/Products/Debug-iphoneos/CobraAppTest.ipa" + --test_path "ddp/Build/Products/Debug-iphoneos/CobraAppTestUITests.zip" diff --git a/.github/workflows/ios-perf.yml b/.github/workflows/ios-perf.yml index 415e7c34..3da6cc86 100644 --- a/.github/workflows/ios-perf.yml +++ b/.github/workflows/ios-perf.yml @@ -19,7 +19,7 @@ defaults: jobs: build: - name: Run iOS Tests on AppCenter + name: Run iOS Tests on BrowserStack runs-on: macos-latest strategy: @@ -33,13 +33,12 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Set up Node.js LTS - uses: actions/setup-node@v3 + - name: Installing Python + uses: actions/setup-python@v5 with: - node-version: lts/* - - - name: Install AppCenter CLI - run: npm install -g appcenter-cli + python-version: '3.10' + - run: + pip3 install requests - name: Make build dir run: mkdir ddp @@ -65,11 +64,22 @@ jobs: -derivedDataPath ddp CODE_SIGNING_ALLOWED=NO - - name: Run Tests on AppCenter - run: appcenter test run xcuitest - --token ${{secrets.APPCENTERAPITOKEN}} - --app "Picovoice/Cobra-iOS" - --devices "Picovoice/${{ matrix.device }}" - --test-series "cobra-ios" - --locale "en_US" - --build-dir ddp/Build/Products/Debug-iphoneos \ No newline at end of file + - name: Generating ipa + run: cd ddp/Build/Products/Debug-iphoneos/ && + mkdir Payload && + cp -r CobraAppTest.app Payload && + zip --symlinks -r CobraAppTest.ipa Payload && + rm -r Payload + + - name: Zipping Tests + run: cd ddp/Build/Products/Debug-iphoneos/ && + zip --symlinks -r CobraAppTestUITests.zip CobraAppTestUITests-Runner.app + + - name: Run tests on BrowserStack + run: python3 ../../../script/automation/browserstack.py + --type xcuitest + --username "${{secrets.BROWSERSTACK_USERNAME}}" + --access_key "${{secrets.BROWSERSTACK_ACCESS_KEY}}" + --project_name "Cobra-Migration-Test-iOS-Perf" + --app_path "ddp/Build/Products/Debug-iphoneos/CobraAppTest.ipa" + --test_path "ddp/Build/Products/Debug-iphoneos/CobraAppTestUITests.zip" \ No newline at end of file diff --git a/binding/android/CobraTestApp/cobra-test-app/build.gradle b/binding/android/CobraTestApp/cobra-test-app/build.gradle index 952da1c4..9c709e51 100644 --- a/binding/android/CobraTestApp/cobra-test-app/build.gradle +++ b/binding/android/CobraTestApp/cobra-test-app/build.gradle @@ -114,7 +114,7 @@ dependencies { androidTestImplementation('androidx.test.espresso:espresso-core:3.2.0', { exclude group: 'com.android.support', module: 'support-annotations' }) - androidTestImplementation('com.microsoft.appcenter:espresso-test-extension:1.4') + androidTestImplementation('com.browserstack.automate:espresso:1.0.0') androidTestImplementation('androidx.test.espresso:espresso-intents:3.5.1') } diff --git a/binding/android/CobraTestApp/cobra-test-app/src/androidTest/java/ai/picovoice/cobra/testapp/CobraTest.java b/binding/android/CobraTestApp/cobra-test-app/src/androidTest/java/ai/picovoice/cobra/testapp/CobraTest.java index 8d6a0d16..40658778 100644 --- a/binding/android/CobraTestApp/cobra-test-app/src/androidTest/java/ai/picovoice/cobra/testapp/CobraTest.java +++ b/binding/android/CobraTestApp/cobra-test-app/src/androidTest/java/ai/picovoice/cobra/testapp/CobraTest.java @@ -16,9 +16,6 @@ import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.platform.app.InstrumentationRegistry; -import com.microsoft.appcenter.espresso.Factory; -import com.microsoft.appcenter.espresso.ReportHelper; - import org.junit.After; import org.junit.Assume; import org.junit.Before; @@ -49,7 +46,6 @@ public class CobraTest { @Rule - public ReportHelper reportHelper = Factory.getReportHelper(); Context testContext; Context appContext; AssetManager assetManager; @@ -57,11 +53,6 @@ public class CobraTest { String accessKey = ""; - @After - public void TearDown() { - reportHelper.label("Stopping App"); - } - @Before public void Setup() throws IOException { testContext = InstrumentationRegistry.getInstrumentation().getContext(); diff --git a/binding/android/CobraTestApp/cobra-test-app/src/androidTest/java/ai/picovoice/cobra/testapp/IntegrationTest.java b/binding/android/CobraTestApp/cobra-test-app/src/androidTest/java/ai/picovoice/cobra/testapp/IntegrationTest.java index 79fff3b6..1eaa18f9 100644 --- a/binding/android/CobraTestApp/cobra-test-app/src/androidTest/java/ai/picovoice/cobra/testapp/IntegrationTest.java +++ b/binding/android/CobraTestApp/cobra-test-app/src/androidTest/java/ai/picovoice/cobra/testapp/IntegrationTest.java @@ -16,9 +16,6 @@ import androidx.test.ext.junit.rules.ActivityScenarioRule; import androidx.test.ext.junit.runners.AndroidJUnit4; -import com.microsoft.appcenter.espresso.Factory; -import com.microsoft.appcenter.espresso.ReportHelper; - import org.hamcrest.Matcher; import org.junit.After; import org.junit.Before; @@ -74,9 +71,6 @@ public void perform(UiController uiController, View view) { @RunWith(AndroidJUnit4.class) public class IntegrationTest { - @Rule - public ReportHelper reportHelper = Factory.getReportHelper(); - @Rule public ActivityScenarioRule activityScenarioRule = new ActivityScenarioRule<>(MainActivity.class); @@ -91,11 +85,6 @@ public void intentsTeardown() { Intents.release(); } - @After - public void TearDown() { - reportHelper.label("Stopping App"); - } - @Test public void testPorcupine() { onView(withId(R.id.testButton)).perform(click()); diff --git a/binding/android/CobraTestApp/cobra-test-app/src/androidTest/java/ai/picovoice/cobra/testapp/PerformanceTest.java b/binding/android/CobraTestApp/cobra-test-app/src/androidTest/java/ai/picovoice/cobra/testapp/PerformanceTest.java index 9f48c31e..c15fc532 100644 --- a/binding/android/CobraTestApp/cobra-test-app/src/androidTest/java/ai/picovoice/cobra/testapp/PerformanceTest.java +++ b/binding/android/CobraTestApp/cobra-test-app/src/androidTest/java/ai/picovoice/cobra/testapp/PerformanceTest.java @@ -8,9 +8,6 @@ import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.platform.app.InstrumentationRegistry; -import com.microsoft.appcenter.espresso.Factory; -import com.microsoft.appcenter.espresso.ReportHelper; - import org.junit.After; import org.junit.Assume; import org.junit.Before; @@ -34,18 +31,12 @@ @RunWith(AndroidJUnit4.class) public class PerformanceTest { @Rule - public ReportHelper reportHelper = Factory.getReportHelper(); Context testContext; Context appContext; AssetManager assetManager; String testResourcesPath; String accessKey; - @After - public void TearDown() { - reportHelper.label("Stopping App"); - } - @Before public void Setup() throws IOException { testContext = InstrumentationRegistry.getInstrumentation().getContext(); diff --git a/script/automation/browserstack.py b/script/automation/browserstack.py new file mode 100644 index 00000000..9c8d3488 --- /dev/null +++ b/script/automation/browserstack.py @@ -0,0 +1,114 @@ +import argparse +import requests +import time + +APP_URI = 'https://api-cloud.browserstack.com/app-automate/{}/v2/app' +TEST_URI = 'https://api-cloud.browserstack.com/app-automate/{}/v2/test-suite' +BUILD_URI = 'https://api-cloud.browserstack.com/app-automate/{}/v2/build' +STATUS_URI = 'https://api-cloud.browserstack.com/app-automate/{}/v2/builds/{}' + +ESPRESSO_DEVICES = [ + 'Google Pixel 6 Pro-12.0' +] + +XCUITEST_DEVICES = [ + 'iPhone 12 Pro-17' +] + +def get_devices(type: str): + if type == 'espresso': + return ESPRESSO_DEVICES + elif type == 'xcuitest': + return XCUITEST_DEVICES + else: + return [] + +def main(args: argparse.Namespace) -> None: + app_files = { + 'file': open(args.app_path, 'rb') + } + + app_response = requests.post( + APP_URI.format(args.type), + files=app_files, + auth=(args.username, args.access_key) + ) + app_response_json = app_response.json() + + if not app_response.ok: + print('App Upload Failed', app_response_json) + exit(1) + + test_files = { + 'file': open(args.test_path, 'rb') + } + test_response = requests.post( + TEST_URI.format(args.type), + files=test_files, + auth=(args.username, args.access_key) + ) + test_response_json = test_response.json() + + if not test_response.ok: + print('Test Upload Failed', test_response_json) + exit(1) + + build_headers = { + 'Content-Type': 'application/json' + } + build_data = { + 'app': app_response_json['app_url'], + 'testSuite': test_response_json['test_suite_url'], + 'project': args.project_name, + 'devices': get_devices(args.type) + } + build_response = requests.post( + BUILD_URI.format(args.type), + headers=build_headers, + json=build_data, + auth=(args.username, args.access_key) + ) + build_response_json = build_response.json() + + if not build_response.ok: + print('Build Failed', build_response_json) + exit(1) + + if build_response_json['message'] != 'Success': + print('Build Unsuccessful') + exit(1) + + print('View build results at https://app-automate.browserstack.com/dashboard/v2/builds/{}'.format(build_response_json['build_id'])) + + while True: + time.sleep(60) + status_response = requests.get( + STATUS_URI.format(args.type, build_response_json['build_id']), + auth=(args.username, args.access_key) + ) + status_response_json = status_response.json() + status = status_response_json['status'] + + if not status_response.ok: + print('Status Request Failed', status_response_json) + exit(1) + + if status != 'queued' and status != 'running': + break + + print('Status:', status) + if status != 'passed': + exit(1) + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--type', choices=['espresso', 'xcuitest'], required=True) + parser.add_argument('--username', required=True) + parser.add_argument('--access_key', required=True) + + parser.add_argument('--project_name', required=True) + parser.add_argument('--app_path', required=True) + parser.add_argument('--test_path', required=True) + args = parser.parse_args() + + main(args) \ No newline at end of file