diff --git a/README.md b/README.md index 75898e3c..6f8f8215 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,12 @@ project_name: flutter tests default_job_timeout: 5 # minutes tmp_dir: /tmp/sylph +artifacts_dir: /tmp/sylph_artifacts + +# local timeout per device farm run +sylph_timeout: 720 # seconds approx +# run on ios and android pools concurrently (for faster results) +concurrent_runs: true device_pools: @@ -35,41 +41,99 @@ device_pools: devices: - name: Samsung Galaxy S9 (Unlocked) model: SM-G960U1 - os: 8.0.0 + os: '8.0.0' - pool_name: ios pool 1 pool_type: ios devices: - name: Apple iPhone X model: A1865 - os: 12.0 + os: '12.0' test_suites: - test_suite: example tests 1 main: test_driver/main1.dart - testspec: test_driver/test_spec.yaml tests: - test_driver/main1_test1.dart - test_driver/main1_test2.dart device_pools: - android pool 1 -# - ios pool 1 - job_timeout: 5 # minutes + - ios pool 1 + job_timeout: 5 # minutes per each device run - test_suite: example tests 2 main: test_driver/main2.dart - testspec: test_driver/test_spec.yaml tests: - test_driver/main2_test1.dart - test_driver/main2_test2.dart - device_pools: + pool_names: - android pool 1 -# - ios pool 1 - job_timeout: 5 # minutes + - ios pool 1 + job_timeout: 5 # minutes per each device run +``` + +# Dependencies +## AWS CLI +Install AWS Command Line Interface (AWS CLI) +``` +curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip" +unzip awscli-bundle.zip +sudo ./awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws +``` +For alternative install options see: +https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html + +## AWS CLI Credentials +Configure the AWS CLI credentials: +``` +$ aws configure +AWS Access Key ID [None]: AKIAIOSFODNN7EXAMPLE +AWS Secret Access Key [None]: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +Default region name [None]: us-west-2 +Default output format [None]: json ``` +For alternative configuration options see: +https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html + +# Configuring a CI Environment for _Sylph_ + +## iOS builds +Special handling for building iOS apps is required for running tests on remote real devices. In particular, provisioning profiles and certificates must be installed on the build machine. To install the dependencies needed to complete the iOS build, Fastlane's match is used. _Sylph_ will detect it is running in a CI environment (using the CI environment variable), and will install fastlane files that in turn will install the dependencies needed to build the iOS app using Fastlane's match. The iOS build can then complete as normal. + +The following environment variable is required my Match: + + +- PUBLISHING_MATCH_CERTIFICATE_REPO +This is the location of the private match repo. For example, https://matchusername:matchpassword@private.mycompany.com/private_repos/match +where + - matchusername + is the username used when setting-up match + - matchpassword + is the password used when setting-up match + - private.mycompany.com/private_repos/match + is the uri path to the match repo (if using git) + +For details on how to configure Match see: +https://docs.fastlane.tools/actions/match/ + +## AWS CLI Credentials for CI +The following AWS CLI credentials are required: +- AWS_ACCESS_KEY_ID +- AWS_SECRET_ACCESS_KEY + +For details on other credentials see: +https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html + +## Example secrets for Travis-CI +_Sylph_ runs on Travis-CI and expects the following environment variables: + +![secret variables](art/travis_env_vars.png) + + + # Live demo -To see _Sylph_ in action, a live demo of the [example](example) app is available. +To see _Sylph_ in action in a CI environment, a demo of the [example](example) app is available. The log of the live run is here: https://travis-ci.com/mmcc007/sylph diff --git a/art/travis_env_vars.png b/art/travis_env_vars.png new file mode 100644 index 00000000..38768d01 Binary files /dev/null and b/art/travis_env_vars.png differ diff --git a/bin/main.dart b/bin/main.dart index 3a4f3e8a..b67ede7e 100644 --- a/bin/main.dart +++ b/bin/main.dart @@ -49,7 +49,7 @@ main(List arguments) async { print( 'Sylph run completed in ${sylphRuntimeFormatted(timestamp, DateTime.now())}.'); if (sylphRunSucceeded) { - print('Sylph run \'$sylphRunName\' suceeded.'); + print('Sylph run \'$sylphRunName\' succeeded.'); exit(0); } else { print('Sylph run \'$sylphRunName\' failed.'); diff --git a/example/sylph.yaml b/example/sylph.yaml index fc3a5a55..4cccc8ed 100644 --- a/example/sylph.yaml +++ b/example/sylph.yaml @@ -15,7 +15,7 @@ concurrent_runs: true # device farm config project_name: test concurrent runs -default_job_timeout: 7 # minutes, set at project creation +default_job_timeout: 10 # minutes, set at project creation device_pools: @@ -50,11 +50,11 @@ test_suites: - test_suite: example tests 1 main: test_driver/main.dart - testspec: test_driver/test_spec.yaml tests: - test_driver/main_test.dart + - test_driver/main_test.dart pool_names: - android pool 1 - ios pool 1 # - iPhone 5c - job_timeout: 7 # minutes, set per job + job_timeout: 8 # minutes, set per job diff --git a/lib/resources/script/local_utils.sh b/lib/resources/script/local_utils.sh index 5d427bd8..7171a252 100755 --- a/lib/resources/script/local_utils.sh +++ b/lib/resources/script/local_utils.sh @@ -12,7 +12,7 @@ main(){ ;; --ci) if [[ -z $2 ]]; then show_help; fi - config_ci "$2" + config_ci "$2" # dev only ;; *) show_help @@ -30,7 +30,7 @@ where: package a debug app as a .ipa (app must include 'enableFlutterDriverExtension()') --ci - configure a CI build environment + configure a CI build environment // dev only --help print this message " "$(basename "$0")" @@ -39,6 +39,7 @@ where: # install certificate and provisioning profile using match # assumes resources unbundled from sylph +# dev only config_ci() { local app_dir=$1 diff --git a/lib/resources/script/test_android.sh b/lib/resources/script/test_android.sh index 2a036321..d3611073 100755 --- a/lib/resources/script/test_android.sh +++ b/lib/resources/script/test_android.sh @@ -16,6 +16,10 @@ main() { if [[ -z $2 ]]; then show_help; fi custom_test_runner "$2" ;; + --run-tests) + if [[ -z $2 ]]; then show_help; fi + run_tests "$2" + ;; --run-driver) if [[ -z $2 ]]; then show_help; fi run_no_build "$2" @@ -31,7 +35,7 @@ main() { } show_help() { - printf "\n\nusage: %s [--help] [--run-test ] [--run-driver + printf "\n\nusage: %s [--help] [--run-test ] [--run-driver ] [--run-tests ] Utility for running integration tests for pre-installed flutter app on android device. (app must be built in debug mode with 'enableFlutterDriverExtension()') @@ -41,6 +45,10 @@ where: run test from dart using a custom setup (similar to --no-build) path of test to run, eg, test_driver/main_test.dart + --run-tests + run tests from dart using a custom setup (similar to --no-build) + + list of test paths (eg, 'test_driver/main_test1.dart,test_driver/main_test2.dart') --run-driver run test using driver --no-build @@ -49,6 +57,24 @@ where: exit 1 } +#run_tests() { +# readarray -td, tests <<<"$1,"; unset 'tests[-1]'; declare -p tests; +# for test in "${tests[@]}" +# do +## echo "test=$test" +# custom_test_runner "$test" +# done +#} + +run_tests() { + while IFS=',' read -ra tests; do + for test in "${tests[@]}"; do +# echo "test=$test" + custom_test_runner "$test" + done + done <<< "$1" +} + # note: assumes debug apk installed on device # note: by-passes flutter drives dependency on Android SDK which requires installing the SDK # (see https://github.com/flutter/flutter/issues/34909) @@ -104,18 +130,24 @@ getAppIdFromApk() { local apk_path="$1" # regular expression (required) + # shellcheck disable=SC2089 local re="^\"L.*/MainActivity;" # sed substitute expression + # shellcheck disable=SC2089 local se="s:^\"L\(.*\)/MainActivity;:\1:p" # tr expression - local te=' / .'; + local te=" / ."; - local app_id="$(unzip -p $apk_path classes.dex | strings | grep -Eo $re | sed -n -e $se | tr $te)" + local app_id + # shellcheck disable=SC2089 + app_id="$(unzip -p "$apk_path" classes.dex | strings | grep -Eo "$re" | sed -n -e "$se" | tr $te)" echo "$app_id" } # note: requires android sdk be installed to get app identifier (eg, com.example.example) +# not currently used +# (see https://github.com/flutter/flutter/issues/34909) run_no_build() { local test_main="$1" diff --git a/lib/resources/script/test_ios.sh b/lib/resources/script/test_ios.sh index 15570b18..a12f86a1 100755 --- a/lib/resources/script/test_ios.sh +++ b/lib/resources/script/test_ios.sh @@ -6,11 +6,6 @@ set -e # run integration test on ios # used on device clouds -# constants -default_debug_ipa_name='Debug_Runner.ipa' -default_debug_ipa_dir="." -debug_app_dir='build/ios/iphoneos' - main() { case $1 in --help) @@ -18,35 +13,41 @@ main() { ;; --unpack) if [[ -z $2 ]]; then show_help; fi - unpack_debug_ipa $2 + unpack_debug_ipa "$2" ;; --dummy-symbols) if [[ -z $2 ]]; then show_help; fi - dummy_symbols $2 + dummy_symbols "$2" ;; - --test) + --run-driver) if [[ -z $2 ]]; then show_help; fi - run_test $2 + run_driver "$2" "$3" + ;; + --run-tests) + if [[ -z $2 || -z $3 ]]; then show_help; fi + run_tests "$2" "$3" ;; - *) + *) show_help ;; esac } show_help() { - printf "\n\nusage: %s [--unpack ] [--dummy-symbols ] [--test ] + printf "\n\nusage: %s [--unpack ] [--dummy-symbols ] [--run-driver []] [--run-tests ] -Utility for building a debug app as a .ipa, unpacking, and running integration test on an iOS device. +Utility for building a debug app as a .ipa, unpacking, and running integration tests on an iOS device. (app must include 'enableFlutterDriverExtension()') where: --unpack unpack debug .ipa to build directory for testing - --dummy-symbols <> + --dummy-symbols generate dummy symbol directories for ios-deploy - --test - run integration test on debug app + --run-driver [] + run integration test on debug app with default or specified test + --run-tests + run integration tests on debug app with list of test paths (eg, 'test_driver/main_test1.dart,test_driver/main_test2.dart') --help print this message " "$(basename "$0")" @@ -57,6 +58,8 @@ where: unpack_debug_ipa(){ local ipa_path=$1 local unpack_dir='Payload' + local debug_app_dir='build/ios/iphoneos' + echo "Unpacking $ipa_path to $debug_app_dir..." @@ -74,14 +77,47 @@ unpack_debug_ipa(){ dummy_symbols() { local dummy_symbols_path=$1 + # shellcheck disable=SC2162 while IFS=$'=' read build os; do echo "creating $HOME/Library/Developer/Xcode/iOS DeviceSupport/$os ($build)/Symbols" mkdir -p "$HOME/Library/Developer/Xcode/iOS DeviceSupport/$os ($build)/Symbols" - done < $dummy_symbols_path + done < "$dummy_symbols_path" +} + +#run_tests() { +# local debug_app_path=$1 +# shift +# local tests=($@) +# for test in "${tests[@]}" +# do +# run_driver "$debug_app_path" "$test" +# done +#} + +#run_tests() { +# local debug_app_path=$1 +# readarray -t -d, tests <<<"$2,"; unset 'tests[-1]'; declare -p tests; +# for test in "${tests[@]}" +# do +## echo "test=$test" +# run_driver "$debug_app_path" "$test" +# done +#} + + +run_tests() { + local debug_app_path=$1 + while IFS=',' read -ra tests; do + for test in "${tests[@]}"; do +# echo "test=$test" + run_driver "$debug_app_path" "$test" + done + done <<< "$2" } -run_test() { +run_driver() { local debug_app_path=$1 + local test_path=$2 # disable reporting analytics flutter config --no-analytics @@ -89,8 +125,13 @@ run_test() { # update .packages in case last build was on a different flutter repo flutter packages get - echo "Running flutter drive --no-build $debug_app_path" - flutter drive --no-build $debug_app_path + if [[ -z "$test_path" ]]; then + echo "Running flutter drive --no-build $debug_app_path" + flutter drive --no-build "$debug_app_path" + else + echo "Running flutter drive --no-build -t $debug_app_path --driver $test_path" + flutter drive --no-build "$debug_app_path" -t "$debug_app_path" --driver "$test_path" + fi } main "$@" \ No newline at end of file diff --git a/example/test_driver/test_spec.yaml b/lib/resources/test_spec.yaml similarity index 84% rename from example/test_driver/test_spec.yaml rename to lib/resources/test_spec.yaml index 3bdb3bd8..de023c8a 100644 --- a/example/test_driver/test_spec.yaml +++ b/lib/resources/test_spec.yaml @@ -18,33 +18,14 @@ phases: - cd flutter_app - ls -la - # check for node - - node --version - # check for bash - bash --version # env - env - - echo $DEVICEFARM_APP_PATH - - # upgrade ios-deploy - #- ios-deploy -V - #- xcodebuild -version - #- xcode-select --print-path - #- gcc --version - #- lldb --version - #- whoami - #- groups - - # install local brew because cannot upgrade ios-deploy - # try skipping upgrade (takes a long time) - #- git clone --depth=1 https://github.com/Homebrew/brew ~/.brew - #- export PATH="$HOME/.brew/bin:$HOME/.brew/sbin:$PATH" - #- brew install ios-deploy - #- which ios-deploy - #- ios-deploy -V + - whoami + - groups # install flutter - echo "Install flutter" @@ -52,10 +33,8 @@ phases: - >- if [ $DEVICEFARM_DEVICE_PLATFORM_NAME = "Android" ]; then - # Run EC2 setup code here - #java -version - ## Install android SDK + #java -version #ANDROID_SDK_TOOLS=4333796 # android-28 #ANDROID_PLATFORM_SDK=28 # required by flutter #ANDROID_BUILD_TOOLS=28.0.3 # required by flutter @@ -77,7 +56,7 @@ phases: #sdkmanager "tools" "platform-tools" "platforms;android-${ANDROID_PLATFORM_SDK}" "build-tools;${ANDROID_BUILD_TOOLS}" > /dev/null #sdkmanager --list | head -15 - + # Install Flutter curl https://storage.googleapis.com/flutter_infra/releases/stable/linux/flutter_linux_$FLUTTER_VERSION.tar.xz -o flutter_linux.tar.xz tar xf flutter_linux.tar.xz fi @@ -87,7 +66,21 @@ phases: # Run Mac setup code here system_profiler SPSoftwareDataType sw_vers + which ios-deploy ios-deploy -V + xcodebuild -version + xcode-select --print-path + gcc --version + lldb --version + + # install local brew because cannot upgrade ios-deploy + # try skipping upgrade (takes a long time) + #- git clone --depth=1 https://github.com/Homebrew/brew ~/.brew + #- export PATH="$HOME/.brew/bin:$HOME/.brew/sbin:$PATH" + #- brew install ios-deploy + #- which ios-deploy + #- ios-deploy -V + # Install Flutter dependencies # brew upgrade ios-deploy # brew install cocoapods @@ -95,16 +88,13 @@ phases: # Install Flutter curl https://storage.googleapis.com/flutter_infra/releases/stable/macos/flutter_macos_$FLUTTER_VERSION.zip -o flutter_macos.zip unzip -qq flutter_macos.zip - # build experimental flutter from fork - #git clone https://github.com/mmcc007/flutter.git -b master fi - mv flutter $HOME/flutter #- echo "export PATH=$PATH:$HOME/flutter/bin:$HOME/flutter/bin/cache/dart-sdk/bin">> ~/.bash_profile - export PATH=$PATH:$HOME/flutter/bin:$HOME/flutter/bin/cache/dart-sdk/bin - export PATH="$PATH":"$HOME/.pub-cache/bin" - #- export PATH=$PWD/flutter/bin:$PWD/flutter/bin/cache/dart-sdk/bin:$PATH - - which flutter + - flutter doctor -v # The pre-test phase includes commands that setup your test environment. pre_test: @@ -129,23 +119,20 @@ phases: # Note: For most use cases, the default command works fine. # Please refer "https://docs.pytest.org/en/latest/usage.html" for more options on running pytests from command line. # - bin/py.test tests/ --junit-xml $DEVICEFARM_LOG_DIR/junitreport.xml - #- APP_ID=com.orbsoft.counter - - MAIN=test_driver/main.dart - - TEST=test_driver/main_test.dart - cd flutter_app + - MAIN=test_driver/main.dart + - TESTS='test_driver/main_test.dart' - >- if [ $DEVICEFARM_DEVICE_PLATFORM_NAME = "Android" ]; then - flutter doctor -v - #./script/test_android.sh --run-driver "$MAIN" - ./script/test_android.sh --run-test "$TEST" + ./script/test_android.sh --run-tests "$TESTS" fi if [ $DEVICEFARM_DEVICE_PLATFORM_NAME = "iOS" ]; then ./script/test_ios.sh --unpack $DEVICEFARM_APP_PATH ./script/test_ios.sh --dummy-symbols build_to_os.txt - ./script/test_ios.sh --test test_driver/main.dart + ./script/test_ios.sh --run-tests "$MAIN" "$TESTS" fi - echo done. diff --git a/lib/src/bundle.dart b/lib/src/bundle.dart index 20c0e99c..55487fdc 100644 --- a/lib/src/bundle.dart +++ b/lib/src/bundle.dart @@ -7,6 +7,7 @@ import 'utils.dart'; const kResourcesUri = 'package:sylph/resources'; const kAppiumTemplateName = 'appium_bundle.zip'; +const kAppiumTestSpecName = 'test_spec.yaml'; const kTestBundleDir = 'test_bundle'; const kTestBundleName = '$kTestBundleDir.zip'; const kDefaultFlutterAppName = 'flutter_app'; @@ -30,14 +31,14 @@ Future bundleFlutterTests(Map config) async { // create default app dir in test bundle cmd('mkdir', [defaultAppDir], '.', false); + // Copy app dir to test bundle + cmd('cp', ['-r', '.', defaultAppDir], '.', false); + // update .packages in case last build was on a different flutter repo - cmd('flutter', ['packages', 'get'], '.', true); + cmd('flutter', ['packages', 'get'], defaultAppDir, true); // clean build dir in case a build is present - cmd('flutter', ['clean'], '.', true); - - // Copy app dir to test bundle - cmd('cp', ['-r', '.', defaultAppDir], '.', false); + cmd('flutter', ['clean'], defaultAppDir, true); // Copy scripts to test bundle cmd('cp', ['-r', 'script', defaultAppDir], stagingDir, false); @@ -74,12 +75,19 @@ Future unpackResources(String tmpDir) async { await writeFileImage(await readResourceImage(kAppiumTemplateName), '$tmpDir/$kAppiumTemplateName'); + // unpack Appium test spec + await unpackFile(kAppiumTestSpecName, tmpDir); + // unpack scripts await unpackScripts(tmpDir); // unpack build to os map file await unpackFile(kBuildToOsMapFileName, tmpDir); + // unpack export options + // todo: configure exportOptions.plist for provisioning profile, team, etc... + await unpackFile('exportOptions.plist', 'ios'); + // unpack components used in a CI environment final envVars = Platform.environment; if (envVars['CI'] == 'true') { @@ -93,9 +101,6 @@ Future unpackResources(String tmpDir) async { // unpack dummy keys await unpackFile('dummy-ssh-keys/key', '.'); await unpackFile('dummy-ssh-keys/key.pub', '.'); - - // unpack export options - await unpackFile('exportOptions.plist', 'ios'); } } diff --git a/lib/src/sylph_run.dart b/lib/src/sylph_run.dart index b7737ca4..62ada3ec 100644 --- a/lib/src/sylph_run.dart +++ b/lib/src/sylph_run.dart @@ -51,15 +51,6 @@ Future sylphRun(String configFilePath, String sylphRunName, for (var testSuite in config['test_suites']) { print('\nRunning \'${testSuite['test_suite']}\' test suite...\n'); - // todo: update test spec with tests in test suite - // (currently only allows one test) -// final List tests = testSuite['tests']; -// for (var test in tests) { -// final poolType = devicePoolInfo['pool_type']; -// print( -// 'bundling test: $test on $poolType devices in device pool $poolName'); -// } - // Initialize device pools and run tests in each pool for (final poolName in testSuite['pool_names']) { bool runTestsSucceeded = false; @@ -105,24 +96,27 @@ Future runSylphJob(Map testSuite, Map config, poolName, String projectArn, // Setup device pool String devicePoolArn = setupDevicePool(devicePoolInfo, projectArn); - // Build debug app for pool type and upload - final appArn = await _buildUploadApp(projectArn, devicePoolInfo['pool_type'], - testSuite['main'], config['tmp_dir']); - + final tmpDir = config['tmp_dir']; // Upload test suite (in 2 parts) // 1. Upload test package - final testBundlePath = '${config['tmp_dir']}/${kTestBundleName}'; + final testBundlePath = '$tmpDir/$kTestBundleName'; print('Uploading tests: $testBundlePath ...'); String testPackageArn = uploadFile(projectArn, testBundlePath, 'APPIUM_PYTHON_TEST_PACKAGE'); // 2. Upload custom test spec yaml - final testSpecPath = testSuite['testspec']; + final testSpecPath = '$tmpDir/$kAppiumTestSpecName'; + // Substitute MAIN and TESTS for actual debug main and tests from test suite. + setTestSpecEnv(testSuite, testSpecPath); print('Uploading test specification: $testSpecPath ...'); String testSpecArn = uploadFile(projectArn, testSpecPath, 'APPIUM_PYTHON_TEST_SPEC'); + // Build debug app for pool type and upload + final appArn = await _buildUploadApp( + projectArn, devicePoolInfo['pool_type'], testSuite['main'], tmpDir); + // run tests and report return _runTests( sylphRunName, @@ -212,3 +206,33 @@ DateTime sylphTimestamp() { DateTime.now().millisecondsSinceEpoch); return timestamp; } + +/// Set MAIN and TESTS env vars in test spec. +void setTestSpecEnv(Map test_suite, String testSpecPath) { + const kMainEnvName = 'MAIN='; + const kTestsEnvName = 'TESTS='; + final mainEnvVal = test_suite['main']; + final testsEnvVal = test_suite['tests'].join(","); + final mainRegExp = RegExp('$kMainEnvName.*'); +// final testsRegExp = RegExp(r'(.*TESTS\+=).*'); + final testsRegExp = RegExp('$kTestsEnvName.*'); + String testSpecStr = File(testSpecPath).readAsStringSync(); + testSpecStr = + testSpecStr.replaceFirst(mainRegExp, '$kMainEnvName$mainEnvVal'); + +// // Device Farm does not accept a literal array declaration so must declare +// // each element of array. +// +// // build replacement string +// String testsStr = ''; +// final testsEnvName = testsRegExp.firstMatch(testSpecStr).group(1); +// for (final testPath in test_suite['tests']) { +// testsStr += '$testsEnvName$testPath\n'; +// } +// testsStr = testsStr.substring(0, testsStr.length - 1); // remove last \n +// +// testSpecStr = testSpecStr.replaceAll(testsRegExp, testsStr); + testSpecStr = + testSpecStr.replaceAll(testsRegExp, '$kTestsEnvName\'$testsEnvVal\''); + File(testSpecPath).writeAsStringSync(testSpecStr); +} diff --git a/test/sylph_test.dart b/test/sylph_test.dart index 030a298f..19416747 100644 --- a/test/sylph_test.dart +++ b/test/sylph_test.dart @@ -389,6 +389,25 @@ void main() { // rounds to milliseconds expect(durationFormatted.contains(RegExp(r'15m:34s:12.ms')), true); }); + + test('substitute MAIN and TESTS for actual debug main and tests', () async { + final filePath = 'test/sylph_test.yaml'; + final config = await parseYaml(filePath); + final test_suite = config['test_suites'][0]; + final expectedMainEnvVal = test_suite['main']; + final expectedTestsEnvVal = test_suite['tests'].join(","); + final testSpecPath = 'test/test_spec_test.yaml'; + final expected = ''' + # - bin/py.test tests/ --junit-xml \$DEVICEFARM_LOG_DIR/junitreport.xml + - MAIN=$expectedMainEnvVal + - TESTS='$expectedTestsEnvVal' + - cd flutter_app +'''; + setTestSpecEnv(test_suite, testSpecPath); + expect(File(testSpecPath).readAsStringSync(), expected); + // restore modified test spec test + cmd('git', ['checkout', testSpecPath]); + }); } int sortSylphDevices(d1, d2) { diff --git a/test/sylph_test.yaml b/test/sylph_test.yaml index 9f1d1cec..c53429a7 100644 --- a/test/sylph_test.yaml +++ b/test/sylph_test.yaml @@ -45,9 +45,9 @@ device_pools: test_suites: - test_suite: my tests 1 main: test_driver/main1.dart - testspec: test_spec.yaml tests: - - test_driver/main_test.dart + - test_driver/main1_test1.dart + - test_driver/main1_test2.dart pool_names: - android pool 1 - ios pool 1 @@ -55,9 +55,9 @@ test_suites: - test_suite: my tests 2 main: test_driver/main2.dart - testspec: test_spec2.yaml tests: - test_driver/main2_test1.dart + - test_driver/main2_test2.dart pool_names: - android pool 1 - ios pool 1 diff --git a/test/test_spec_test.yaml b/test/test_spec_test.yaml new file mode 100644 index 00000000..85dc4e57 --- /dev/null +++ b/test/test_spec_test.yaml @@ -0,0 +1,4 @@ + # - bin/py.test tests/ --junit-xml $DEVICEFARM_LOG_DIR/junitreport.xml + - MAIN=test_driver/main1.dart + - TESTS='test_driver/main_test.dart' + - cd flutter_app