Merge pull request #29 from mmcc007/#28_test_spec_resource

Added test_spec.yaml as a resource and allowed multiple tests to run on each device

README.md

@@ -27,6 +27,12 @@ project_name: flutter tests
 default_job_timeout: 5 # minutes
 
 tmp_dir: /tmp/sylph
+artifacts_dir: /tmp/sylph_artifacts
+
+# local timeout per device farm run
+sylph_timeout: 720 # seconds approx
+# run on ios and android pools concurrently (for faster results)
+concurrent_runs: true
 
 device_pools:
 
@@ -35,41 +41,99 @@ device_pools:
     devices:
       - name: Samsung Galaxy S9 (Unlocked)
         model: SM-G960U1
-        os: 8.0.0
+        os: '8.0.0'
 
   - pool_name: ios pool 1
     pool_type: ios
     devices:
       - name: Apple iPhone X
         model: A1865
-        os: 12.0
+        os: '12.0'
 
 test_suites:
 
   - test_suite: example tests 1
     main: test_driver/main1.dart
-    testspec: test_driver/test_spec.yaml
     tests:
       - test_driver/main1_test1.dart
       - test_driver/main1_test2.dart
     device_pools:
       - android pool 1
-#      - ios pool 1
-    job_timeout: 5 # minutes
+      - ios pool 1
+    job_timeout: 5 # minutes per each device run
 
   - test_suite: example tests 2
     main: test_driver/main2.dart
-    testspec: test_driver/test_spec.yaml
     tests:
       - test_driver/main2_test1.dart
       - test_driver/main2_test2.dart
-    device_pools:
+    pool_names:
       - android pool 1
-#      - ios pool 1
-    job_timeout: 5 # minutes
+      - ios pool 1
+    job_timeout: 5 # minutes per each device run
+```
+
+# Dependencies
+## AWS CLI
+Install AWS Command Line Interface (AWS CLI)
+```
+curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip"
+unzip awscli-bundle.zip
+sudo ./awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws
+``` 
+For alternative install options see:  
+https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html
+
+## AWS CLI Credentials
+Configure the AWS CLI credentials:
+```
+$ aws configure
+AWS Access Key ID [None]: AKIAIOSFODNN7EXAMPLE
+AWS Secret Access Key [None]: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+Default region name [None]: us-west-2
+Default output format [None]: json
 ```
+For alternative configuration options see:  
+https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html
+
+# Configuring a CI Environment for _Sylph_
+
+## iOS builds
+Special handling for building iOS apps is required for running tests on remote real devices. In particular, provisioning profiles and certificates must be installed on the build machine. To install the dependencies needed to complete the iOS build, Fastlane's match is used. _Sylph_ will detect it is running in a CI environment (using the CI environment variable), and will install fastlane files that in turn will install the dependencies needed to build the iOS app using Fastlane's match. The iOS build can then complete as normal.
+
+The following environment variable is required my Match:
+
+
+- PUBLISHING_MATCH_CERTIFICATE_REPO  
+This is the location of the private match repo. For example, https://matchusername:[email protected]/private_repos/match  
+where  
+    - matchusername  
+    is the username used when setting-up match
+    - matchpassword  
+    is the password used when setting-up match
+    - private.mycompany.com/private_repos/match  
+    is the uri path to the match repo (if using git)
+
+For details on how to configure Match see:  
+https://docs.fastlane.tools/actions/match/
+
+## AWS CLI Credentials for CI
+The following AWS CLI credentials are required:
+- AWS_ACCESS_KEY_ID
+- AWS_SECRET_ACCESS_KEY
+
+For details on other credentials see:  
+https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html
+
+## Example secrets for Travis-CI
+_Sylph_ runs on Travis-CI and expects the following environment variables:
+
+![secret variables](art/travis_env_vars.png)
+
+
+
 # Live demo
-To see _Sylph_ in action, a live demo of the [example](example) app is available.  
+To see _Sylph_ in action in a CI environment, a  demo of the [example](example) app is available.  
 
 The log of the live run is here:  
 https://travis-ci.com/mmcc007/sylph

bin/main.dart

@@ -49,7 +49,7 @@ main(List<String> arguments) async {
   print(
       'Sylph run completed in ${sylphRuntimeFormatted(timestamp, DateTime.now())}.');
   if (sylphRunSucceeded) {
-    print('Sylph run \'$sylphRunName\' suceeded.');
+    print('Sylph run \'$sylphRunName\' succeeded.');
     exit(0);
   } else {
     print('Sylph run \'$sylphRunName\' failed.');

example/sylph.yaml

@@ -15,7 +15,7 @@ concurrent_runs: true
 
 # device farm config
 project_name: test concurrent runs
-default_job_timeout: 7 # minutes, set at project creation
+default_job_timeout: 10 # minutes, set at project creation
 
 device_pools:
 
@@ -50,11 +50,11 @@ test_suites:
 
   - test_suite: example tests 1
     main: test_driver/main.dart
-    testspec: test_driver/test_spec.yaml
     tests:
       - test_driver/main_test.dart
+      - test_driver/main_test.dart
     pool_names:
       - android pool 1
       - ios pool 1
 #      - iPhone 5c
-    job_timeout: 7 # minutes, set per job
+    job_timeout: 8 # minutes, set per job

lib/resources/script/local_utils.sh

@@ -12,7 +12,7 @@ main(){
         ;;
     --ci)
         if [[ -z $2 ]]; then show_help; fi
-        config_ci "$2"
+        config_ci "$2" # dev only
         ;;
     *)
         show_help
@@ -30,7 +30,7 @@ where:
         package a debug app as a .ipa
         (app must include 'enableFlutterDriverExtension()')
     --ci <staging dir>
-        configure a CI build environment
+        configure a CI build environment // dev only
     --help
         print this message
 " "$(basename "$0")"
@@ -39,6 +39,7 @@ where:
 
 # install certificate and provisioning profile using match
 # assumes resources unbundled from sylph
+# dev only
 config_ci() {
   local app_dir=$1
 

lib/resources/script/test_android.sh

@@ -16,6 +16,10 @@ main() {
         if [[ -z $2 ]]; then show_help; fi
         custom_test_runner "$2"
         ;;
+    --run-tests)
+        if [[ -z $2 ]]; then show_help; fi
+        run_tests "$2"
+        ;;
     --run-driver)
         if [[ -z $2 ]]; then show_help; fi
         run_no_build "$2"
@@ -31,7 +35,7 @@ main() {
 }
 
 show_help() {
-    printf "\n\nusage: %s [--help] [--run-test <test path>] [--run-driver <test main path>
+    printf "\n\nusage: %s [--help] [--run-test <test path>] [--run-driver <test main path>] [--run-tests <comma-delimited list of test paths>]
 
 Utility for running integration tests for pre-installed flutter app on android device.
 (app must be built in debug mode with 'enableFlutterDriverExtension()')
@@ -41,6 +45,10 @@ where:
         run test from dart using a custom setup (similar to --no-build)
         <test path>
             path of test to run, eg, test_driver/main_test.dart
+    --run-tests <array of test paths>
+        run tests from dart using a custom setup (similar to --no-build)
+        <comma-delimited list of test paths>
+            list of test paths (eg, 'test_driver/main_test1.dart,test_driver/main_test2.dart')
     --run-driver
         run test using driver --no-build
         <test main path>
@@ -49,6 +57,24 @@ where:
     exit 1
 }
 
+#run_tests() {
+#  readarray -td, tests <<<"$1,"; unset 'tests[-1]'; declare -p tests;
+#  for test in "${tests[@]}"
+#  do
+##  echo "test=$test"
+#    custom_test_runner "$test"
+#  done
+#}
+
+run_tests() {
+  while IFS=',' read -ra tests; do
+    for test in "${tests[@]}"; do
+#      echo "test=$test"
+      custom_test_runner "$test"
+    done
+  done <<< "$1"
+}
+
 # note: assumes debug apk installed on device
 # note: by-passes flutter drives dependency on Android SDK which requires installing the SDK
 #       (see https://github.com/flutter/flutter/issues/34909)
@@ -104,18 +130,24 @@ getAppIdFromApk() {
   local apk_path="$1"
 
   # regular expression (required)
+  # shellcheck disable=SC2089
   local re="^\"L.*/MainActivity;"
   # sed substitute expression
+  # shellcheck disable=SC2089
   local se="s:^\"L\(.*\)/MainActivity;:\1:p"
   # tr expression
-  local te=' / .';
+  local te=" / .";
 
-  local app_id="$(unzip -p $apk_path classes.dex | strings | grep -Eo $re | sed -n -e $se | tr $te)"
+  local app_id
+  # shellcheck disable=SC2089
+  app_id="$(unzip -p "$apk_path" classes.dex | strings | grep -Eo "$re" | sed -n -e "$se" | tr $te)"
 
   echo "$app_id"
 }
 
 # note: requires android sdk be installed to get app identifier (eg, com.example.example)
+# not currently used
+#       (see https://github.com/flutter/flutter/issues/34909)
 run_no_build() {
   local test_main="$1"