ci: smoke test automation (#13162)

.github/actions/install-cli/action.yml

@@ -0,0 +1,26 @@
+name: Install Amplify CLI
+description: Installs Amplify CLI
+inputs:
+  cli-version:
+    description: a version of Amplify CLI
+    required: true
+runs:
+  using: 'composite'
+  steps:
+    - name: Install CLI
+      shell: bash
+      if: runner.os != 'Windows'
+      run: |
+        npm install -g @aws-amplify/cli@${{ inputs.cli-version }}
+        AMPLIFY_PATH=$(which amplify)
+        echo "AMPLIFY_PATH=$AMPLIFY_PATH"
+        echo "AMPLIFY_PATH=$AMPLIFY_PATH" >> $GITHUB_ENV
+
+    - name: Install CLI Windows
+      shell: powershell
+      if: runner.os == 'Windows'
+      run: |
+        npm install -g @aws-amplify/cli@${{ inputs.cli-version }}
+        $AMPLIFY_PATH=(Get-Command amplify).Path
+        echo "AMPLIFY_PATH=$AMPLIFY_PATH"
+        "AMPLIFY_PATH=$AMPLIFY_PATH" >> $env:GITHUB_ENV

.github/actions/run-smoke-tests-windows/README.md

@@ -0,0 +1,2 @@
+The Run Smoke Tests contains lightweight version of smoke tests for Windows.
+GitHub Action's Windows workers are small and using our E2E testing framework for smoke testing overwhelms them.

.github/actions/run-smoke-tests-windows/action.yml

@@ -0,0 +1,53 @@
+name: Run Smoke Tests
+description: Executes Smoke Tests
+inputs:
+  role-to-assume:
+    description: an IAM role to use in tests
+    required: true
+  region:
+    description: an AWS region to run in
+    required: false
+    default: us-west-2
+runs:
+  using: 'composite'
+  steps:
+    - name: Verify Amplify Path
+      shell: bash
+      run: |
+        if [[ -z "$AMPLIFY_PATH" ]]; then
+          echo "AMPLIFY_PATH must be defined"
+          exit 1
+        fi
+        echo "AMPLIFY_PATH=$AMPLIFY_PATH"
+        $AMPLIFY_PATH version
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@04b98b3f9e85f563fb061be8751a0352327246b0 # v3.0.1
+      with:
+        role-to-assume: ${{ inputs.role-to-assume }}
+        aws-region: ${{ inputs.region }}
+
+    - name: Install AWS CLI
+      shell: powershell
+      run: |
+        msiexec.exe /i https://awscli.amazonaws.com/AWSCLIV2.msi /qn
+        aws --version
+
+    - name: Configure Profile
+      shell: bash
+      run: |
+        aws configure set region ${{ inputs.region }} --profile default
+        aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
+        aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
+        aws configure set aws_session_token  $AWS_SESSION_TOKEN --profile default
+
+    - name: Run Smoke Tests
+      shell: bash
+      run: .github/actions/run-smoke-tests-windows/windows-smoke-tests.sh
+
+    - name: Cleanup
+      shell: bash
+      if: always()
+      run: |
+        cd $AMPLIFY_PROJ_DIR
+        $AMPLIFY_PATH delete --force

.github/actions/run-smoke-tests-windows/add_api_request.json

@@ -0,0 +1,9 @@
+{
+  "version": 1,
+  "serviceConfiguration": {
+    "serviceName": "AppSync",
+    "apiName": "myApiName",
+    "transformSchema": "type Todo @model { id: ID!  content: String}",
+    "defaultAuthType": { "mode": "API_KEY" }
+  }
+}

.github/actions/run-smoke-tests-windows/add_auth_request.json

@@ -0,0 +1,9 @@
+{
+  "version": 2,
+  "resourceName": "myAuthResource",
+  "serviceConfiguration": {
+    "serviceName": "Cognito",
+    "includeIdentityPool": false,
+    "userPoolConfiguration": { "requiredSignupAttributes": ["EMAIL", "PHONE_NUMBER"], "signinMethod": "USERNAME" }
+  }
+}

.github/actions/run-smoke-tests-windows/windows-smoke-tests.sh

@@ -0,0 +1,41 @@
+#!/bin/bash -e
+
+$AMPLIFY_PATH version
+
+random_guid=$(powershell -Command "[guid]::NewGuid().ToString()")
+random_guid=${random_guid//-/}
+random_guid=${random_guid:0:6}
+projDir="smoke-test-${random_guid}"
+echo "AMPLIFY_PROJ_DIR=$projDir" >> $GITHUB_ENV
+
+# Read headless requests content. This must be done before we change directory to $projDir.
+THIS_SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}")
+ADD_AUTH_REQUEST=$(cat $THIS_SCRIPT_PATH/add_auth_request.json)
+ADD_AUTH_REQUEST=${ADD_AUTH_REQUEST//[$'\r\n']}
+ADD_API_REQUEST=$(cat $THIS_SCRIPT_PATH/add_api_request.json)
+ADD_API_REQUEST=${ADD_API_REQUEST//[$'\r\n']}
+
+mkdir $projDir
+cd $projDir
+
+$AMPLIFY_PATH init --yes
+
+$AMPLIFY_PATH status
+
+echo $ADD_AUTH_REQUEST
+$AMPLIFY_PATH add auth --headless <<< $ADD_AUTH_REQUEST
+
+$AMPLIFY_PATH status
+
+$AMPLIFY_PATH push --yes
+
+$AMPLIFY_PATH status
+
+echo $ADD_API_REQUEST
+$AMPLIFY_PATH add api --headless <<< $ADD_API_REQUEST
+
+$AMPLIFY_PATH status
+
+$AMPLIFY_PATH push --yes
+
+$AMPLIFY_PATH status

.github/actions/run-smoke-tests/action.yml

@@ -0,0 +1,66 @@
+name: Run Smoke Tests
+description: Executes Smoke Tests
+inputs:
+  role-to-assume:
+    description: an IAM role to use in tests
+    required: true
+  region:
+    description: an AWS region to run in
+    required: false
+    default: us-west-2
+runs:
+  using: 'composite'
+  steps:
+    - name: Verify Amplify Path
+      shell: bash
+      run: |
+        if [[ -z "$AMPLIFY_PATH" ]]; then
+          echo "AMPLIFY_PATH must be defined"
+          exit 1
+        fi
+        echo "AMPLIFY_PATH=$AMPLIFY_PATH"
+        $AMPLIFY_PATH version
+
+    - name: Install Dependencies
+      shell: bash
+      run: yarn install --immutable
+
+    - name: Build
+      shell: bash
+      run: yarn build-tests
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@04b98b3f9e85f563fb061be8751a0352327246b0 # v3.0.1
+      with:
+        role-to-assume: ${{ inputs.role-to-assume }}
+        aws-region: ${{ inputs.region }}
+
+      # For iOS tests
+    - name: Setup Ruby
+      uses: ruby/setup-ruby@250fcd6a742febb1123a77a841497ccaa8b9e939 # v1.152.0
+      if: runner.os == 'macOS'
+      with:
+        ruby-version: '3.2.1'
+
+      # For iOS tests
+    - name: Set Default Xcode Version
+      if: runner.os == 'macOS'
+      shell: bash
+      run: |
+        sudo xcode-select -s "/Applications/Xcode_14.1.app"
+        xcodebuild -version
+
+    - name: Run Smoke Tests
+      shell: bash
+      run: yarn smoketest -- --forceExit --no-cache --maxWorkers=2
+      env:
+        CLI_REGION: ${{ inputs.region }}
+        CI: true
+        CIRCLECI: true
+
+    - name: Upload Report
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce #v3.1.2
+      if: always()
+      with:
+        name: test report
+        path: packages/amplify-e2e-tests/amplify-e2e-reports

.github/workflows/smoke-tests-canaries.yml

@@ -0,0 +1,17 @@
+name: Smoke Tests - Canaries
+
+# This is required by aws-actions/configure-aws-credentials
+permissions:
+  id-token: write
+  contents: read
+
+on:
+  schedule:
+    - cron: '0 16 * * *' # Everyday 16:00 UTC
+
+jobs:
+  call-smoke-tests:
+    uses: ./.github/workflows/smoke-tests.yml
+    secrets: inherit
+    with:
+      versions: '["rc", "latest"]'

.github/workflows/smoke-tests-manual.yml

@@ -0,0 +1,27 @@
+name: Smoke Tests - Manual Run
+
+# This is required by aws-actions/configure-aws-credentials
+permissions:
+  id-token: write
+  contents: read
+
+on:
+  workflow_dispatch:
+    inputs:
+      os:
+        required: false
+        type: string
+        default: '["macos-latest-xl", "ubuntu-latest", "windows-latest"]'
+      versions:
+        required: false
+        type: string
+        default: '["rc", "latest"]'
+
+jobs:
+  call-smoke-tests:
+    if: github.event_name == 'workflow_dispatch'
+    uses: ./.github/workflows/smoke-tests.yml
+    secrets: inherit
+    with:
+      os: ${{ github.event.inputs.os }}
+      versions: ${{ github.event.inputs.versions }}

.github/workflows/smoke-tests.yml

@@ -0,0 +1,53 @@
+name: Smoke Tests
+
+# This is required by aws-actions/configure-aws-credentials
+permissions:
+  id-token: write
+  contents: read
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        required: false
+        type: string
+        default: '["macos-latest-xl", "ubuntu-latest", "windows-latest"]'
+      versions:
+        required: false
+        type: string
+        default: '["rc", "latest"]'
+
+env:
+  NODE_OPTIONS: --max-old-space-size=8096
+
+jobs:
+  run-smoke-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ${{ fromJson(inputs.os) }}
+        cli-version: ${{ fromJson(inputs.versions) }}
+    name: Smoke Tests ${{ matrix.os }} ${{ matrix.cli-version }}
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        with:
+          persist-credentials: false
+
+      - name: Install Amplify CLI
+        uses: ./.github/actions/install-cli
+        with:
+          cli-version: ${{ matrix.cli-version }}
+
+      - name: Run Smoke Tests Unix
+        if: runner.os != 'Windows'
+        uses: ./.github/actions/run-smoke-tests
+        with:
+          role-to-assume: ${{ secrets.SMOKE_TESTS_ROLE_ARN }}
+
+      - name: Run Smoke Tests Windows
+        if: runner.os == 'Windows'
+        uses: ./.github/actions/run-smoke-tests-windows
+        with:
+          role-to-assume: ${{ secrets.SMOKE_TESTS_ROLE_ARN }}

.prettierignore

@@ -14,6 +14,7 @@ packages/amplify-cli-core/src/__tests__/testFiles/packageManager-npm/package-loc
 packages/amplify-cli-core/src/__tests__/testFiles/testProject-initialize-json-error/amplify/cli.json
 packages/amplify-e2e-core/dist/index.html
 packages/amplify-e2e-tests/amplify-e2e-reports
+packages/amplify-e2e-tests/resources/example-ios-app
 packages/amplify-environment-parameters/src/schemas/BackendParameters.schema.json
 packages/amplify-nodejs-function-runtime-provider/src/__tests__/utils/handlerWithSyntaxError.js
 packages/amplify-velocity-template/src/parse/velocity.yy