fix: Restore iOS tests, move to dotnet test (#12085)

unoplatform · May 7, 2023 · b51b6bd · b51b6bd
2 parents ebae5c4 + 338115d
commit b51b6bd
Show file tree

Hide file tree

Showing 72 changed files with 631 additions and 490 deletions.
diff --git a/.vsts-ci.yml b/.vsts-ci.yml
@@ -51,9 +51,9 @@ variables:
   linuxVMImage: 'ubuntu-latest'
   macOSVMImage: 'macOS-12'
   macOSVMImage_UITests: 'macOS-12'
-  xCodeRoot: '/Applications/Xcode_14.2.app'
+  xCodeRoot: '/Applications/Xcode_14.1.app'
   XamarinSDKVersion: 6_12_24
-  xCodeRoot_iOS_UITests: '/Applications/Xcode_14.2.app'
+  xCodeRoot_iOS_UITests: '/Applications/Xcode_14.1.app'
   XamarinSDKVersion_iOS_UITests: 6_12_24
 
   # https://github.com/microsoft/azure-pipelines-tasks/issues/11864
@@ -116,10 +116,6 @@ jobs:
     vmMacImage: '$(macOSVMImage)'
     poolName: '$(windows2022HostedVMImage)'
 
-- template: build/ci/.azure-devops-samplesapp-uitests-build.yml
-  parameters:
-    vmImage: '$(windows2022HostedVMImage)'
-
 - template: build/ci/.azure-devops-wasm-uitests.yml
   parameters:
     vmImage: '$(linuxVMImage)'

diff --git a/build/ci/.azure-devops-android-tests.yml b/build/ci/.azure-devops-android-tests.yml
@@ -71,7 +71,6 @@ jobs:
   displayName: 'Run Android UI Tests'
   dependsOn:
   - Android_Build_NetCoreMobile_For_Tests
-  - SamplesApp_UITests_Build
 
   timeoutInMinutes: 90
   variables:
@@ -98,7 +97,7 @@ jobs:
           TARGETPLATFORM_NAME: net7
           FAILBUILD_ON_FAILURE: true
           ALLOW_RERUN: true
-          UITEST_TEST_TIMEOUT: '270000'
+          UITEST_TEST_TIMEOUT: '270s'
 
       ${{ each testGroup in parameters.runtimeTestsGroups }}:
         Android_9.0_RuntimeTests_${{ testGroup }}_net7:
@@ -111,7 +110,7 @@ jobs:
           TARGETPLATFORM_NAME: net7
           FAILBUILD_ON_FAILURE: true
           ALLOW_RERUN: false
-          UITEST_TEST_TIMEOUT: '2600000'
+          UITEST_TEST_TIMEOUT: '2600s'
 
       Android_9.0_Snapshots_net7:
         ANDROID_SIMULATOR_APILEVEL: 28
@@ -121,7 +120,7 @@ jobs:
         TARGETPLATFORM_NAME: net7
         FAILBUILD_ON_FAILURE: true
         ALLOW_RERUN: true
-        UITEST_TEST_TIMEOUT: '270000'
+        UITEST_TEST_TIMEOUT: '270s'
 
       # Android 10 testing is disabled because of https://github.com/microsoft/appcenter/issues/1451
       # Android_10.0:
@@ -138,12 +137,6 @@ jobs:
         artifactName: $(SAMPLEAPP_ARTIFACT_NAME)
         downloadPath: '$(build.sourcesdirectory)/build'
 
-  - task: DownloadBuildArtifacts@0
-    displayName: 'Download SamplesApp UnitTests'
-    inputs:
-        artifactName: samplesapp-uitest-binaries
-        downloadPath: '$(build.sourcesdirectory)/build'
-
   - task: DownloadBuildArtifacts@0
     condition: gt(variables['System.JobAttempt'], 1)
     continueOnError: true

diff --git a/build/ci/.azure-devops-ios-tests-run.yml b/build/ci/.azure-devops-ios-tests-run.yml
@@ -15,7 +15,6 @@ jobs:
   timeoutInMinutes: ${{ parameters.JobTimeoutInMinutes }}
   dependsOn:
   - iOS_Build
-  - SamplesApp_UITests_Build
   - Commitsar
   displayName: ${{ parameters.JobDisplayName }}
 
@@ -38,12 +37,6 @@ jobs:
         artifactName: ios-uitest-build
         downloadPath: '$(build.sourcesdirectory)/build'
 
-  - task: DownloadBuildArtifacts@0
-    displayName: 'Download iOS SamplesApp UnitTests'
-    inputs:
-        artifactName: samplesapp-uitest-binaries
-        downloadPath: '$(build.sourcesdirectory)/build'
-
   - task: DownloadBuildArtifacts@0
     condition: gt(variables['System.JobAttempt'], 1)
     continueOnError: true

diff --git a/build/ci/.azure-devops-ios-tests.yml b/build/ci/.azure-devops-ios-tests.yml
@@ -18,6 +18,29 @@ parameters:
   - key: '04'
     value: '05'
 
+  # Runtime Test Groups
+  UITEST_RUNTIME_TEST_GROUP_COUNT: 5
+  RUNTIME_TESTS_GROUPS:
+  - key: '0'
+    value: '01'
+  - key: '1'
+    value: '02'
+  - key: '2'
+    value: '03'
+  - key: '3'
+    value: '04'
+  - key: '4'
+    value: '05'
+
+  # Autmated Test Groups
+  AUTOMATED_TESTS_GROUPS:
+  - key: '1'
+    value: '01'
+  - key: '2'
+    value: '02'
+  - key: '3'
+    value: '03'
+
 jobs:
 - job: iOS_Build
   displayName: 'Build iOS Samples App UI Tests'
@@ -78,123 +101,68 @@ jobs:
       ArtifactName: ios-uitest-build
       ArtifactType: Container
 
+##
+## Automated tests
+##
+- ${{ each AutomatedTestGroup in parameters.AUTOMATED_TESTS_GROUPS }}:
+  - template: .azure-devops-ios-tests-run.yml
+    parameters:
+      nugetPackages: $(NUGET_PACKAGES)
+      JobName: 'iOS_Automated_Tests_Group_${{ AutomatedTestGroup.value }}'
+      JobDisplayName: 'iOS Automated Tests ${{ AutomatedTestGroup.value }}'
+      JobTimeoutInMinutes: 120
+      vmImage: ${{ parameters.vmImageTest }}
+      UITEST_SNAPSHOTS_ONLY: false
+      UITEST_AUTOMATED_GROUP: ${{ AutomatedTestGroup.key }}
+      UITEST_TEST_TIMEOUT: '20m'
+      UITEST_ALLOW_RERUN: 'true'
+      xCodeRoot: ${{ parameters.xCodeRootTest }}
+      XamarinSDKVersion: ${{ parameters.XamarinSDKVersionTest }}
+
+##
+## Runtime tests
+##
+- ${{ each RuntimeTestGroup in parameters.RUNTIME_TESTS_GROUPS }}:
+  - template: .azure-devops-ios-tests-run.yml
+    parameters:
+      nugetPackages: $(NUGET_PACKAGES)
+      JobName: 'iOS_Automated_Tests_Runtime_Tests_group_${{ RuntimeTestGroup.value }}'
+      JobDisplayName: 'iOS Runtime Tests ${{ RuntimeTestGroup.value }}'
+      JobTimeoutInMinutes: 120
+      vmImage: ${{ parameters.vmImageTest }}
+      UITEST_SNAPSHOTS_ONLY: false
+      UITEST_TEST_TIMEOUT: '70m'
+      UITEST_AUTOMATED_GROUP: 4
+      UITEST_RUNTIME_TEST_GROUP: ${{ RuntimeTestGroup.key }}
+      UITEST_RUNTIME_TEST_GROUP_COUNT: ${{ parameters.UITEST_RUNTIME_TEST_GROUP_COUNT }}
+      UITEST_ALLOW_RERUN: 'false'
+      xCodeRoot: ${{ parameters.xCodeRootTest }}
+      XamarinSDKVersion: ${{ parameters.XamarinSDKVersionTest }}
+
+# Disabled for benchmkarks timeout
 # - template: .azure-devops-ios-tests-run.yml
 #   parameters:
 #     nugetPackages: $(NUGET_PACKAGES)
-#     JobName: 'iOS_Automated_Tests_Group_01'
-#     JobDisplayName: 'Run iOS Automated Tests Group 01'
-#     JobTimeoutInMinutes: 90
-#     vmImage: ${{ parameters.vmImageTest }}
-#     UITEST_SNAPSHOTS_ONLY: false
-#     UITEST_AUTOMATED_GROUP: 1
-#     UITEST_TEST_TIMEOUT: '270000'
-#     UITEST_ALLOW_RERUN: 'true'
-#     xCodeRoot: ${{ parameters.xCodeRootTest }}
-#     XamarinSDKVersion: ${{ parameters.XamarinSDKVersionTest }}
-# 
-# - template: .azure-devops-ios-tests-run.yml
-#   parameters:
-#     nugetPackages: $(NUGET_PACKAGES)
-#     JobName: 'iOS_Automated_Tests_Group_02'
-#     JobDisplayName: 'Run iOS Automated Tests Group 02'
-#     JobTimeoutInMinutes: 90
-#     vmImage: ${{ parameters.vmImageTest }}
-#     UITEST_SNAPSHOTS_ONLY: false
-#     UITEST_AUTOMATED_GROUP: 2
-#     UITEST_TEST_TIMEOUT: '270000'
-#     UITEST_ALLOW_RERUN: 'true'
-#     xCodeRoot: ${{ parameters.xCodeRootTest }}
-#     XamarinSDKVersion: ${{ parameters.XamarinSDKVersionTest }}
-# 
-# - template: .azure-devops-ios-tests-run.yml
-#   parameters:
-#     nugetPackages: $(NUGET_PACKAGES)
-#     JobName: 'iOS_Automated_Tests_Group_03'
-#     JobDisplayName: 'Run iOS Automated Tests Group 03'
-#     JobTimeoutInMinutes: 90
-#     vmImage: ${{ parameters.vmImageTest }}
-#     UITEST_SNAPSHOTS_ONLY: false
-#     UITEST_AUTOMATED_GROUP: 3
-#     UITEST_TEST_TIMEOUT: '270000'
-#     UITEST_ALLOW_RERUN: 'true'
-#     xCodeRoot: ${{ parameters.xCodeRootTest }}
-#     XamarinSDKVersion: ${{ parameters.XamarinSDKVersionTest }}
-# 
-# ##
-# ## Runtime tests
-# ##
-# - template: .azure-devops-ios-tests-run.yml
-#   parameters:
-#     nugetPackages: $(NUGET_PACKAGES)
-#     JobName: 'iOS_Automated_Tests_Runtime_Tests_group_0'
-#     JobDisplayName: 'Run iOS Automated Runtime Tests Group 0'
-#     JobTimeoutInMinutes: 60
-#     vmImage: ${{ parameters.vmImageTest }}
-#     UITEST_SNAPSHOTS_ONLY: false
-#     UITEST_TEST_TIMEOUT: '3600000'
-#     UITEST_AUTOMATED_GROUP: 4
-#     UITEST_RUNTIME_TEST_GROUP: 0
-#     UITEST_RUNTIME_TEST_GROUP_COUNT: 3
-#     UITEST_ALLOW_RERUN: 'false'
-#     xCodeRoot: ${{ parameters.xCodeRootTest }}
-#     XamarinSDKVersion: ${{ parameters.XamarinSDKVersionTest }}
-# 
-# - template: .azure-devops-ios-tests-run.yml
-#   parameters:
-#     nugetPackages: $(NUGET_PACKAGES)
-#     JobName: 'iOS_Automated_Tests_Runtime_Tests_group_1'
-#     JobDisplayName: 'Run iOS Automated Runtime Tests Group 1'
-#     JobTimeoutInMinutes: 60
-#     vmImage: ${{ parameters.vmImageTest }}
-#     UITEST_SNAPSHOTS_ONLY: false
-#     UITEST_TEST_TIMEOUT: '3600000'
-#     UITEST_AUTOMATED_GROUP: 4
-#     UITEST_RUNTIME_TEST_GROUP: 1
-#     UITEST_RUNTIME_TEST_GROUP_COUNT: 3
-#     UITEST_ALLOW_RERUN: 'false'
-#     xCodeRoot: ${{ parameters.xCodeRootTest }}
-#     XamarinSDKVersion: ${{ parameters.XamarinSDKVersionTest }}
-# 
-# - template: .azure-devops-ios-tests-run.yml
-#   parameters:
-#     nugetPackages: $(NUGET_PACKAGES)
-#     JobName: 'iOS_Automated_Tests_Runtime_Tests_group_2'
-#     JobDisplayName: 'Run iOS Automated Runtime Tests Group 2'
-#     JobTimeoutInMinutes: 60
+#     JobName: 'iOS_Automated_Tests_Benchmarks'
+#     JobDisplayName: 'iOS Automated Benchmarks'
+#     JobTimeoutInMinutes: 40
 #     vmImage: ${{ parameters.vmImageTest }}
 #     UITEST_SNAPSHOTS_ONLY: false
-#     UITEST_TEST_TIMEOUT: '3600000'
-#     UITEST_AUTOMATED_GROUP: 4
-#     UITEST_RUNTIME_TEST_GROUP: 2
-#     UITEST_RUNTIME_TEST_GROUP_COUNT: 3
-#     UITEST_ALLOW_RERUN: 'false'
+#     UITEST_AUTOMATED_GROUP: Benchmarks
 #     xCodeRoot: ${{ parameters.xCodeRootTest }}
 #     XamarinSDKVersion: ${{ parameters.XamarinSDKVersionTest }}
-# 
-# # Disabled for benchmkarks timeout
-# # - template: .azure-devops-ios-tests-run.yml
-# #   parameters:
-# #     nugetPackages: $(NUGET_PACKAGES)
-# #     JobName: 'iOS_Automated_Tests_Benchmarks'
-# #     JobDisplayName: 'iOS Automated Benchmarks'
-# #     JobTimeoutInMinutes: 40
-# #     vmImage: ${{ parameters.vmImageTest }}
-# #     UITEST_SNAPSHOTS_ONLY: false
-# #     UITEST_AUTOMATED_GROUP: Benchmarks
-# #     xCodeRoot: ${{ parameters.xCodeRootTest }}
-# #     XamarinSDKVersion: ${{ parameters.XamarinSDKVersionTest }}
-# 
-# - ${{ each snapshotGroup in parameters.SNAPSHOT_GROUPS }}:
-#   - template: .azure-devops-ios-tests-run.yml
-#     parameters:
-#       nugetPackages: $(NUGET_PACKAGES)
-#       JobName: 'iOS_Snapshot_Tests_Group_${{ snapshotGroup.value }}'
-#       JobDisplayName: 'Run iOS Snapshot Tests Group ${{ snapshotGroup.value }}'
-#       JobTimeoutInMinutes: 45
-#       vmImage: ${{ parameters.vmImageTest }}
-#       UITEST_SNAPSHOTS_ONLY: true
-#       UITEST_SNAPSHOTS_GROUP: ${{ snapshotGroup.key }}
-#       UITEST_TEST_TIMEOUT: '270000'
-#       UITEST_ALLOW_RERUN: 'true'
-#       xCodeRoot: ${{ parameters.xCodeRootTest }}
-#       XamarinSDKVersion: ${{ parameters.XamarinSDKVersionTest }}
+
+- ${{ each snapshotGroup in parameters.SNAPSHOT_GROUPS }}:
+  - template: .azure-devops-ios-tests-run.yml
+    parameters:
+      nugetPackages: $(NUGET_PACKAGES)
+      JobName: 'iOS_Snapshot_Tests_Group_${{ snapshotGroup.value }}'
+      JobDisplayName: 'iOS Snapshot Tests ${{ snapshotGroup.value }}'
+      JobTimeoutInMinutes: 60
+      vmImage: ${{ parameters.vmImageTest }}
+      UITEST_SNAPSHOTS_ONLY: true
+      UITEST_SNAPSHOTS_GROUP: ${{ snapshotGroup.key }}
+      UITEST_TEST_TIMEOUT: '10m'
+      UITEST_ALLOW_RERUN: 'false'
+      xCodeRoot: ${{ parameters.xCodeRootTest }}
+      XamarinSDKVersion: ${{ parameters.XamarinSDKVersionTest }}
diff --git a/build/ci/.azure-devops-samplesapp-uitests-build.yml b/build/ci/.azure-devops-samplesapp-uitests-build.yml