diff --git a/.github/workflows/integration-tests-benchmarks.yml b/.github/workflows/integration-tests-benchmarks.yml deleted file mode 100644 index 66e4498dcb5..00000000000 --- a/.github/workflows/integration-tests-benchmarks.yml +++ /dev/null @@ -1,114 +0,0 @@ -name: 'Integration Tests - Benchmarks' -on: - schedule: - - cron: '0 0 * * *' # every night at midnight UTC - pull_request: - paths: - - '**/sentry/**' - - '**/sentry-android/**' - - '**/sentry-android-core/**' - - '**/sentry-android-ndk/**' - - '**/sentry-android-integration-tests/**' - - '**/.github/**' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - name: Benchmarks - runs-on: ubuntu-latest - - # we copy the secret to the env variable in order to access it in the workflow - env: - SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} - GRADLE_ENCRYPTION_KEY: ${{ secrets.GRADLE_ENCRYPTION_KEY }} - - steps: - - name: Git checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - submodules: 'recursive' - - - name: 'Set up Java: 17' - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5 - with: - distribution: 'temurin' - java-version: '17' - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6.3.0 - with: - cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} - - # Clean, build and release a test apk, but only if we will run the benchmark - - name: Make assembleBenchmarks - if: env.SAUCE_USERNAME != null - run: make assembleBenchmarks - - - name: Run All Tests in SauceLab - uses: saucelabs/saucectl-run-action@283660aa934c02723c497efa151d582a3acc5801 # pin@v3 - if: github.event_name != 'pull_request' && env.SAUCE_USERNAME != null - env: - GITHUB_TOKEN: ${{ github.token }} - with: - sauce-username: ${{ secrets.SAUCE_USERNAME }} - sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} - config-file: .sauce/sentry-uitest-android-benchmark.yml - - - name: Run one test in SauceLab - uses: saucelabs/saucectl-run-action@283660aa934c02723c497efa151d582a3acc5801 # pin@v3 - if: github.event_name == 'pull_request' && env.SAUCE_USERNAME != null - env: - GITHUB_TOKEN: ${{ github.token }} - with: - sauce-username: ${{ secrets.SAUCE_USERNAME }} - sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} - config-file: .sauce/sentry-uitest-android-benchmark-lite.yml - - app-metrics: - runs-on: ubuntu-latest - - # we copy the secret to the env variable in order to access it in the workflow - env: - SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} - GRADLE_ENCRYPTION_KEY: ${{ secrets.GRADLE_ENCRYPTION_KEY }} - - steps: - - name: Git checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - submodules: 'recursive' - - - name: 'Set up Java: 17' - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5 - with: - distribution: 'temurin' - java-version: '17' - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6.3.0 - with: - cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} - - - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - id: app-plain-cache - with: - path: sentry-android-integration-tests/test-app-plain/build/outputs/apk/release/test-app-plain-release.apk - key: ${{ github.workflow }}-${{ github.job }}-appplain-${{ hashFiles('sentry-android-integration-tests/test-app-plain/**') }} - - - name: Build Android app plain - if: steps.app-plain-cache.outputs['cache-hit'] != 'true' - run: ./gradlew :sentry-android-integration-tests:test-app-plain:assembleRelease - - - name: Build Android app with Sentry - run: ./gradlew :sentry-android-integration-tests:test-app-sentry:assembleRelease - - - name: Collect app metrics - uses: getsentry/action-app-sdk-overhead-metrics@44fb5489ac4ac252c87d84811972dc93a1e490b8 - with: - config: sentry-android-integration-tests/metrics-test.yml - sauce-user: ${{ secrets.SAUCE_USERNAME }} - sauce-key: ${{ secrets.SAUCE_ACCESS_KEY }} - if: env.SAUCE_USERNAME != null diff --git a/.sauce/sentry-uitest-android-benchmark-lite.yml b/.sauce/sentry-uitest-android-benchmark-lite.yml deleted file mode 100644 index fec4a141def..00000000000 --- a/.sauce/sentry-uitest-android-benchmark-lite.yml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: v1alpha -kind: espresso -sauce: - region: us-west-1 - concurrency: 1 - metadata: - build: sentry-uitest-android-benchmark-lite-$GITHUB_REF-$GITHUB_SHA - tags: - - benchmarks - - android - -defaults: - timeout: 40m - -espresso: - app: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/release/sentry-uitest-android-benchmark-release.apk - testApp: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/androidTest/release/sentry-uitest-android-benchmark-release-androidTest.apk - -suites: - - - name: "Android 15 Benchmark lite (api 35)" - testOptions: - clearPackageData: true - useTestOrchestrator: true - devices: - - name: ".*" - platformVersion: "15" - -artifacts: - download: - when: always - match: - - junit.xml - directory: ./artifacts/ diff --git a/.sauce/sentry-uitest-android-benchmark.yml b/.sauce/sentry-uitest-android-benchmark.yml deleted file mode 100644 index 12995ea5e07..00000000000 --- a/.sauce/sentry-uitest-android-benchmark.yml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: v1alpha -kind: espresso -sauce: - region: us-west-1 - concurrency: 3 - metadata: - build: sentry-uitest-android-benchmark-$GITHUB_REF-$GITHUB_SHA - tags: - - benchmarks - - android - -defaults: - timeout: 90m - -espresso: - app: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/release/sentry-uitest-android-benchmark-release.apk - testApp: ./sentry-android-integration-tests/sentry-uitest-android-benchmark/build/outputs/apk/androidTest/release/sentry-uitest-android-benchmark-release-androidTest.apk - -suites: - - # Devices are chosen so that there is a high-end and a low-end device for each api level - - name: "Android 15 (api 35)" - testOptions: - clearPackageData: true - useTestOrchestrator: true - devices: - - id: Google_Pixel_9_Pro_XL_15_real_sjc1 # Google Pixel 9 Pro XL - api 35 (15) - high end - - id: Samsung_Galaxy_S23_15_real_sjc1 # Samsung Galaxy S23 - api 35 (15) - mid end - - id: Google_Pixel_6a_15_real_sjc1 # Google Pixel 6a - api 35 (15) - low end - - - name: "Android 14 (api 34)" - testOptions: - clearPackageData: true - useTestOrchestrator: true - devices: - - id: Google_Pixel_9_Pro_XL_real_sjc1 # Google Pixel 9 Pro XL - api 34 (14) - high end - - id: Samsung_Galaxy_A54_real_sjc1 # Samsung Galaxy A54 - api 34 (14) - low end - - - name: "Android 13 (api 33)" - testOptions: - clearPackageData: true - useTestOrchestrator: true - devices: - - id: Google_Pixel_7_Pro_real_us # Google Pixel 7 Pro - api 33 (13) - high end - - id: Samsung_Galaxy_A32_5G_real_sjc1 # Samsung Galaxy A32 5G - api 33 (13) - low end - -# At the time of writing (August, 13, 2025), the market share per android version is: -# 15.0 = 26.75%, 14.0 = 19.5%, 13 = 15.95% -# Using these 3 versions we cover 62.2% of all devices out there. Currently, this is enough for benchmarking scope - -artifacts: - download: - when: always - match: - - junit.xml - directory: ./artifacts/ diff --git a/Makefile b/Makefile index 3967ff856ad..2b75e56160b 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,6 @@ .PHONY: all clean compile javadocs dryRelease update checkFormat api assembleBenchmarkTestRelease assembleUiTestRelease assembleUiTestCriticalRelease runUiTestCritical setupPython systemTest systemTestInteractive check preMerge publish all: stop clean javadocs compile -assembleBenchmarks: assembleBenchmarkTestRelease assembleUiTests: assembleUiTestRelease preMerge: check publish: clean dryRelease diff --git a/build.gradle.kts b/build.gradle.kts index a663628b467..5aae1f0bc4e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -79,8 +79,6 @@ apiValidation { "sentry-uitest-android", "sentry-uitest-android-benchmark", "sentry-uitest-android-critical", - "test-app-plain", - "test-app-sentry", "test-app-size", "sentry-samples-netflix-dgs", "sentry-samples-console-otlp", diff --git a/sentry-android-integration-tests/README.md b/sentry-android-integration-tests/README.md index ecbd15b34f5..46443752f27 100644 --- a/sentry-android-integration-tests/README.md +++ b/sentry-android-integration-tests/README.md @@ -1,8 +1,5 @@ # Android performance-impact and integration tests -* [Sample app without sentry](./test-app-plain) created with Android Studio -> New Project -> Basic Activity -* [Same app, but with Sentry included](./test-app-sentry) - made part of the root project -* [App metrics test specification (yaml)](./metrics-test.yml) -* [Espresso-based benchmarks](./sentry-uitest-android-benchmark) - run within SauceLabs (see /.sauce/*.yml) +* [Espresso-based benchmarks](./sentry-uitest-android-benchmark) - no longer wired into CI, run locally * [Espresso-based UI tests](./sentry-uitest-android) - run within SauceLabs (see /.sauce/*.yml) * Also used for compatibility test matrix against new AGP versions diff --git a/sentry-android-integration-tests/metrics-test.yml b/sentry-android-integration-tests/metrics-test.yml deleted file mode 100644 index a73ca1ef7c0..00000000000 --- a/sentry-android-integration-tests/metrics-test.yml +++ /dev/null @@ -1,12 +0,0 @@ -apps: - - name: io.sentry.java.tests.perf.appplain - activity: MainActivity - path: ./test-app-plain/build/outputs/apk/release/test-app-plain-release.apk - - name: io.sentry.java.tests.perf.appsentry - activity: MainActivity - path: ./test-app-sentry/build/outputs/apk/release/test-app-sentry-release.apk - -startupTimeTest: - runs: 50 - diffMin: 0 - diffMax: 150 diff --git a/sentry-android-integration-tests/test-app-plain/.gitignore b/sentry-android-integration-tests/test-app-plain/.gitignore deleted file mode 100644 index 42afabfd2ab..00000000000 --- a/sentry-android-integration-tests/test-app-plain/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/build \ No newline at end of file diff --git a/sentry-android-integration-tests/test-app-plain/build.gradle.kts b/sentry-android-integration-tests/test-app-plain/build.gradle.kts deleted file mode 100644 index 9778363ede8..00000000000 --- a/sentry-android-integration-tests/test-app-plain/build.gradle.kts +++ /dev/null @@ -1,51 +0,0 @@ -plugins { id("com.android.application") } - -android { - compileSdk = libs.versions.compileSdk.get().toInt() - namespace = "io.sentry.java.tests.perf.appplain" - - defaultConfig { - applicationId = "io.sentry.java.tests.perf.appplain" - minSdk = libs.versions.minSdk.get().toInt() - targetSdk = libs.versions.targetSdk.get().toInt() - versionCode = 1 - versionName = "1.0" - } - - buildTypes { - getByName("release") { - isMinifyEnabled = true - signingConfig = signingConfigs.getByName("debug") // to be able to run release mode - proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro") - ndk { - abiFilters.clear() - abiFilters.add("arm64-v8a") - } - } - } - compileOptions { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 - } - buildFeatures { viewBinding = true } - signingConfigs { - getByName("debug") { - storeFile = rootProject.file("debug.keystore") - storePassword = "android" - keyAlias = "androiddebugkey" - keyPassword = "android" - } - } - - androidComponents.beforeVariants { - it.enable = !Config.Android.shouldSkipDebugVariant(it.buildType) - } -} - -dependencies { - implementation("androidx.appcompat:appcompat:1.3.0") - implementation("com.google.android.material:material:1.4.0") - implementation("androidx.constraintlayout:constraintlayout:2.2.1") - implementation("androidx.navigation:navigation-fragment:2.3.5") - implementation("androidx.navigation:navigation-ui:2.3.5") -} diff --git a/sentry-android-integration-tests/test-app-plain/proguard-rules.pro b/sentry-android-integration-tests/test-app-plain/proguard-rules.pro deleted file mode 100644 index d5b66a5b790..00000000000 --- a/sentry-android-integration-tests/test-app-plain/proguard-rules.pro +++ /dev/null @@ -1,23 +0,0 @@ -# Add project specific ProGuard rules here. -# You can control the set of applied configuration files using the -# proguardFiles setting in build.gradle.kts. -# -# For more details, see -# http://developer.android.com/guide/developing/tools/proguard.html - -# If your project uses WebView with JS, uncomment the following -# and specify the fully qualified class name to the JavaScript interface -# class: -#-keepclassmembers class fqcn.of.javascript.interface.for.webview { -# public *; -#} - -# Uncomment this to preserve the line number information for -# debugging stack traces. -#-keepattributes SourceFile,LineNumberTable - -# If you keep the line number information, uncomment this to -# hide the original source file name. -#-renamesourcefileattribute SourceFile - --keep,allowobfuscation,allowshrinking class * extends androidx.navigation.Navigator diff --git a/sentry-android-integration-tests/test-app-plain/src/main/AndroidManifest.xml b/sentry-android-integration-tests/test-app-plain/src/main/AndroidManifest.xml deleted file mode 100644 index 9b9f7a910cb..00000000000 --- a/sentry-android-integration-tests/test-app-plain/src/main/AndroidManifest.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - - - - - - - - - diff --git a/sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/FirstFragment.java b/sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/FirstFragment.java deleted file mode 100644 index 473aaf8aa76..00000000000 --- a/sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/FirstFragment.java +++ /dev/null @@ -1,42 +0,0 @@ -package io.sentry.java.tests.perf.appplain; - -import android.os.Bundle; -import android.view.LayoutInflater; -import android.view.View; -import android.view.ViewGroup; -import androidx.annotation.NonNull; -import androidx.fragment.app.Fragment; -import androidx.navigation.fragment.NavHostFragment; -import io.sentry.java.tests.perf.appplain.databinding.FragmentFirstBinding; - -public class FirstFragment extends Fragment { - - private FragmentFirstBinding binding; - - @Override - public View onCreateView( - LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) { - - binding = FragmentFirstBinding.inflate(inflater, container, false); - return binding.getRoot(); - } - - public void onViewCreated(@NonNull View view, Bundle savedInstanceState) { - super.onViewCreated(view, savedInstanceState); - - binding.buttonFirst.setOnClickListener( - new View.OnClickListener() { - @Override - public void onClick(View view) { - NavHostFragment.findNavController(FirstFragment.this) - .navigate(R.id.action_FirstFragment_to_SecondFragment); - } - }); - } - - @Override - public void onDestroyView() { - super.onDestroyView(); - binding = null; - } -} diff --git a/sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/MainActivity.java b/sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/MainActivity.java deleted file mode 100644 index e37de2be3a3..00000000000 --- a/sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/MainActivity.java +++ /dev/null @@ -1,74 +0,0 @@ -package io.sentry.java.tests.perf.appplain; - -import android.os.Bundle; -import android.view.Menu; -import android.view.MenuItem; -import android.view.View; -import androidx.appcompat.app.AppCompatActivity; -import androidx.navigation.NavController; -import androidx.navigation.Navigation; -import androidx.navigation.ui.AppBarConfiguration; -import androidx.navigation.ui.NavigationUI; -import com.google.android.material.snackbar.Snackbar; -import io.sentry.java.tests.perf.appplain.databinding.ActivityMainBinding; - -public class MainActivity extends AppCompatActivity { - - private AppBarConfiguration appBarConfiguration; - private ActivityMainBinding binding; - - @Override - protected void onCreate(Bundle savedInstanceState) { - super.onCreate(savedInstanceState); - - binding = ActivityMainBinding.inflate(getLayoutInflater()); - setContentView(binding.getRoot()); - - setSupportActionBar(binding.toolbar); - - NavController navController = - Navigation.findNavController(this, R.id.nav_host_fragment_content_main); - appBarConfiguration = new AppBarConfiguration.Builder(navController.getGraph()).build(); - NavigationUI.setupActionBarWithNavController(this, navController, appBarConfiguration); - - binding.fab.setOnClickListener( - new View.OnClickListener() { - @Override - public void onClick(View view) { - Snackbar.make(view, "Replace with your own action", Snackbar.LENGTH_LONG) - .setAction("Action", null) - .show(); - } - }); - } - - @Override - public boolean onCreateOptionsMenu(Menu menu) { - // Inflate the menu; this adds items to the action bar if it is present. - getMenuInflater().inflate(R.menu.menu_main, menu); - return true; - } - - @Override - public boolean onOptionsItemSelected(MenuItem item) { - // Handle action bar item clicks here. The action bar will - // automatically handle clicks on the Home/Up button, so long - // as you specify a parent activity in AndroidManifest.xml. - int id = item.getItemId(); - - //noinspection SimplifiableIfStatement - if (id == R.id.action_settings) { - return true; - } - - return super.onOptionsItemSelected(item); - } - - @Override - public boolean onSupportNavigateUp() { - NavController navController = - Navigation.findNavController(this, R.id.nav_host_fragment_content_main); - return NavigationUI.navigateUp(navController, appBarConfiguration) - || super.onSupportNavigateUp(); - } -} diff --git a/sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/SecondFragment.java b/sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/SecondFragment.java deleted file mode 100644 index cf04c92febf..00000000000 --- a/sentry-android-integration-tests/test-app-plain/src/main/java/io/sentry/java/tests/perf/appplain/SecondFragment.java +++ /dev/null @@ -1,42 +0,0 @@ -package io.sentry.java.tests.perf.appplain; - -import android.os.Bundle; -import android.view.LayoutInflater; -import android.view.View; -import android.view.ViewGroup; -import androidx.annotation.NonNull; -import androidx.fragment.app.Fragment; -import androidx.navigation.fragment.NavHostFragment; -import io.sentry.java.tests.perf.appplain.databinding.FragmentSecondBinding; - -public class SecondFragment extends Fragment { - - private FragmentSecondBinding binding; - - @Override - public View onCreateView( - LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) { - - binding = FragmentSecondBinding.inflate(inflater, container, false); - return binding.getRoot(); - } - - public void onViewCreated(@NonNull View view, Bundle savedInstanceState) { - super.onViewCreated(view, savedInstanceState); - - binding.buttonSecond.setOnClickListener( - new View.OnClickListener() { - @Override - public void onClick(View view) { - NavHostFragment.findNavController(SecondFragment.this) - .navigate(R.id.action_SecondFragment_to_FirstFragment); - } - }); - } - - @Override - public void onDestroyView() { - super.onDestroyView(); - binding = null; - } -} diff --git a/sentry-android-integration-tests/test-app-plain/src/main/res/drawable-v24/ic_launcher_foreground.xml b/sentry-android-integration-tests/test-app-plain/src/main/res/drawable-v24/ic_launcher_foreground.xml deleted file mode 100644 index 2b068d11462..00000000000 --- a/sentry-android-integration-tests/test-app-plain/src/main/res/drawable-v24/ic_launcher_foreground.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/sentry-android-integration-tests/test-app-plain/src/main/res/drawable/ic_launcher_background.xml b/sentry-android-integration-tests/test-app-plain/src/main/res/drawable/ic_launcher_background.xml deleted file mode 100644 index 07d5da9cbf1..00000000000 --- a/sentry-android-integration-tests/test-app-plain/src/main/res/drawable/ic_launcher_background.xml +++ /dev/null @@ -1,170 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/sentry-android-integration-tests/test-app-plain/src/main/res/layout/activity_main.xml b/sentry-android-integration-tests/test-app-plain/src/main/res/layout/activity_main.xml deleted file mode 100644 index 1acd302c3fc..00000000000 --- a/sentry-android-integration-tests/test-app-plain/src/main/res/layout/activity_main.xml +++ /dev/null @@ -1,35 +0,0 @@ - - - - - - - - - - - - - - diff --git a/sentry-android-integration-tests/test-app-plain/src/main/res/layout/content_main.xml b/sentry-android-integration-tests/test-app-plain/src/main/res/layout/content_main.xml deleted file mode 100644 index e416e1c18d5..00000000000 --- a/sentry-android-integration-tests/test-app-plain/src/main/res/layout/content_main.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - \ No newline at end of file diff --git a/sentry-android-integration-tests/test-app-plain/src/main/res/layout/fragment_first.xml b/sentry-android-integration-tests/test-app-plain/src/main/res/layout/fragment_first.xml deleted file mode 100644 index fb44a3d9176..00000000000 --- a/sentry-android-integration-tests/test-app-plain/src/main/res/layout/fragment_first.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - - -