From df531e15e6499084c48de321cfbe45d0748993bf Mon Sep 17 00:00:00 2001 From: igorpeshansky <7594381+igorpeshansky@users.noreply.github.com> Date: Thu, 20 Jun 2024 19:45:17 -0400 Subject: [PATCH] Testing: Add TEST_SELECTOR to Kokoro scripts to allow running a subset of the tests. (#1739) --- integration_test/README.md | 18 +++++++++------- .../third_party_apps_test/main_test.go | 21 +++++++++++++++++-- kokoro/scripts/test/go_test.sh | 3 +++ 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/integration_test/README.md b/integration_test/README.md index 2cc52bea6b..0463238ca9 100644 --- a/integration_test/README.md +++ b/integration_test/README.md @@ -14,7 +14,7 @@ a firewall that allows connections over port 22 for ssh. It is recommended for Googlers to use our prebuilt testing project. Ask a teammate (e.g. martijnvs@) for the project ID. -You will also need a GCS bucket that is used to transfer files onto the +You will also need a GCS bucket that is used to transfer files onto the testing VMs. This is referred to as `${TRANSFERS_BUCKET}`. For Googlers, `stackdriver-test-143416-untrusted-file-transfers` is recommended. @@ -46,15 +46,15 @@ When the setup steps are complete, you can run ops_agent_test from the ``` make integration_tests PROJECT=${PROJECT} TRANSFERS_BUCKET=${TRANSFERS_BUCKET} ``` -Alternatively, you can export `PROJECT` and `TRANSFERS_BUCKET` in your -environment and simply call the target. -You can also specify the `ZONES` and `PLATFORMS` variables if you would like +Alternatively, you can export `PROJECT` and `TRANSFERS_BUCKET` in your +environment and simply call the target. +You can also specify the `ZONES` and `IMAGE_SPECS` variables if you would like to run the tests on something other than the defaults (`us-central1-b` for -ZONES and `debian-cloud:debian-11` for `PLATFORMS`). +ZONES and `debian-cloud:debian-11` for `IMAGE_SPECS`). The above command will run the tests against the stable Ops Agent. To test against a pre-built but unreleased agent, you can use add the -AGENT_PACKAGES_IN_GCS environment variable onto your command like this: +`AGENT_PACKAGES_IN_GCS` environment variable onto your command like this: ``` AGENT_PACKAGES_IN_GCS=gs://ops-agents-public-buckets-test-logs/prod/stackdriver_agents/testing/consumer/ops_agent/build/buster/2068/20220926-132259/result \ @@ -99,7 +99,7 @@ go test -v ./integration_test \ -test.run="TestThirdPartyApps/.*/(nvml|dcgm)" ``` -Make sure the platform you specify is included in the PLATFORMS environment +Make sure the platform you specify is included in the `IMAGE_SPECS` environment variable. ### Testing Flow @@ -130,6 +130,10 @@ The test is designed so that simply modifying files in the right thing. But we do expect that we will need to make big changes to both the data directory and the test runner before it is really meeting our needs. +By default, the test will skip any applications that were not +impacted by the currently modified set of files. However, if the modified files +are unrelated to any apps, it assumes that all apps are impacted. + ### Adding a new third-party application You will need to add and modify a few files. Start by adding your new diff --git a/integration_test/third_party_apps_test/main_test.go b/integration_test/third_party_apps_test/main_test.go index c9d8c4a614..2f1f606490 100644 --- a/integration_test/third_party_apps_test/main_test.go +++ b/integration_test/third_party_apps_test/main_test.go @@ -31,7 +31,7 @@ REPO_SUFFIX: If provided, a package repository suffix to install the agent from. AGENT_PACKAGES_IN_GCS takes precedence over REPO_SUFFIX. ARTIFACT_REGISTRY_REGION: If provided, signals to the install scripts that the above REPO_SUFFIX is an artifact registry repo and specifies what region it - is in. + is in. */ package third_party_apps_test @@ -39,6 +39,7 @@ package third_party_apps_test import ( "context" "embed" + "flag" "fmt" "log" "math" @@ -745,10 +746,20 @@ func isCriticalFile(f string) bool { // integration_test/third_party_apps_test/applications// // // Checks the extracted app names against the set of all known apps. +// If tests were explicitly selected, or if no app is found as impacted, assume +// all apps are. func determineImpactedApps(modifiedFiles []string, allApps map[string]metadata.IntegrationMetadata) map[string]bool { impactedApps := make(map[string]bool) defer log.Printf("impacted apps: %v", impactedApps) + if flag.Lookup("test.run") != nil { + // Honor explicit test selectors. + for app := range allApps { + impactedApps[app] = true + } + return impactedApps + } + for _, f := range modifiedFiles { if isCriticalFile(f) { // Consider all apps as impacted. @@ -761,7 +772,6 @@ func determineImpactedApps(modifiedFiles []string, allApps map[string]metadata.I for _, f := range modifiedFiles { if strings.HasPrefix(f, "apps/") { - // File names: apps/.go f := strings.TrimPrefix(f, "apps/") f = strings.TrimSuffix(f, ".go") @@ -782,8 +792,15 @@ func determineImpactedApps(modifiedFiles []string, allApps map[string]metadata.I // The directories here are already authoritative, no // need to check against list. impactedApps[f] = true + } + } + if len(impactedApps) == 0 { + // If none of the apps are impacted, treat all of them as impacted. + for app := range allApps { + impactedApps[app] = true } + return impactedApps } return impactedApps } diff --git a/kokoro/scripts/test/go_test.sh b/kokoro/scripts/test/go_test.sh index ef64bbcd44..10a00be4a6 100755 --- a/kokoro/scripts/test/go_test.sh +++ b/kokoro/scripts/test/go_test.sh @@ -208,6 +208,9 @@ go_test_args=( if [[ "${SHORT:-false}" == "true" ]]; then go_test_args+=( "-test.short" ) fi +if [[ -n "${TEST_SELECTOR:-}" ]]; then + go_test_args+=( "-test.run=${TEST_SELECTOR}" ) +fi TEST_UNDECLARED_OUTPUTS_DIR="${LOGS_DIR}" \ gotestsum "${gotestsum_args[@]}" \