Merge branch 'main' into enable-self-instrumentation-tracing

.github/workflows/benchmarks.yml

@@ -51,6 +51,7 @@ jobs:
       SSH_KEY: ./id_rsa_terraform
       TF_VAR_private_key: ./id_rsa_terraform
       TF_VAR_public_key: ./id_rsa_terraform.pub
+      TF_VAR_run_standalone: ${{ inputs.runStandalone || github.event.schedule=='0 5 */5 * *' }}
       RUN_STANDALONE: ${{ inputs.runStandalone || github.event.schedule=='0 5 */5 * *' }}
       TFVARS_SOURCE: ${{ inputs.profile || 'system-profiles/8GBx1zone.tfvars' }} # // Default to use an 8gb profile
       TF_VAR_BUILD_ID: ${{ github.run_id }}
@@ -133,10 +134,6 @@ jobs:
           admin_console_url=$(terraform output -raw admin_console_url)
           echo "admin_console_url=$admin_console_url" >> "$GITHUB_OUTPUT"
           echo "-> infra setup done"
-        env:
-          TF_VAR_worker_region: ${{ env.AWS_REGION }}
-          TF_VAR_run_standalone: ${{ env.RUN_STANDALONE }}
-
       - name: Run benchmarks autotuned
         if: ${{ inputs.benchmarkAgents == '' }}
         run: make run-benchmark-autotuned
@@ -189,7 +186,7 @@ jobs:
 
       - name: Copy CPU profile
         run: make cp-cpuprof
-      
+
       - name: Upload CPU profile
         uses: actions/upload-artifact@v4
         with:
@@ -234,6 +231,17 @@ jobs:
           GITHUB_TOKEN: ${{ steps.get_token.outputs.token }}
           WORKFLOW: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}
 
+      # Secrets are rotated daily, if the benchmarks run between the rotation window, then
+      # there is a high chance things will stop working
+      # This is trying to reduce the chances of that happening.
+      # See https://github.com/elastic/observability-test-environments/actions/workflows/cluster-rotate-api-keys.yml
+      - uses: google-github-actions/get-secretmanager-secrets@95a0b09b8348ef3d02c68c6ba5662a037e78d713 # v2.1.4
+        if: always()
+        with:
+          export_to_environment: true
+          secrets: |-
+            EC_API_KEY:elastic-observability/elastic-cloud-observability-team-pro-api-key
+
       - name: Tear down benchmark environment
         if: always()
         run: make init destroy

.github/workflows/smoke-tests-ess.yml

@@ -72,6 +72,18 @@ jobs:
 
       - name: Run smoke tests ${{ matrix.test }} for ${{ matrix.version }}
         run: make smoketest/run-version TEST_DIR=${{ matrix.test }} SMOKETEST_VERSION=${{ matrix.version }}
+
+      # Secrets are rotated daily, if the benchmarks run between the rotation window, then
+      # there is a high chance things will stop working
+      # This is trying to reduce the chances of that happening.
+      # See https://github.com/elastic/observability-test-environments/actions/workflows/cluster-rotate-api-keys.yml
+      - uses: google-github-actions/get-secretmanager-secrets@95a0b09b8348ef3d02c68c6ba5662a037e78d713 # v2.1.4
+        if: always()
+        with:
+          export_to_environment: true
+          secrets: |-
+            EC_API_KEY:elastic-observability/elastic-cloud-observability-team-pro-api-key
+
       - if: always()
         name: Teardown smoke test infra
         run: make smoketest/cleanup TEST_DIR=${{ matrix.test }}

NOTICE.txt

@@ -829,11 +829,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.12.0
+Version: v0.12.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].1/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

docker-compose.yml

@@ -11,7 +11,7 @@ x-logging: &default-logging
 services:
   elasticsearch:
     # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-    image: docker.elastic.co/elasticsearch/elasticsearch:9.0.0-3b9dafe0-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.0.0-9b4a1f9a-SNAPSHOT
     ports:
       - 9200:9200
     healthcheck:
@@ -43,7 +43,7 @@ services:
 
   kibana:
     # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-    image: docker.elastic.co/kibana/kibana:9.0.0-3b9dafe0-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:9.0.0-9b4a1f9a-SNAPSHOT
     ports:
       - 5601:5601
     healthcheck:
@@ -63,7 +63,7 @@ services:
 
   metricbeat:
     # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-    image: docker.elastic.co/beats/metricbeat:9.0.0-3b9dafe0-SNAPSHOT
+    image: docker.elastic.co/beats/metricbeat:9.0.0-9b4a1f9a-SNAPSHOT
     environment:
       ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
       ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}"

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/elastic/apm-data v1.13.1
 	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240823193033-af33fad4d50b
 	github.com/elastic/elastic-agent-client/v7 v7.16.0
-	github.com/elastic/elastic-agent-libs v0.12.0
+	github.com/elastic/elastic-agent-libs v0.12.1
 	github.com/elastic/elastic-agent-system-metrics v0.11.3
 	github.com/elastic/gmux v0.3.2
 	github.com/elastic/go-docappender/v2 v2.3.0

go.sum

@@ -139,8 +139,8 @@ github.com/elastic/elastic-agent-autodiscover v0.8.1 h1:u6TWqh7wfevu6S4GUq4SIxYB
 github.com/elastic/elastic-agent-autodiscover v0.8.1/go.mod h1:0gzGsaDCAqBfUZjuCqqWsSI60eaZ778A5tQZV72rPV0=
 github.com/elastic/elastic-agent-client/v7 v7.16.0 h1:yKGq2+CxAuW8Kh0EoNl202tqAyQKfBcPRawVKs2Jve0=
 github.com/elastic/elastic-agent-client/v7 v7.16.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI=
-github.com/elastic/elastic-agent-libs v0.12.0 h1:xfVVCYIaI6XEPVpJNq7HQav7O/VxLj+YbQK/poWb7wA=
-github.com/elastic/elastic-agent-libs v0.12.0/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M=
+github.com/elastic/elastic-agent-libs v0.12.1 h1:5jkxMx15Bna8cq7/Sz/XUIVUXfNWiJ80iSk4ICQ7KJ0=
+github.com/elastic/elastic-agent-libs v0.12.1/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M=
 github.com/elastic/elastic-agent-system-metrics v0.11.3 h1:LDzRwP8kxvsYEtMDgMSKZs1TgPcSEukit+/EAP5Y28A=
 github.com/elastic/elastic-agent-system-metrics v0.11.3/go.mod h1:saqLKe9fuyuAo6IADAnnuy1kaBI7VNlxfwMo8KzSRyQ=
 github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA=

testing/benchmark/outputs.tf

@@ -43,6 +43,6 @@ output "apm_server_ip" {
 }
 
 output "admin_console_url" {
-  value       = var.run_standalone ? "" : module.ec_deployment[0].admin_console_url
+  value       = var.run_standalone ? "https://cloud.elastic.co/deployments" : module.ec_deployment[0].admin_console_url
   description = "The admin console URL"
 }

testing/infra/k8s/base/stack/apm-server.yaml

@@ -4,7 +4,7 @@ metadata:
   name: apm-server
 spec:
   # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-  version: 9.0.0-3b9dafe0-SNAPSHOT
+  version: 9.0.0-9b4a1f9a-SNAPSHOT
   count: 1
   http:
     tls:

testing/infra/k8s/base/stack/elasticsearch.yaml

@@ -4,7 +4,7 @@ metadata:
   name: elasticsearch
 spec:
   # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-  version: 9.0.0-3b9dafe0-SNAPSHOT
+  version: 9.0.0-9b4a1f9a-SNAPSHOT
   auth:
     fileRealm:
       - secretName: elasticsearch-admin

testing/infra/k8s/base/stack/kibana.yaml

@@ -4,7 +4,7 @@ metadata:
   name: kibana
 spec:
   # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-  version: 9.0.0-3b9dafe0-SNAPSHOT
+  version: 9.0.0-9b4a1f9a-SNAPSHOT
   count: 1
   elasticsearchRef:
     name: elasticsearch