Merge branch 'main' into ci/test-race

kruskall · Sep 23, 2024 · c332683 · c332683
2 parents 5300f8b + e258527
commit c332683
Show file tree

Hide file tree

Showing 88 changed files with 1,314 additions and 823 deletions.
diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh
@@ -34,7 +34,11 @@ DRA_BRANCH="$BUILDKITE_BRANCH"
 dra_command=collect
 BRANCHES_URL=https://storage.googleapis.com/artifacts-api/snapshots/branches.json
 curl -s "${BRANCHES_URL}" > active-branches.json
-if ! grep -q "\"$BUILDKITE_BRANCH\"" active-branches.json ; then
+# as long as `8.x` is not in the active branches, we will explicitly add the condition.
+if [ "$BUILDKITE_BRANCH" == "8.x" ] || grep -q "\"$BUILDKITE_BRANCH\"" active-branches.json ; then
+  echo "--- :arrow_right: Release Manager only supports the current active branches and 8.x, running"
+else
+  # If no active branches are found, let's see if it is a feature branch.
   echo "--- :arrow_right: Release Manager only supports the current active branches, skipping"
   echo "BUILDKITE_BRANCH=$BUILDKITE_BRANCH"
   echo "BUILDKITE_COMMIT=$BUILDKITE_COMMIT"
@@ -98,6 +102,7 @@ dra() {
 }
 
 dra "snapshot" "$dra_command"
-if [[ "${DRA_BRANCH}" != "main" ]]; then
+if [[ "${DRA_BRANCH}" != "main" && "${DRA_BRANCH}" != "8.x" ]]; then
+  echo "DRA_BRANCH is neither 'main' nor '8.x'"
   dra "staging" "$dra_command"
 fi
diff --git a/.ci/bump-elastic-stack-snapshot.yml → ...updatecli/bump-elastic-stack-snapshot.yml b/.ci/bump-elastic-stack-snapshot.yml → ...updatecli/bump-elastic-stack-snapshot.yml
@@ -21,8 +21,9 @@ scms:
     kind: github
     spec:
       user: '{{ requiredEnv "GITHUB_ACTOR" }}'
-      owner: elastic
-      repository: apm-server
+      username: '{{ requiredEnv "GITHUB_ACTOR" }}'
+      owner: '{{ .scm.owner }}'
+      repository: '{{ .scm.repository }}'
       token: '{{ requiredEnv "GITHUB_TOKEN" }}'
       branch: '{{ requiredEnv "BRANCH" }}'
       commitusingapi: true

diff --git a/.ci/bump-golang.yml → .ci/updatecli/bump-golang.yml b/.ci/bump-golang.yml → .ci/updatecli/bump-golang.yml
@@ -24,8 +24,9 @@ scms:
     kind: github
     spec:
       user: '{{ requiredEnv "GITHUB_ACTOR" }}'
-      owner: elastic
-      repository: apm-server
+      username: '{{ requiredEnv "GITHUB_ACTOR" }}'
+      owner: '{{ .scm.owner }}'
+      repository: '{{ .scm.repository }}'
       token: '{{ requiredEnv "GITHUB_TOKEN" }}'
       branch: '{{ requiredEnv "GITHUB_BRANCH" }}'
       commitusingapi: true

diff --git a/.ci/update-beats.yml → .ci/updatecli/update-beats.yml b/.ci/update-beats.yml → .ci/updatecli/update-beats.yml
@@ -7,8 +7,9 @@ scms:
     kind: github
     spec:
       user: '{{ requiredEnv "GITHUB_ACTOR" }}'
-      owner: elastic
-      repository: apm-server
+      username: '{{ requiredEnv "GITHUB_ACTOR" }}'
+      owner: '{{ .scm.owner }}'
+      repository: '{{ .scm.repository }}'
       token: '{{ requiredEnv "GITHUB_TOKEN" }}'
       branch: '{{ requiredEnv "BRANCH_NAME" }}'
       commitusingapi: true

diff --git a/.ci/updatecli/values.d/ironbank.yml b/.ci/updatecli/values.d/ironbank.yml
@@ -0,0 +1,2 @@
+config:
+  - path: packaging/ironbank
diff --git a/.ci/updatecli/values.d/scm.yml b/.ci/updatecli/values.d/scm.yml
@@ -0,0 +1,9 @@
+scm:
+  enabled: true
+  owner: elastic
+  repository: apm-server
+  branch: main
+  commitusingapi: true
+  # begin updatecli-compose policy values
+  user: obltmachine
+  # end updatecli-compose policy values
diff --git a/.ci/updatecli/values.d/updatecli-compose.yml b/.ci/updatecli/values.d/updatecli-compose.yml
@@ -0,0 +1,3 @@
+spec:
+  files:
+    - "updatecli-compose.yaml"
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -90,7 +90,7 @@ jobs:
         with:
           role-duration-seconds: 18000 # 5 hours
 
-      - uses: google-github-actions/get-secretmanager-secrets@dc4a1392bad0fd60aee00bb2097e30ef07a1caae # v2.1.3
+      - uses: google-github-actions/get-secretmanager-secrets@95a0b09b8348ef3d02c68c6ba5662a037e78d713 # v2.1.4
         with:
           export_to_environment: true
           secrets: |-
@@ -170,7 +170,7 @@ jobs:
 
       # Notify result to Slack only on schedule (nightly run)
       - if: github.event_name == 'schedule'
-        uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
+        uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
         with:

diff --git a/.github/workflows/bump-elastic-stack.yml b/.github/workflows/bump-elastic-stack.yml
@@ -17,7 +17,7 @@ jobs:
       matrix: ${{ steps.generator.outputs.matrix }}
     steps:
       - id: generator
-        uses: elastic/apm-pipeline-library/.github/actions/elastic-stack-snapshot-branches@current
+        uses: elastic/oblt-actions/elastic/active-branches@v1
 
   bump-elastic-stack:
     runs-on: ubuntu-latest
@@ -30,15 +30,27 @@ jobs:
         with:
           ref: ${{ matrix.branch }}
 
-      - uses: elastic/oblt-actions/updatecli/[email protected]
+      - name: Get token
+        id: get_token
+        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
         with:
-          command: --experimental apply --config .ci/bump-elastic-stack-snapshot.yml
+          app_id: ${{ secrets.OBS_AUTOMATION_APP_ID }}
+          private_key: ${{ secrets.OBS_AUTOMATION_APP_PEM }}
+          permissions: >-
+            {
+              "contents": "write",
+              "pull_requests": "write"
+            }
+
+      - uses: elastic/oblt-actions/updatecli/run@v1
+        with:
+          command: --experimental apply --config .ci/updatecli/bump-elastic-stack-snapshot.yml --values .ci/updatecli/values.d/scm.yml
         env:
           BRANCH: ${{ matrix.branch }}
-          GITHUB_TOKEN: ${{ secrets.UPDATECLI_GH_TOKEN }}
+          GITHUB_TOKEN: ${{ steps.get_token.outputs.token }}
 
       - if: ${{ failure()  }}
-        uses: elastic/oblt-actions/slack/send@v1.9.1
+        uses: elastic/oblt-actions/slack/send@v1
         with:
           channel-id: '#apm-server'
           message: ":traffic_cone: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, @robots-ci please look what's going on <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>"

diff --git a/.github/workflows/bump-golang.yml b/.github/workflows/bump-golang.yml
@@ -19,7 +19,7 @@ jobs:
     steps:
       - id: generate
         name: Generate matrix
-        uses: elastic/apm-pipeline-library/.github/actions/elastic-stack-snapshot-branches@current
+        uses: elastic/oblt-actions/elastic/active-branches@v1
         with:
           exclude-branches: '7.17,main'
       - uses: actions/github-script@v7
@@ -41,11 +41,23 @@ jobs:
 
       - uses: actions/checkout@v4
 
-      - uses: elastic/oblt-actions/updatecli/[email protected]
+      - name: Get token
+        id: get_token
+        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
         with:
-          command: --experimental apply --config .ci/bump-golang.yml
+          app_id: ${{ secrets.OBS_AUTOMATION_APP_ID }}
+          private_key: ${{ secrets.OBS_AUTOMATION_APP_PEM }}
+          permissions: >-
+            {
+              "contents": "write",
+              "pull_requests": "write"
+            }
+
+      - uses: elastic/oblt-actions/updatecli/run@v1
+        with:
+          command: --experimental apply --config .ci/updatecli/bump-golang.yml --values .ci/updatecli/values.d/scm.yml
         env:
-          GITHUB_TOKEN: ${{ secrets.UPDATECLI_GH_TOKEN }}
+          GITHUB_TOKEN: ${{ steps.get_token.outputs.token }}
           GITHUB_BRANCH: 'main'
           GITHUB_LABELS: ${{ needs.labels.outputs.backports }}
 
@@ -56,11 +68,11 @@ jobs:
         with:
           ref: '7.17'
 
-      - uses: elastic/oblt-actions/updatecli/run@v1.9.1
+      - uses: elastic/oblt-actions/updatecli/run@v1
         with:
-          command: --experimental apply --config .ci/bump-golang.yml
+          command: --experimental apply --config .ci/updatecli/bump-golang.yml --values .ci/updatecli/values.d/scm.yml
         env:
-          GITHUB_TOKEN: ${{ secrets.UPDATECLI_GH_TOKEN }}
+          GITHUB_TOKEN: ${{ steps.get_token.outputs.token }}
           GITHUB_BRANCH: '7.17'
           GITHUB_LABELS: 'backport-skip'
 
@@ -70,11 +82,11 @@ jobs:
     if: always()
     steps:
       - id: check
-        uses: elastic/apm-pipeline-library/.github/actions/check-dependent-jobs@current
+        uses: elastic/oblt-actions/check-dependent-jobs@v1
         with:
-          needs: ${{ toJSON(needs) }}
+          jobs: ${{ toJSON(needs) }}
       - if: ${{ steps.check.outputs.isSuccess == 'false' }}
-        uses: elastic/oblt-actions/slack/send@v1.9.1
+        uses: elastic/oblt-actions/slack/send@v1
         with:
           bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
           channel-id: "#apm-server"

diff --git a/.github/workflows/check-docker-compose.yml b/.github/workflows/check-docker-compose.yml
@@ -17,7 +17,7 @@ jobs:
       matrix: ${{ steps.generator.outputs.matrix }}
     steps:
       - id: generator
-        uses: elastic/apm-pipeline-library/.github/actions/elastic-stack-snapshot-branches@current
+        uses: elastic/oblt-actions/elastic/active-branches@v1
 
   check-docker-compose:
     needs:
@@ -47,9 +47,9 @@ jobs:
       - check-docker-compose
     steps:
       - id: check
-        uses: elastic/apm-pipeline-library/.github/actions/check-dependent-jobs@current
+        uses: elastic/oblt-actions/check-dependent-jobs@v1
         with:
-          needs: ${{ toJSON(needs) }}
+          jobs: ${{ toJSON(needs) }}
       - run: ${{ steps.check.outputs.isSuccess }}
       - if: failure()
         uses: elastic/oblt-actions/slack/notify-result@v1

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -111,3 +111,27 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
       - run: make publish-docker-images
+
+  # Only for forked PRs, when changing the .go-version, then we need to note that the wolfi docker image needs to be
+  # validated
+  validate-wolfi-docker-image:
+    runs-on: ubuntu-latest
+    if: ( github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == true ) || github.actor == 'dependabot[bot]'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@48d8f15b2aaa3d255ca5af3eba4870f807ce6b3c # v45.0.2
+        with:
+          files: .go-version
+
+      - name: If .go-version changed validate docker image is available.
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: |
+          echo "If you change the .go-version please use a branch in the upstream repository to validate the wolfi images with test-package-and-push."
+          echo "Otherwise, this validation will run and fail the CI build."
+          echo "Please validate the wolfi image is available by running the following command:"
+          echo "::notice::docker pull docker.elastic.co/wolfi/go:$(cat .go-version)"
+          echo "If they are available you could skip this validation."
+          echo "However, we recommend to use an upstream branch to run the CI specialised steps for the packaging system."
+          exit 1
diff --git a/.github/workflows/microbenchmark.yml b/.github/workflows/microbenchmark.yml
@@ -8,6 +8,7 @@ on:
     branches:
       - main
       - "8.[0-9]+"
+      - "8.x"
     paths-ignore:
       - '**.md'
       - '**.asciidoc'

diff --git a/.github/workflows/smoke-tests-ess.yml b/.github/workflows/smoke-tests-ess.yml
@@ -59,7 +59,7 @@ jobs:
 
       - uses: elastic/oblt-actions/google/auth@v1
 
-      - uses: google-github-actions/get-secretmanager-secrets@dc4a1392bad0fd60aee00bb2097e30ef07a1caae # v2.1.3
+      - uses: google-github-actions/get-secretmanager-secrets@95a0b09b8348ef3d02c68c6ba5662a037e78d713 # v2.1.4
         with:
           export_to_environment: true
           secrets: |-

diff --git a/.github/workflows/smoke-tests-os.yml b/.github/workflows/smoke-tests-os.yml
@@ -53,7 +53,7 @@ jobs:
 
       - uses: elastic/oblt-actions/google/auth@v1
 
-      - uses: google-github-actions/get-secretmanager-secrets@dc4a1392bad0fd60aee00bb2097e30ef07a1caae # v2.1.3
+      - uses: google-github-actions/get-secretmanager-secrets@95a0b09b8348ef3d02c68c6ba5662a037e78d713 # v2.1.4
         with:
           export_to_environment: true
           secrets: |-

diff --git a/.github/workflows/smoke-tests-schedule.yml b/.github/workflows/smoke-tests-schedule.yml
@@ -20,7 +20,7 @@ jobs:
       - uses: actions/checkout@v4
       - id: generate
         name: Generate matrix
-        uses: elastic/apm-pipeline-library/.github/actions/elastic-stack-snapshot-branches@current
+        uses: elastic/oblt-actions/elastic/active-branches@v1
         with:
           exclude-branches: '7.17'
 
@@ -57,9 +57,9 @@ jobs:
       - smoke-tests-ess
     steps:
       - id: check
-        uses: elastic/apm-pipeline-library/.github/actions/check-dependent-jobs@current
+        uses: elastic/oblt-actions/check-dependent-jobs@v1
         with:
-          needs: ${{ toJSON(needs) }}
+          jobs: ${{ toJSON(needs) }}
       - uses: elastic/oblt-actions/slack/[email protected]
         with:
           bot-token: ${{ secrets.SLACK_BOT_TOKEN }}

diff --git a/.github/workflows/test-reporter.yml b/.github/workflows/test-reporter.yml
@@ -16,9 +16,9 @@ jobs:
   system-test-results:
     runs-on: ubuntu-latest
     steps:
-      - uses: elastic/apm-pipeline-library/.github/actions/test-report@current
+      - uses: elastic/oblt-actions/test-report@v1
         with:
-          artifact: test-results            # artifact name
-          name: System Tests Results        # Name of the check run which will be created
+          artifact: /test-results-(.*)/     # artifact name pattern
+          name: 'Test Results $1'        # Name of the check run which will be created
           path: "*.xml"                     # Path to test results (inside artifact .zip)
           reporter: java-junit              # Format of test results
diff --git a/.github/workflows/update-beats.yml b/.github/workflows/update-beats.yml
@@ -16,7 +16,7 @@ jobs:
       matrix: ${{ steps.generator.outputs.matrix }}
     steps:
       - id: generator
-        uses: elastic/apm-pipeline-library/.github/actions/elastic-stack-snapshot-branches@current
+        uses: elastic/oblt-actions/elastic/active-branches@v1
   bump:
     needs:
       - filter
@@ -29,19 +29,31 @@ jobs:
         with:
           ref: ${{ matrix.branch }}
 
+      - name: Get token
+        id: get_token
+        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
+        with:
+          app_id: ${{ secrets.OBS_AUTOMATION_APP_ID }}
+          private_key: ${{ secrets.OBS_AUTOMATION_APP_PEM }}
+          permissions: >-
+            {
+              "contents": "write",
+              "pull_requests": "write"
+            }
+
       - uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
 
-      - uses: elastic/oblt-actions/updatecli/run@v1.9.1
+      - uses: elastic/oblt-actions/updatecli/run@v1
         with:
-          command: --experimental apply --config .ci/update-beats.yml
+          command: --experimental apply --config .ci/updatecli/update-beats.yml --values .ci/updatecli/values.d/scm.yml
         env:
           BRANCH_NAME: ${{ matrix.branch }}
-          GITHUB_TOKEN: ${{ secrets.UPDATECLI_GH_TOKEN }}
+          GITHUB_TOKEN: ${{ steps.get_token.outputs.token }}
 
       - if: ${{ failure()  }}
-        uses: elastic/oblt-actions/slack/send@v1.9.1
+        uses: elastic/oblt-actions/slack/send@v1
         with:
           channel-id: '#apm-server'
           message: ":traffic_cone: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, @robots-ci please look what's going on <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|here>"