Merge branch 'main' into enable-self-instrumentation-tracing

.github/workflows/add-to-docs-project.yml

@@ -11,6 +11,17 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event.label.name == 'Team:Docs'
     steps:
+      - name: Get token
+        id: get_token
+        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
+        with:
+          app_id: ${{ secrets.OBS_AUTOMATION_APP_ID }}
+          private_key: ${{ secrets.OBS_AUTOMATION_APP_PEM }}
+          permissions: >-
+            {
+              "organization_projects": "write",
+              "issues": "read"
+            }
       - uses: octokit/[email protected]
         id: add_to_project
         with:
@@ -28,4 +39,4 @@ jobs:
           contentid: ${{ github.event.issue.node_id }}
         env:
           PROJECT_ID: "PVT_kwDOAGc3Zs0iZw"
-          GITHUB_TOKEN: ${{ secrets.APM_TECH_USER_TOKEN }}
+          GITHUB_TOKEN: ${{ steps.get_token.outputs.token }}

.github/workflows/add-to-project.yml

@@ -14,7 +14,18 @@ jobs:
     name: Add issue to project
     runs-on: ubuntu-latest
     steps:
+      - name: Get token
+        id: get_token
+        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
+        with:
+          app_id: ${{ secrets.OBS_AUTOMATION_APP_ID }}
+          private_key: ${{ secrets.OBS_AUTOMATION_APP_PEM }}
+          permissions: >-
+            {
+              "organization_projects": "write",
+              "issues": "read"
+            }
       - uses: actions/[email protected]
         with:
           project-url: https://github.com/orgs/elastic/projects/1286
-          github-token: ${{ secrets.APM_TECH_USER_TOKEN }}
+          github-token: ${{ steps.get_token.outputs.token }}

.github/workflows/ci.yml

@@ -123,7 +123,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Get changed files
         id: changed-files
-        uses: tj-actions/changed-files@48d8f15b2aaa3d255ca5af3eba4870f807ce6b3c # v45.0.2
+        uses: tj-actions/changed-files@c3a1bb2c992d77180ae65be6ae6c166cf40f857c # v45.0.3
         with:
           files: .go-version
 

.go-version

@@ -1 +1 @@
-1.23.1
+1.23.2

NOTICE.txt

@@ -573,11 +573,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/beats/v7
-Version: v7.0.0-alpha2.0.20240924123012-ecc521545d94
+Version: v7.0.0-alpha2.0.20240823193033-af33fad4d50b
 Licence type (autodetected): Elastic
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20240924123012-ecc521545d94/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/[email protected].20240823193033-af33fad4d50b/LICENSE.txt:
 
 Source code in this repository is variously licensed under the Apache License
 Version 2.0, an Apache compatible license, or the Elastic License. Outside of
@@ -2379,12 +2379,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 --------------------------------------------------------------------------------
-Dependency : github.com/gofrs/uuid/v5
-Version: v5.3.0
+Dependency : github.com/gofrs/uuid
+Version: v4.4.0+incompatible
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/gofrs/uuid/[email protected]/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/gofrs/uuid@v4.4.0+incompatible/LICENSE:
 
 Copyright (C) 2013-2018 by Maxim Bublis <[email protected]>
 
@@ -6609,11 +6609,11 @@ Contents of probable licence file $GOMODCACHE/google.golang.org/[email protected]/LIC
 
 --------------------------------------------------------------------------------
 Dependency : google.golang.org/protobuf
-Version: v1.34.2
+Version: v1.35.1
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.34.2/LICENSE:
+Contents of probable licence file $GOMODCACHE/google.golang.org/protobuf@v1.35.1/LICENSE:
 
 Copyright (c) 2018 The Go Authors. All rights reserved.
 
@@ -10597,6 +10597,36 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/gofrs/uuid/v5
+Version: v5.2.0
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/gofrs/uuid/[email protected]/LICENSE:
+
+Copyright (C) 2013-2018 by Maxim Bublis <[email protected]>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/gogo/googleapis
 Version: v1.4.1

cmd/apm-server/main.go

@@ -25,7 +25,6 @@ import (
 )
 
 func main() {
-	beatcmd.InitBeatCmd()
 	rootCmd := beatcmd.NewRootCommand(beatcmd.BeatParams{
 		NewRunner: func(args beatcmd.RunnerParams) (beatcmd.Runner, error) {
 			return beater.NewRunner(beater.RunnerParams{

docker-compose.yml

@@ -11,7 +11,7 @@ x-logging: &default-logging
 services:
   elasticsearch:
     # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-    image: docker.elastic.co/elasticsearch/elasticsearch:9.0.0-201d2251-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.0.0-3b9dafe0-SNAPSHOT
     ports:
       - 9200:9200
     healthcheck:
@@ -43,7 +43,7 @@ services:
 
   kibana:
     # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-    image: docker.elastic.co/kibana/kibana:9.0.0-201d2251-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:9.0.0-3b9dafe0-SNAPSHOT
     ports:
       - 5601:5601
     healthcheck:
@@ -63,7 +63,7 @@ services:
 
   metricbeat:
     # TODO: replace with a pinned version such as 9.0.0-aaaaaaaa-SNAPSHOT
-    image: docker.elastic.co/beats/metricbeat:9.0.0-201d2251-SNAPSHOT
+    image: docker.elastic.co/beats/metricbeat:9.0.0-3b9dafe0-SNAPSHOT
     environment:
       ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
       ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}"

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/dustin/go-humanize v1.0.1
 	github.com/elastic/apm-aggregation v1.1.0
 	github.com/elastic/apm-data v1.13.1
-	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240924123012-ecc521545d94
+	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240823193033-af33fad4d50b
 	github.com/elastic/elastic-agent-client/v7 v7.16.0
 	github.com/elastic/elastic-agent-libs v0.12.0
 	github.com/elastic/elastic-agent-system-metrics v0.11.3
@@ -19,7 +19,7 @@ require (
 	github.com/elastic/go-ucfg v0.8.8
 	github.com/go-sourcemap/sourcemap v2.1.4+incompatible
 	github.com/gofrs/flock v0.12.1
-	github.com/gofrs/uuid/v5 v5.3.0
+	github.com/gofrs/uuid v4.4.0+incompatible
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/go-cmp v0.6.0
 	github.com/gorilla/mux v1.8.1
@@ -53,7 +53,7 @@ require (
 	golang.org/x/term v0.25.0
 	golang.org/x/time v0.7.0
 	google.golang.org/grpc v1.67.1
-	google.golang.org/protobuf v1.34.2
+	google.golang.org/protobuf v1.35.1
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -98,6 +98,7 @@ require (
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/gofrs/uuid/v5 v5.2.0 // indirect
 	github.com/gogo/googleapis v1.4.1 // indirect
 	github.com/golang/glog v1.2.2 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect

go.sum

@@ -133,10 +133,10 @@ github.com/elastic/apm-aggregation v1.1.0 h1:eTHGd5w99JoRFJ763MJP6nUX4hkkeSil3KW
 github.com/elastic/apm-aggregation v1.1.0/go.mod h1:YBQ77Jt7vOy2/7w4q5SsEifhc0V02dJxRcyyeC6HsvQ=
 github.com/elastic/apm-data v1.13.1 h1:vQUWqi9H6QC4FCZ05kILkE+2BeKWw6iT+NXXyadWJm8=
 github.com/elastic/apm-data v1.13.1/go.mod h1:m2FSH7alN07VmtmN10KmgxAa8tgXlJDe/rC0g4FnZUI=
-github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240924123012-ecc521545d94 h1:nXkblLNf1SnW8QIUHsIJgHEFxiP4QhPqBTyC6lDks6A=
-github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240924123012-ecc521545d94/go.mod h1:HTRccLDjsAVZ12slH942u9UyEzRs35K9v+gyZS3KBA8=
-github.com/elastic/elastic-agent-autodiscover v0.8.2 h1:Fs2FhR33AMBPfm5/jz4drVzaEZaqOIHlDBvGtkUZdIk=
-github.com/elastic/elastic-agent-autodiscover v0.8.2/go.mod h1:VZnU53EVaFTxR8Xf6YsLN8FHD5DKQzHSPlKax9/4w+o=
+github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240823193033-af33fad4d50b h1:i3WdOw//Vg76Vseqxgn47GPTVqUSucAdE42jkih1ViM=
+github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240823193033-af33fad4d50b/go.mod h1:m8d3EBr2z7YvWfxRgoE7SsUNbMTpyWXi67fG7B7vn7w=
+github.com/elastic/elastic-agent-autodiscover v0.8.1 h1:u6TWqh7wfevu6S4GUq4SIxYBRo4b/P5RZmx/rSvT10A=
+github.com/elastic/elastic-agent-autodiscover v0.8.1/go.mod h1:0gzGsaDCAqBfUZjuCqqWsSI60eaZ778A5tQZV72rPV0=
 github.com/elastic/elastic-agent-client/v7 v7.16.0 h1:yKGq2+CxAuW8Kh0EoNl202tqAyQKfBcPRawVKs2Jve0=
 github.com/elastic/elastic-agent-client/v7 v7.16.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI=
 github.com/elastic/elastic-agent-libs v0.12.0 h1:xfVVCYIaI6XEPVpJNq7HQav7O/VxLj+YbQK/poWb7wA=
@@ -200,8 +200,10 @@ github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TC
 github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
 github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E=
 github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0=
-github.com/gofrs/uuid/v5 v5.3.0 h1:m0mUMr+oVYUdxpMLgSYCZiXe7PuVPnI94+OMeVBNedk=
-github.com/gofrs/uuid/v5 v5.3.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8=
+github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
+github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gofrs/uuid/v5 v5.2.0 h1:qw1GMx6/y8vhVsx626ImfKMuS5CvJmhIKKtuyvfajMM=
+github.com/gofrs/uuid/v5 v5.2.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8=
 github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0=
 github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -583,8 +585,8 @@ google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E=
 google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA=
 google.golang.org/grpc/examples v0.0.0-20231016154744-cb430bed4d27 h1:EB/3dtnYKOItaNPpOI/HmOCGbVZUiXcstRfiuxN+cFg=
 google.golang.org/grpc/examples v0.0.0-20231016154744-cb430bed4d27/go.mod h1:Crtq1t+mykyL5d6PR3z8zCxKx/Qjq/mlPWDPoWJANYA=
-google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
-google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
+google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
+google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/beatcmd/beat.go

@@ -31,7 +31,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/gofrs/uuid/v5"
+	"github.com/gofrs/uuid"
 	"go.uber.org/zap"
 	"golang.org/x/sync/errgroup"
 
@@ -123,7 +123,6 @@ func NewBeat(args BeatParams) (*Beat, error) {
 			Keystore:   keystore,
 			Config:     &beat.BeatConfig{Output: cfg.Output},
 			BeatConfig: cfg.APMServer,
-			Registry:   reload.NewRegistry(),
 		},
 		Config:    cfg,
 		newRunner: args.NewRunner,
@@ -153,7 +152,7 @@ func (b *Beat) init() error {
 	logp.Info("Beat ID: %v", b.Info.ID)
 
 	// Initialize central config manager.
-	manager, err := management.NewManager(b.Config.Management, b.Registry)
+	manager, err := management.NewManager(b.Config.Management, reload.RegisterV2)
 	if err != nil {
 		return err
 	}
@@ -367,7 +366,7 @@ func (b *Beat) Run(ctx context.Context) error {
 	}
 
 	if b.Manager.Enabled() {
-		reloader, err := NewReloader(b.Info, b.Registry, b.newRunner)
+		reloader, err := NewReloader(b.Info, b.newRunner)
 		if err != nil {
 			return err
 		}

internal/beatcmd/beat_test.go

@@ -125,6 +125,10 @@ func TestUnmanagedOutputRequired(t *testing.T) {
 }
 
 func TestRunManager(t *testing.T) {
+	oldRegistry := reload.RegisterV2
+	defer func() { reload.RegisterV2 = oldRegistry }()
+	reload.RegisterV2 = reload.NewRegistry()
+
 	// Register our own mock management implementation.
 	manager := newMockManager()
 	management.SetManagerFactory(func(cfg *config.C, registry *reload.Registry) (management.Manager, error) {
@@ -147,20 +151,20 @@ func TestRunManager(t *testing.T) {
 	expectEvent(t, manager.started, "manager should have been started")
 	expectNoEvent(t, manager.stopped, "manager should not have been stopped")
 
-	err := b.Registry.GetInputList().Reload([]*reload.ConfigWithMeta{{
+	err := reload.RegisterV2.GetInputList().Reload([]*reload.ConfigWithMeta{{
 		Config: config.MustNewConfigFrom(`{
 			"revision": 1,
 			"apm-server.host": "localhost:1234"
 		}`),
 	}})
 	assert.NoError(t, err)
-	err = b.Registry.GetReloadableOutput().Reload(&reload.ConfigWithMeta{
+	err = reload.RegisterV2.GetReloadableOutput().Reload(&reload.ConfigWithMeta{
 		Config: config.MustNewConfigFrom(`{"console.enabled": true}`),
 	})
 	assert.NoError(t, err)
 
 	expectRunnerParams(t, calls)
-	err = b.Registry.GetReloadableAPM().Reload(&reload.ConfigWithMeta{
+	err = reload.RegisterV2.GetReloadableAPM().Reload(&reload.ConfigWithMeta{
 		Config: config.MustNewConfigFrom(`{"elastic.enabled": true, "elastic.environment": "testenv"}`),
 	})
 	assert.NoError(t, err)
@@ -232,6 +236,9 @@ func resetGlobals() {
 			registry.Clear()
 		}
 	}
+
+	// Create a new reload registry, as the Beat.Run method will register with it.
+	reload.RegisterV2 = reload.NewRegistry()
 }
 
 type runnerFunc func(ctx context.Context) error

internal/beatcmd/init.go

@@ -19,7 +19,6 @@ package beatcmd
 
 import (
 	cryptorand "crypto/rand"
-	"flag"
 	"log"
 	"math"
 	"math/big"
@@ -39,7 +38,7 @@ import (
 	_ "github.com/elastic/beats/v7/libbeat/publisher/queue/memqueue"
 )
 
-func InitBeatCmd() {
+func init() {
 	initRand()
 	initFlags()
 }
@@ -63,7 +62,6 @@ func initFlags() {
 		}
 	}
 
-	flag.Parse()
 	if err := cfgfile.HandleFlags(); err != nil {
 		log.Fatal(err)
 	}

internal/beatcmd/reloader.go

@@ -58,20 +58,20 @@ type Runner interface {
 
 // NewReloader returns a new Reloader which creates Runners using the provided
 // beat.Info and NewRunnerFunc.
-func NewReloader(info beat.Info, registry *reload.Registry, newRunner NewRunnerFunc) (*Reloader, error) {
+func NewReloader(info beat.Info, newRunner NewRunnerFunc) (*Reloader, error) {
 	r := &Reloader{
 		info:      info,
 		logger:    logp.NewLogger(""),
 		newRunner: newRunner,
 		stopped:   make(chan struct{}),
 	}
-	if err := registry.RegisterList(reload.InputRegName, reloadableListFunc(r.reloadInputs)); err != nil {
+	if err := reload.RegisterV2.RegisterList(reload.InputRegName, reloadableListFunc(r.reloadInputs)); err != nil {
 		return nil, fmt.Errorf("failed to register inputs reloader: %w", err)
 	}
-	if err := registry.Register(reload.OutputRegName, reload.ReloadableFunc(r.reloadOutput)); err != nil {
+	if err := reload.RegisterV2.Register(reload.OutputRegName, reload.ReloadableFunc(r.reloadOutput)); err != nil {
 		return nil, fmt.Errorf("failed to register output reloader: %w", err)
 	}
-	if err := registry.Register(reload.APMRegName, reload.ReloadableFunc(r.reloadAPMTracing)); err != nil {
+	if err := reload.RegisterV2.Register(reload.APMRegName, reload.ReloadableFunc(r.reloadAPMTracing)); err != nil {
 		return nil, fmt.Errorf("failed to register apm tracing reloader: %w", err)
 	}
 	return r, nil