From f4026a1dcc2c1f31e6e672160ed735ec4676a048 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 9 Dec 2024 22:15:41 +0100 Subject: [PATCH 1/3] feat: auto set GOMEMLIMIT when memory is limited use automemlimit lib to set memory limit run check every 30s --- go.mod | 3 +++ go.sum | 6 ++++++ internal/beatcmd/beat.go | 7 +++++++ internal/beatcmd/memlimit.go | 20 ++++++++++++++++++++ 4 files changed, 36 insertions(+) create mode 100644 internal/beatcmd/memlimit.go diff --git a/go.mod b/go.mod index 60817deeeb9..215bf0528dc 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/elastic/apm-server go 1.23.0 require ( + github.com/KimMachineGun/automemlimit v0.7.0-pre.3 github.com/cespare/xxhash/v2 v2.3.0 github.com/dgraph-io/badger/v2 v2.2007.4 github.com/dustin/go-humanize v1.0.1 @@ -46,6 +47,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.32.0 go.uber.org/automaxprocs v1.6.0 go.uber.org/zap v1.27.0 + go.uber.org/zap/exp v0.3.0 golang.org/x/net v0.32.0 golang.org/x/sync v0.10.0 golang.org/x/term v0.27.0 @@ -122,6 +124,7 @@ require ( github.com/mitchellh/hashstructure v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect diff --git a/go.sum b/go.sum index 15c6db3cba3..e037489ce05 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,8 @@ github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/IBM/sarama v1.43.3 h1:Yj6L2IaNvb2mRBop39N7mmJAHBVY3dTPncr3qGVkxPA= github.com/IBM/sarama v1.43.3/go.mod h1:FVIRaLrhK3Cla/9FfRF5X9Zua2KpS3SYIXxhac1H+FQ= +github.com/KimMachineGun/automemlimit v0.7.0-pre.3 h1:aZVmBE7SmIsRoVHIzpgCJ6rvwnKRnguDZv5pAZWcZyQ= +github.com/KimMachineGun/automemlimit v0.7.0-pre.3/go.mod h1:QZxpHaGOQoYvFhv/r4u3U0JTC2ZcOwbSr11UZF46UBM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -332,6 +334,8 @@ github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2sz github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -476,6 +480,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= +go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/internal/beatcmd/beat.go b/internal/beatcmd/beat.go index 8a8b38d0f79..bd9fe63f731 100644 --- a/internal/beatcmd/beat.go +++ b/internal/beatcmd/beat.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "io" + "log/slog" "os" "os/user" "runtime" @@ -33,6 +34,7 @@ import ( "github.com/gofrs/uuid/v5" "go.uber.org/zap" + "go.uber.org/zap/exp/zapslog" "golang.org/x/sync/errgroup" "github.com/elastic/beats/v7/libbeat/api" @@ -348,6 +350,11 @@ func (b *Beat) Run(ctx context.Context) error { return adjustMaxProcs(ctx, 30*time.Second, logger) }) + slogger := slog.New(zapslog.NewHandler(logger.Core())) + if err := adjustMemlimit(1*time.Second, slogger); err != nil { + return err + } + logSystemInfo(b.Info) cleanup, err := b.registerElasticsearchVersionCheck() diff --git a/internal/beatcmd/memlimit.go b/internal/beatcmd/memlimit.go new file mode 100644 index 00000000000..fcc05b60045 --- /dev/null +++ b/internal/beatcmd/memlimit.go @@ -0,0 +1,20 @@ +package beatcmd + +import ( + "fmt" + "log/slog" + "time" + + "github.com/KimMachineGun/automemlimit/memlimit" +) + +func adjustMemlimit(d time.Duration, logger *slog.Logger) error { + if _, err := memlimit.SetGoMemLimitWithOpts( + memlimit.WithLogger(logger), + memlimit.WithRefreshInterval(d), + memlimit.WithRatio(0.9), + ); err != nil { + return fmt.Errorf("failed to set go memlimit: %w", err) + } + return nil +} From f0b7386de01204eb5580db2356ddf3c4eb3ff983 Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Mon, 9 Dec 2024 22:28:49 +0100 Subject: [PATCH 2/3] lint: run make update --- NOTICE.txt | 99 ++++++++++++++++++++++++++++++++++++ internal/beatcmd/memlimit.go | 17 +++++++ 2 files changed, 116 insertions(+) diff --git a/NOTICE.txt b/NOTICE.txt index 1ee72cbb9c5..d10d1f0fc21 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -9,6 +9,37 @@ Third party libraries used by the Elastic APM Server project: ================================================================================ +-------------------------------------------------------------------------------- +Dependency : github.com/KimMachineGun/automemlimit +Version: v0.7.0-pre.3 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/!kim!machine!gun/automemlimit@v0.7.0-pre.3/LICENSE: + +MIT License + +Copyright (c) 2022 Geon Kim + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/cespare/xxhash/v2 Version: v2.3.0 @@ -5644,6 +5675,35 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : go.uber.org/zap/exp +Version: v0.3.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/go.uber.org/zap/exp@v0.3.0/LICENSE: + +Copyright (c) 2016-2024 Uber Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : golang.org/x/net Version: v0.32.0 @@ -13367,6 +13427,45 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : github.com/pbnjay/memory +Version: v0.0.0-20210728143218-7b4eea64cf58 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/pbnjay/memory@v0.0.0-20210728143218-7b4eea64cf58/LICENSE: + +BSD 3-Clause License + +Copyright (c) 2017, Jeremy Jay +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : github.com/pierrec/lz4/v4 Version: v4.1.21 diff --git a/internal/beatcmd/memlimit.go b/internal/beatcmd/memlimit.go index fcc05b60045..0ea89c0d817 100644 --- a/internal/beatcmd/memlimit.go +++ b/internal/beatcmd/memlimit.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package beatcmd import ( From a034fe3d32be507102605607d33360dbd9b1bc3a Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Thu, 12 Dec 2024 12:54:01 +0100 Subject: [PATCH 3/3] feat: fallback to system if cgroup is not available --- internal/beatcmd/memlimit.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/beatcmd/memlimit.go b/internal/beatcmd/memlimit.go index 0ea89c0d817..d813fca4956 100644 --- a/internal/beatcmd/memlimit.go +++ b/internal/beatcmd/memlimit.go @@ -27,6 +27,12 @@ import ( func adjustMemlimit(d time.Duration, logger *slog.Logger) error { if _, err := memlimit.SetGoMemLimitWithOpts( + memlimit.WithProvider( + memlimit.ApplyFallback( + memlimit.FromCgroup, + memlimit.FromSystem, + ), + ), memlimit.WithLogger(logger), memlimit.WithRefreshInterval(d), memlimit.WithRatio(0.9),