diff --git a/go.mod b/go.mod index e6371c035a..60826c0f59 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/minio/minio-go/v7 v7.0.28 github.com/percona-platform/dbaas-api v0.0.0-20220110092915-5aacd784d472 github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16 - github.com/percona/pmm v0.0.0-20220613185940-593b9a167d9f + github.com/percona/pmm v0.0.0-20220616055438-f0ab9605caf4 github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 diff --git a/go.sum b/go.sum index c9421beb07..48b3e854d8 100644 --- a/go.sum +++ b/go.sum @@ -466,8 +466,8 @@ github.com/percona-platform/dbaas-api v0.0.0-20220110092915-5aacd784d472 h1:Henk github.com/percona-platform/dbaas-api v0.0.0-20220110092915-5aacd784d472/go.mod h1:WZZ3Hi+lAWCaGWmsrfkkvRQPkIa8n1OZ0s8Su+vbgus= github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16 h1:0fx16uGtl4MwrBwm9/VSoNEhjL0cXYxS0quEhLthGcc= github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16/go.mod h1:gFUwaFp6Ugu5qsBwiOVJYbDlzgZ77tmXdXGO7tG5xVI= -github.com/percona/pmm v0.0.0-20220613185940-593b9a167d9f h1:zoseK4ixNYbRYKObv6/u8m3gp5C0Ti6qg9eL2CuQTug= -github.com/percona/pmm v0.0.0-20220613185940-593b9a167d9f/go.mod h1:c22C8QvyFlcxr61TbqPlLGVC2u4xDptqYuYAoV0Umbs= +github.com/percona/pmm v0.0.0-20220616055438-f0ab9605caf4 h1:8A01OaWEfROUBPmptu/xuS0N2s4/ZHKq0mIgbhU+R6c= +github.com/percona/pmm v0.0.0-20220616055438-f0ab9605caf4/go.mod h1:u0BhxYrre/70xLykGPY10qwKj2tOghRSik343E8sHyQ= github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54 h1:aI1emmycDTGWKsBdxFPKZqohfBbK4y2ta9G4+RX7gVg= github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54/go.mod h1:Y2uXi5QNk71+ceJHuI9poank+0S1kjxd3K105fXKVkg= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= diff --git a/main.go b/main.go index 0d222259f8..9f13408efc 100644 --- a/main.go +++ b/main.go @@ -95,6 +95,7 @@ import ( "github.com/percona/pmm-managed/utils/clean" "github.com/percona/pmm-managed/utils/interceptors" "github.com/percona/pmm-managed/utils/logger" + "github.com/percona/pmm-managed/utils/pprof" ) const ( @@ -107,14 +108,33 @@ const ( cleanInterval = 10 * time.Minute cleanOlderThan = 30 * time.Minute + + defaultContextTimeout = 10 * time.Second + pProfProfileDuration = 30 * time.Second + pProfTraceDuration = 10 * time.Second ) func addLogsHandler(mux *http.ServeMux, logs *supervisord.Logs) { l := logrus.WithField("component", "logs.zip") mux.HandleFunc("/logs.zip", func(rw http.ResponseWriter, req *http.Request) { + contextTimeout := defaultContextTimeout + // increase context timeout if pprof query parameter exist in request + pprofQueryParameter, err := strconv.ParseBool(req.FormValue("pprof")) + if err != nil { + l.Debug("Unable to read 'pprof' query param. Using default: pprof=false") + } + var pprofConfig *pprof.Config + if pprofQueryParameter { + contextTimeout += pProfProfileDuration + pProfTraceDuration + pprofConfig = &pprof.Config{ + ProfileDuration: pProfProfileDuration, + TraceDuration: pProfTraceDuration, + } + } + // fail-safe - ctx, cancel := context.WithTimeout(req.Context(), 10*time.Second) + ctx, cancel := context.WithTimeout(req.Context(), contextTimeout) defer cancel() filename := fmt.Sprintf("pmm-server_%s.zip", time.Now().UTC().Format("2006-01-02_15-04")) @@ -123,7 +143,7 @@ func addLogsHandler(mux *http.ServeMux, logs *supervisord.Logs) { rw.Header().Set(`Content-Disposition`, `attachment; filename="`+filename+`"`) ctx = logger.Set(ctx, "logs") - if err := logs.Zip(ctx, rw); err != nil { + if err := logs.Zip(ctx, rw, pprofConfig); err != nil { l.Errorf("%+v", err) } }) diff --git a/services/config/config.go b/services/config/config.go index eda8a0246d..023781327e 100644 --- a/services/config/config.go +++ b/services/config/config.go @@ -94,7 +94,6 @@ func (s *Service) Load() error { if err := cfg.Services.Telemetry.Init(s.l); err != nil { return err } - s.Config = cfg return nil diff --git a/services/supervisord/logs.go b/services/supervisord/logs.go index c03f0c4d92..eeff8300c7 100644 --- a/services/supervisord/logs.go +++ b/services/supervisord/logs.go @@ -31,6 +31,7 @@ import ( "os/exec" "path/filepath" "sort" + "sync" "time" "github.com/percona/pmm/utils/pdeathsig" @@ -38,6 +39,7 @@ import ( "golang.org/x/sys/unix" "github.com/percona/pmm-managed/utils/logger" + pprofUtils "github.com/percona/pmm-managed/utils/pprof" ) const ( @@ -69,7 +71,7 @@ func NewLogs(pmmVersion string, pmmUpdateChecker *PMMUpdateChecker) *Logs { } // Zip creates .zip archive with all logs. -func (l *Logs) Zip(ctx context.Context, w io.Writer) error { +func (l *Logs) Zip(ctx context.Context, w io.Writer, pprofConfig *pprofUtils.Config) error { start := time.Now() log := logger.Get(ctx).WithField("component", "logs") log.WithField("d", time.Since(start).Seconds()).Info("Starting...") @@ -80,7 +82,7 @@ func (l *Logs) Zip(ctx context.Context, w io.Writer) error { zw := zip.NewWriter(w) now := time.Now().UTC() - files := l.files(ctx) + files := l.files(ctx, pprofConfig) log.WithField("d", time.Since(start).Seconds()).Infof("Collected %d files.", len(files)) for _, file := range files { @@ -127,8 +129,8 @@ func (l *Logs) Zip(ctx context.Context, w io.Writer) error { return nil } -// files reads log/config files and returns content. -func (l *Logs) files(ctx context.Context) []fileContent { +// files reads log/config/pprof files and returns content. +func (l *Logs) files(ctx context.Context, pprofConfig *pprofUtils.Config) []fileContent { files := make([]fileContent, 0, 20) // add logs @@ -214,6 +216,45 @@ func (l *Logs) files(ctx context.Context) []fileContent { Err: err, }) + // add pprof + if pprofConfig != nil { + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + traceBytes, err := pprofUtils.Trace(pprofConfig.TraceDuration) + files = append(files, fileContent{ + Name: "pprof/trace.out", + Data: traceBytes, + Err: err, + }) + }() + + wg.Add(1) + go func() { + defer wg.Done() + profileBytes, err := pprofUtils.Profile(pprofConfig.ProfileDuration) + files = append(files, fileContent{ + Name: "pprof/profile.pb.gz", + Data: profileBytes, + Err: err, + }) + }() + + wg.Add(1) + go func() { + defer wg.Done() + heapBytes, err := pprofUtils.Heap(true) + files = append(files, fileContent{ + Name: "pprof/heap.pb.gz", + Data: heapBytes, + Err: err, + }) + }() + + wg.Wait() + } + sort.Slice(files, func(i, j int) bool { return files[i].Name < files[j].Name }) return files } diff --git a/services/supervisord/logs_test.go b/services/supervisord/logs_test.go index e5b2a2207b..cac791bac8 100644 --- a/services/supervisord/logs_test.go +++ b/services/supervisord/logs_test.go @@ -124,7 +124,7 @@ func TestFiles(t *testing.T) { l := NewLogs("2.4.5", checker) ctx := logger.Set(context.Background(), t.Name()) - files := l.files(ctx) + files := l.files(ctx, nil) actual := make([]string, 0, len(files)) for _, f := range files { // present only after update @@ -157,7 +157,7 @@ func TestZip(t *testing.T) { ctx := logger.Set(context.Background(), t.Name()) var buf bytes.Buffer - require.NoError(t, l.Zip(ctx, &buf)) + require.NoError(t, l.Zip(ctx, &buf, nil)) reader := bytes.NewReader(buf.Bytes()) r, err := zip.NewReader(reader, reader.Size()) require.NoError(t, err) diff --git a/utils/pprof/pprof.go b/utils/pprof/pprof.go new file mode 100644 index 0000000000..de1486daaf --- /dev/null +++ b/utils/pprof/pprof.go @@ -0,0 +1,79 @@ +// pmm-managed +// Copyright (C) 2017 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package pprof + +import ( + "bytes" + "fmt" + "runtime" + "runtime/pprof" + "runtime/trace" + "time" +) + +// Profile responds with the pprof-formatted cpu profile. +// Profiling lasts for duration specified in seconds. +func Profile(duration time.Duration) ([]byte, error) { + var profileBuf bytes.Buffer + if err := pprof.StartCPUProfile(&profileBuf); err != nil { + return nil, err + } + + time.Sleep(duration) + pprof.StopCPUProfile() + + return profileBuf.Bytes(), nil +} + +// Trace responds with the execution trace in binary form. +// Tracing lasts for duration specified in seconds. +func Trace(duration time.Duration) ([]byte, error) { + var traceBuf bytes.Buffer + if err := trace.Start(&traceBuf); err != nil { + return nil, err + } + + time.Sleep(duration) + trace.Stop() + + return traceBuf.Bytes(), nil +} + +// Heap responds with the pprof-formatted profile named "heap". +// listing the available profiles. +// You can specify the gc parameter to run gc before taking the heap sample. +func Heap(gc bool) ([]byte, error) { + var heapBuf bytes.Buffer + debug := 0 + profile := "heap" + + p := pprof.Lookup(profile) + if p == nil { + return nil, fmt.Errorf("profile cannot be found: %s", profile) + } + + if gc { + runtime.GC() + } + + err := p.WriteTo(&heapBuf, debug) + if err != nil { + return nil, err + } + + return heapBuf.Bytes(), nil +} diff --git a/utils/pprof/pprof_config.go b/utils/pprof/pprof_config.go new file mode 100644 index 0000000000..03709795ed --- /dev/null +++ b/utils/pprof/pprof_config.go @@ -0,0 +1,31 @@ +// pmm-managed +// Copyright (C) 2017 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package pprof + +import ( + "time" +) + +// Config pprof settings. +type Config struct { + ProfileDuration time.Duration `yaml:"profile_duration"` //nolint:tagliatelle + TraceDuration time.Duration `yaml:"trace_duration"` //nolint:tagliatelle +} + +// Init pprof config init. +func (c *Config) Init() { +} diff --git a/utils/pprof/pprof_test.go b/utils/pprof/pprof_test.go new file mode 100644 index 0000000000..5b037dccab --- /dev/null +++ b/utils/pprof/pprof_test.go @@ -0,0 +1,72 @@ +// pmm-managed +// Copyright (C) 2017 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package pprof + +import ( + "bytes" + "compress/gzip" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestHeap(t *testing.T) { + t.Parallel() + t.Run("Heap test", func(t *testing.T) { + heapBytes, err := Heap(true) + + // read gzip + reader, err := gzip.NewReader(bytes.NewBuffer(heapBytes)) + assert.NoError(t, err) + + var resB bytes.Buffer + _, err = resB.ReadFrom(reader) + assert.NoError(t, err) + assert.NotEmpty(t, resB.Bytes()) + }) +} + +func TestProfile(t *testing.T) { + t.Parallel() + t.Run("Profile test", func(t *testing.T) { + profileBytes, err := Profile(1 * time.Second) + + assert.NoError(t, err) + assert.NotEmpty(t, profileBytes) + + // read gzip + reader, err := gzip.NewReader(bytes.NewBuffer(profileBytes)) + assert.NoError(t, err) + + var resB bytes.Buffer + _, err = resB.ReadFrom(reader) + assert.NoError(t, err) + + assert.NotEmpty(t, resB.Bytes()) + }) +} + +func TestTrace(t *testing.T) { + t.Parallel() + t.Run("Trace test", func(t *testing.T) { + traceBytes, err := Trace(1 * time.Second) + + assert.NoError(t, err) + assert.NotEmpty(t, traceBytes) + }) +}