PMM-5492 Add Pprof data to logs.zip

go.mod

@@ -37,7 +37,7 @@ require (
 	github.com/minio/minio-go/v7 v7.0.28
 	github.com/percona-platform/dbaas-api v0.0.0-20220110092915-5aacd784d472
 	github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16
-	github.com/percona/pmm v0.0.0-20220613185940-593b9a167d9f
+	github.com/percona/pmm v0.0.0-20220616055438-f0ab9605caf4
 	github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.0

go.sum

@@ -466,8 +466,8 @@ github.com/percona-platform/dbaas-api v0.0.0-20220110092915-5aacd784d472 h1:Henk
 github.com/percona-platform/dbaas-api v0.0.0-20220110092915-5aacd784d472/go.mod h1:WZZ3Hi+lAWCaGWmsrfkkvRQPkIa8n1OZ0s8Su+vbgus=
 github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16 h1:0fx16uGtl4MwrBwm9/VSoNEhjL0cXYxS0quEhLthGcc=
 github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16/go.mod h1:gFUwaFp6Ugu5qsBwiOVJYbDlzgZ77tmXdXGO7tG5xVI=
-github.com/percona/pmm v0.0.0-20220613185940-593b9a167d9f h1:zoseK4ixNYbRYKObv6/u8m3gp5C0Ti6qg9eL2CuQTug=
-github.com/percona/pmm v0.0.0-20220613185940-593b9a167d9f/go.mod h1:c22C8QvyFlcxr61TbqPlLGVC2u4xDptqYuYAoV0Umbs=
+github.com/percona/pmm v0.0.0-20220616055438-f0ab9605caf4 h1:8A01OaWEfROUBPmptu/xuS0N2s4/ZHKq0mIgbhU+R6c=
+github.com/percona/pmm v0.0.0-20220616055438-f0ab9605caf4/go.mod h1:u0BhxYrre/70xLykGPY10qwKj2tOghRSik343E8sHyQ=
 github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54 h1:aI1emmycDTGWKsBdxFPKZqohfBbK4y2ta9G4+RX7gVg=
 github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54/go.mod h1:Y2uXi5QNk71+ceJHuI9poank+0S1kjxd3K105fXKVkg=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=

main.go

@@ -95,6 +95,7 @@ import (
 	"github.com/percona/pmm-managed/utils/clean"
 	"github.com/percona/pmm-managed/utils/interceptors"
 	"github.com/percona/pmm-managed/utils/logger"
+	"github.com/percona/pmm-managed/utils/pprof"
 )
 
 const (
@@ -107,14 +108,33 @@ const (
 
 	cleanInterval  = 10 * time.Minute
 	cleanOlderThan = 30 * time.Minute
+
+	defaultContextTimeout = 10 * time.Second
+	pProfProfileDuration  = 30 * time.Second
+	pProfTraceDuration    = 10 * time.Second
 )
 
 func addLogsHandler(mux *http.ServeMux, logs *supervisord.Logs) {
 	l := logrus.WithField("component", "logs.zip")
 
 	mux.HandleFunc("/logs.zip", func(rw http.ResponseWriter, req *http.Request) {
+		contextTimeout := defaultContextTimeout
+		// increase context timeout if pprof query parameter exist in request
+		pprofQueryParameter, err := strconv.ParseBool(req.FormValue("pprof"))
+		if err != nil {
+			l.Debug("Unable to read 'pprof' query param. Using default: pprof=false")
+		}
+		var pprofConfig *pprof.Config
+		if pprofQueryParameter {
+			contextTimeout += pProfProfileDuration + pProfTraceDuration
+			pprofConfig = &pprof.Config{
+				ProfileDuration: pProfProfileDuration,
+				TraceDuration:   pProfTraceDuration,
+			}
+		}
+
 		// fail-safe
-		ctx, cancel := context.WithTimeout(req.Context(), 10*time.Second)
+		ctx, cancel := context.WithTimeout(req.Context(), contextTimeout)
 		defer cancel()
 
 		filename := fmt.Sprintf("pmm-server_%s.zip", time.Now().UTC().Format("2006-01-02_15-04"))
@@ -123,7 +143,7 @@ func addLogsHandler(mux *http.ServeMux, logs *supervisord.Logs) {
 		rw.Header().Set(`Content-Disposition`, `attachment; filename="`+filename+`"`)
 
 		ctx = logger.Set(ctx, "logs")
-		if err := logs.Zip(ctx, rw); err != nil {
+		if err := logs.Zip(ctx, rw, pprofConfig); err != nil {
 			l.Errorf("%+v", err)
 		}
 	})

services/config/config.go

@@ -94,7 +94,6 @@ func (s *Service) Load() error {
 	if err := cfg.Services.Telemetry.Init(s.l); err != nil {
 		return err
 	}
-
 	s.Config = cfg
 
 	return nil

services/supervisord/logs.go

@@ -31,13 +31,15 @@ import (
 	"os/exec"
 	"path/filepath"
 	"sort"
+	"sync"
 	"time"
 
 	"github.com/percona/pmm/utils/pdeathsig"
 	"github.com/pkg/errors"
 	"golang.org/x/sys/unix"
 
 	"github.com/percona/pmm-managed/utils/logger"
+	pprofUtils "github.com/percona/pmm-managed/utils/pprof"
 )
 
 const (
@@ -69,7 +71,7 @@ func NewLogs(pmmVersion string, pmmUpdateChecker *PMMUpdateChecker) *Logs {
 }
 
 // Zip creates .zip archive with all logs.
-func (l *Logs) Zip(ctx context.Context, w io.Writer) error {
+func (l *Logs) Zip(ctx context.Context, w io.Writer, pprofConfig *pprofUtils.Config) error {
 	start := time.Now()
 	log := logger.Get(ctx).WithField("component", "logs")
 	log.WithField("d", time.Since(start).Seconds()).Info("Starting...")
@@ -80,7 +82,7 @@ func (l *Logs) Zip(ctx context.Context, w io.Writer) error {
 	zw := zip.NewWriter(w)
 	now := time.Now().UTC()
 
-	files := l.files(ctx)
+	files := l.files(ctx, pprofConfig)
 	log.WithField("d", time.Since(start).Seconds()).Infof("Collected %d files.", len(files))
 
 	for _, file := range files {
@@ -127,8 +129,8 @@ func (l *Logs) Zip(ctx context.Context, w io.Writer) error {
 	return nil
 }
 
-// files reads log/config files and returns content.
-func (l *Logs) files(ctx context.Context) []fileContent {
+// files reads log/config/pprof files and returns content.
+func (l *Logs) files(ctx context.Context, pprofConfig *pprofUtils.Config) []fileContent {
 	files := make([]fileContent, 0, 20)
 
 	// add logs
@@ -214,6 +216,45 @@ func (l *Logs) files(ctx context.Context) []fileContent {
 		Err:  err,
 	})
 
+	// add pprof
+	if pprofConfig != nil {
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			traceBytes, err := pprofUtils.Trace(pprofConfig.TraceDuration)
+			files = append(files, fileContent{
+				Name: "pprof/trace.out",
+				Data: traceBytes,
+				Err:  err,
+			})
+		}()
+
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			profileBytes, err := pprofUtils.Profile(pprofConfig.ProfileDuration)
+			files = append(files, fileContent{
+				Name: "pprof/profile.pb.gz",
+				Data: profileBytes,
+				Err:  err,
+			})
+		}()
+
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			heapBytes, err := pprofUtils.Heap(true)
+			files = append(files, fileContent{
+				Name: "pprof/heap.pb.gz",
+				Data: heapBytes,
+				Err:  err,
+			})
+		}()
+
+		wg.Wait()
+	}
+
 	sort.Slice(files, func(i, j int) bool { return files[i].Name < files[j].Name })
 	return files
 }

services/supervisord/logs_test.go

@@ -124,7 +124,7 @@ func TestFiles(t *testing.T) {
 	l := NewLogs("2.4.5", checker)
 	ctx := logger.Set(context.Background(), t.Name())
 
-	files := l.files(ctx)
+	files := l.files(ctx, nil)
 	actual := make([]string, 0, len(files))
 	for _, f := range files {
 		// present only after update
@@ -157,7 +157,7 @@ func TestZip(t *testing.T) {
 	ctx := logger.Set(context.Background(), t.Name())
 
 	var buf bytes.Buffer
-	require.NoError(t, l.Zip(ctx, &buf))
+	require.NoError(t, l.Zip(ctx, &buf, nil))
 	reader := bytes.NewReader(buf.Bytes())
 	r, err := zip.NewReader(reader, reader.Size())
 	require.NoError(t, err)

utils/pprof/pprof.go

@@ -0,0 +1,79 @@
+// pmm-managed
+// Copyright (C) 2017 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package pprof
+
+import (
+	"bytes"
+	"fmt"
+	"runtime"
+	"runtime/pprof"
+	"runtime/trace"
+	"time"
+)
+
+// Profile responds with the pprof-formatted cpu profile.
+// Profiling lasts for duration specified in seconds.
+func Profile(duration time.Duration) ([]byte, error) {
+	var profileBuf bytes.Buffer
+	if err := pprof.StartCPUProfile(&profileBuf); err != nil {
+		return nil, err
+	}
+
+	time.Sleep(duration)
+	pprof.StopCPUProfile()
+
+	return profileBuf.Bytes(), nil
+}
+
+// Trace responds with the execution trace in binary form.
+// Tracing lasts for duration specified in seconds.
+func Trace(duration time.Duration) ([]byte, error) {
+	var traceBuf bytes.Buffer
+	if err := trace.Start(&traceBuf); err != nil {
+		return nil, err
+	}
+
+	time.Sleep(duration)
+	trace.Stop()
+
+	return traceBuf.Bytes(), nil
+}
+
+// Heap responds with the pprof-formatted profile named "heap".
+// listing the available profiles.
+// You can specify the gc parameter to run gc before taking the heap sample.
+func Heap(gc bool) ([]byte, error) {
+	var heapBuf bytes.Buffer
+	debug := 0
+	profile := "heap"
+
+	p := pprof.Lookup(profile)
+	if p == nil {
+		return nil, fmt.Errorf("profile cannot be found: %s", profile)
+	}
+
+	if gc {
+		runtime.GC()
+	}
+
+	err := p.WriteTo(&heapBuf, debug)
+	if err != nil {
+		return nil, err
+	}
+
+	return heapBuf.Bytes(), nil
+}

utils/pprof/pprof_config.go

@@ -0,0 +1,31 @@
+// pmm-managed
+// Copyright (C) 2017 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package pprof
+
+import (
+	"time"
+)
+
+// Config pprof settings.
+type Config struct {
+	ProfileDuration time.Duration `yaml:"profile_duration"` //nolint:tagliatelle
+	TraceDuration   time.Duration `yaml:"trace_duration"`   //nolint:tagliatelle
+}
+
+// Init pprof config init.
+func (c *Config) Init() {
+}

utils/pprof/pprof_test.go

@@ -0,0 +1,72 @@
+// pmm-managed
+// Copyright (C) 2017 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package pprof
+
+import (
+	"bytes"
+	"compress/gzip"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHeap(t *testing.T) {
+	t.Parallel()
+	t.Run("Heap test", func(t *testing.T) {
+		heapBytes, err := Heap(true)
+
+		// read gzip
+		reader, err := gzip.NewReader(bytes.NewBuffer(heapBytes))
+		assert.NoError(t, err)
+
+		var resB bytes.Buffer
+		_, err = resB.ReadFrom(reader)
+		assert.NoError(t, err)
+		assert.NotEmpty(t, resB.Bytes())
+	})
+}
+
+func TestProfile(t *testing.T) {
+	t.Parallel()
+	t.Run("Profile test", func(t *testing.T) {
+		profileBytes, err := Profile(1 * time.Second)
+
+		assert.NoError(t, err)
+		assert.NotEmpty(t, profileBytes)
+
+		// read gzip
+		reader, err := gzip.NewReader(bytes.NewBuffer(profileBytes))
+		assert.NoError(t, err)
+
+		var resB bytes.Buffer
+		_, err = resB.ReadFrom(reader)
+		assert.NoError(t, err)
+
+		assert.NotEmpty(t, resB.Bytes())
+	})
+}
+
+func TestTrace(t *testing.T) {
+	t.Parallel()
+	t.Run("Trace test", func(t *testing.T) {
+		traceBytes, err := Trace(1 * time.Second)
+
+		assert.NoError(t, err)
+		assert.NotEmpty(t, traceBytes)
+	})
+}