PMM-5492 Adding pprof query parameter to /logs.zip

admin/commands/summary.go

@@ -152,9 +152,9 @@ func addClientData(ctx context.Context, zipW *zip.Writer) {
 }
 
 // addServerData adds logs.zip from PMM Server to zip file.
-func addServerData(ctx context.Context, zipW *zip.Writer) {
+func addServerData(ctx context.Context, zipW *zip.Writer, usePprof bool) {
 	var buf bytes.Buffer
-	_, err := client.Default.Server.Logs(&server.LogsParams{Context: ctx}, &buf)
+	_, err := client.Default.Server.Logs(&server.LogsParams{Context: ctx, Pprof: &usePprof}, &buf)
 	if err != nil {
 		logrus.Errorf("%s", err)
 		return
@@ -271,7 +271,6 @@ func addPprofData(ctx context.Context, zipW *zip.Writer, skipServer bool) {
 	isRunOnPmmServer, _ := helpers.IsOnPmmServer()
 
 	if !skipServer && isRunOnPmmServer {
-		sources["server/pprof/pmm-managed"] = fmt.Sprintf("http://%s:7773/debug/pprof", agentlocal.Localhost)
 		sources["server/pprof/qan-api2"] = fmt.Sprintf("http://%s:9933/debug/pprof", agentlocal.Localhost)
 	}
 
@@ -339,7 +338,7 @@ func (cmd *summaryCommand) makeArchive(ctx context.Context) (err error) {
 	}
 
 	if !cmd.SkipServer {
-		addServerData(ctx, zipW)
+		addServerData(ctx, zipW, cmd.Pprof)
 	}
 
 	return //nolint:nakedret

api/serverpb/json/header.json

@@ -17,6 +17,14 @@
         "description": "Returns the PMM Server logs.",
         "summary": "Logs",
         "operationId": "Logs",
+        "parameters": [
+          {
+            "type": "boolean",
+            "description": "Include performance profiling data,",
+            "name": "pprof",
+            "in": "query"
+          }
+        ],
         "produces": ["application/zip"],
         "responses": {
           "200": {

api/serverpb/json/serverpb.json

@@ -26,6 +26,14 @@
         ],
         "summary": "Logs",
         "operationId": "Logs",
+        "parameters": [
+          {
+            "type": "boolean",
+            "description": "Include performance profiling data,",
+            "name": "pprof",
+            "in": "query"
+          }
+        ],
         "responses": {
           "200": {
             "description": "A successful response.",

api/swagger/swagger-dev.json

@@ -27,6 +27,14 @@
         ],
         "summary": "Logs",
         "operationId": "Logs",
+        "parameters": [
+          {
+            "type": "boolean",
+            "description": "Include performance profiling data,",
+            "name": "pprof",
+            "in": "query"
+          }
+        ],
         "responses": {
           "200": {
             "description": "A successful response.",

api/swagger/swagger.json

@@ -26,6 +26,14 @@
         ],
         "summary": "Logs",
         "operationId": "Logs",
+        "parameters": [
+          {
+            "type": "boolean",
+            "description": "Include performance profiling data,",
+            "name": "pprof",
+            "in": "query"
+          }
+        ],
         "responses": {
           "200": {
             "description": "A successful response.",

managed/cmd/pmm-managed/main.go

@@ -108,14 +108,32 @@ const (
 
 	cleanInterval  = 10 * time.Minute
 	cleanOlderThan = 30 * time.Minute
+
+	defaultContextTimeout = 10 * time.Second
+	pProfProfileDuration  = 30 * time.Second
+	pProfTraceDuration    = 10 * time.Second
 )
 
 func addLogsHandler(mux *http.ServeMux, logs *supervisord.Logs) {
 	l := logrus.WithField("component", "logs.zip")
 
 	mux.HandleFunc("/logs.zip", func(rw http.ResponseWriter, req *http.Request) {
+		contextTimeout := defaultContextTimeout
+		// increase context timeout if pprof query parameter exist in request
+		pprofQueryParameter, err := strconv.ParseBool(req.FormValue("pprof"))
+		if err != nil {
+			l.Debug("Unable to read 'pprof' query param. Using default: pprof=false")
+		}
+		var pprofConfig *supervisord.PprofConfig
+		if pprofQueryParameter {
+			contextTimeout += pProfProfileDuration + pProfTraceDuration
+			pprofConfig = &supervisord.PprofConfig{
+				ProfileDuration: pProfProfileDuration,
+				TraceDuration:   pProfTraceDuration,
+			}
+		}
 		// fail-safe
-		ctx, cancel := context.WithTimeout(req.Context(), 10*time.Second)
+		ctx, cancel := context.WithTimeout(req.Context(), contextTimeout)
 		defer cancel()
 
 		filename := fmt.Sprintf("pmm-server_%s.zip", time.Now().UTC().Format("2006-01-02_15-04"))
@@ -124,7 +142,7 @@ func addLogsHandler(mux *http.ServeMux, logs *supervisord.Logs) {
 		rw.Header().Set(`Content-Disposition`, `attachment; filename="`+filename+`"`)
 
 		ctx = logger.Set(ctx, "logs")
-		if err := logs.Zip(ctx, rw); err != nil {
+		if err := logs.Zip(ctx, rw, pprofConfig); err != nil {
 			l.Errorf("%+v", err)
 		}
 	})

managed/services/supervisord/logs.go

@@ -30,12 +30,14 @@ import (
 	"os/exec"
 	"path/filepath"
 	"sort"
+	"sync"
 	"time"
 
 	"github.com/pkg/errors"
 	"golang.org/x/sys/unix"
 
 	"github.com/percona/pmm/managed/utils/logger"
+	pprofUtils "github.com/percona/pmm/managed/utils/pprof"
 	"github.com/percona/pmm/utils/pdeathsig"
 )
 
@@ -68,7 +70,7 @@ func NewLogs(pmmVersion string, pmmUpdateChecker *PMMUpdateChecker) *Logs {
 }
 
 // Zip creates .zip archive with all logs.
-func (l *Logs) Zip(ctx context.Context, w io.Writer) error {
+func (l *Logs) Zip(ctx context.Context, w io.Writer, pprofConfig *PprofConfig) error {
 	start := time.Now()
 	log := logger.Get(ctx).WithField("component", "logs")
 	log.WithField("d", time.Since(start).Seconds()).Info("Starting...")
@@ -79,7 +81,7 @@ func (l *Logs) Zip(ctx context.Context, w io.Writer) error {
 	zw := zip.NewWriter(w)
 	now := time.Now().UTC()
 
-	files := l.files(ctx)
+	files := l.files(ctx, pprofConfig)
 	log.WithField("d", time.Since(start).Seconds()).Infof("Collected %d files.", len(files))
 
 	for _, file := range files {
@@ -126,8 +128,8 @@ func (l *Logs) Zip(ctx context.Context, w io.Writer) error {
 	return nil
 }
 
-// files reads log/config files and returns content.
-func (l *Logs) files(ctx context.Context) []fileContent {
+// files reads log/config/pprof files and returns content.
+func (l *Logs) files(ctx context.Context, pprofConfig *PprofConfig) []fileContent {
 	files := make([]fileContent, 0, 20)
 
 	// add logs
@@ -213,6 +215,45 @@ func (l *Logs) files(ctx context.Context) []fileContent {
 		Err:  err,
 	})
 
+	// add pprof
+	if pprofConfig != nil {
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			traceBytes, err := pprofUtils.Trace(ctx, pprofConfig.TraceDuration)
+			files = append(files, fileContent{
+				Name: "pprof/trace.out",
+				Data: traceBytes,
+				Err:  err,
+			})
+		}()
+
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			profileBytes, err := pprofUtils.Profile(ctx, pprofConfig.ProfileDuration)
+			files = append(files, fileContent{
+				Name: "pprof/profile.pb.gz",
+				Data: profileBytes,
+				Err:  err,
+			})
+		}()
+
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			heapBytes, err := pprofUtils.Heap(true)
+			files = append(files, fileContent{
+				Name: "pprof/heap.pb.gz",
+				Data: heapBytes,
+				Err:  err,
+			})
+		}()
+
+		wg.Wait()
+	}
+
 	sort.Slice(files, func(i, j int) bool { return files[i].Name < files[j].Name })
 	return files
 }

managed/services/supervisord/logs_test.go

@@ -123,7 +123,7 @@ func TestFiles(t *testing.T) {
 	l := NewLogs("2.4.5", checker)
 	ctx := logger.Set(context.Background(), t.Name())
 
-	files := l.files(ctx)
+	files := l.files(ctx, nil)
 	actual := make([]string, 0, len(files))
 	for _, f := range files {
 		// present only after update
@@ -156,7 +156,7 @@ func TestZip(t *testing.T) {
 	ctx := logger.Set(context.Background(), t.Name())
 
 	var buf bytes.Buffer
-	require.NoError(t, l.Zip(ctx, &buf))
+	require.NoError(t, l.Zip(ctx, &buf, nil))
 	reader := bytes.NewReader(buf.Bytes())
 	r, err := zip.NewReader(reader, reader.Size())
 	require.NoError(t, err)

managed/services/supervisord/pprof_config.go

@@ -0,0 +1,25 @@
+// pmm-managed
+// Copyright (C) 2017 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package supervisord
+
+import "time"
+
+// PprofConfig pprof settings.
+type PprofConfig struct {
+	ProfileDuration time.Duration
+	TraceDuration   time.Duration
+}