Skip to content

Commit

Permalink
Clean-up JFR files left behind by previous instances of alloy to mini…
Browse files Browse the repository at this point in the history
…mize risk of filling up disk when alloy's in a crash loop
  • Loading branch information
swar8080 committed Dec 27, 2024
1 parent 2b42ec4 commit 037e9bd
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 2 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ Main (unreleased)

- Fixed an issue where the `otelcol.processor.interval` could not be used because the debug metrics were not set to default. (@wildum)

- Fixed an issue where `pyroscope.java` did not remove unused JFR files created by previous Alloy instances. (@swar8080)

### Other changes

- Change the stability of the `livedebugging` feature from "experimental" to "generally available". (@wildum)
Expand Down
41 changes: 39 additions & 2 deletions internal/component/pyroscope/java/loop.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
_ "embed"
"fmt"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
Expand All @@ -23,7 +25,12 @@ import (
gopsutil "github.com/shirou/gopsutil/v3/process"
)

const spyName = "alloy.java"
const (
spyName = "alloy.java"
processJfrDir = "/tmp"
)

var jfrFileNamePattern = regexp.MustCompile("^asprof-\\d+-\\d+\\.jfr$")

type profilingLoop struct {
logger log.Logger
Expand All @@ -45,14 +52,15 @@ type profilingLoop struct {
func newProfilingLoop(pid int, target discovery.Target, logger log.Logger, profiler *asprof.Profiler, output *pyroscope.Fanout, cfg ProfilingConfig) *profilingLoop {
ctx, cancel := context.WithCancel(context.Background())
dist, err := profiler.DistributionForProcess(pid)
jfrFileName := fmt.Sprintf("asprof-%d-%d.jfr", os.Getpid(), pid)
p := &profilingLoop{
logger: log.With(logger, "pid", pid),
output: output,
pid: pid,
target: target,
cancel: cancel,
dist: dist,
jfrFile: fmt.Sprintf("/tmp/asprof-%d-%d.jfr", os.Getpid(), pid),
jfrFile: filepath.Join(processJfrDir, jfrFileName),
cfg: cfg,
profiler: profiler,
}
Expand All @@ -63,6 +71,16 @@ func newProfilingLoop(pid int, target discovery.Target, logger log.Logger, profi
return p
}

p.wg.Add(1)
go func() {
defer p.wg.Done()
// Clean-up files that weren't removed by a previous instance of alloy
err := p.cleanupOldJFRFiles(jfrFileName)
if err != nil {
_ = level.Warn(p.logger).Log("msg", "failed cleaning-up java jfr files created by a previous instance of alloy", "err", err)
}
}()

p.wg.Add(1)
go func() {
defer p.wg.Done()
Expand Down Expand Up @@ -275,3 +293,22 @@ func (p *profilingLoop) alive() bool {
}
return err == nil && exists
}

func (p *profilingLoop) cleanupOldJFRFiles(myFileName string) error {
dir := asprof.ProcessPath(processJfrDir, p.pid)
files, err := os.ReadDir(dir)
if err != nil {
return err
}

for _, file := range files {
if !file.IsDir() && jfrFileNamePattern.MatchString(file.Name()) && file.Name() != myFileName {
_ = level.Debug(p.logger).Log("msg", "deleting jfr file created by previous alloy process", "file", file.Name())
err := os.Remove(filepath.Join(dir, file.Name()))
if err != nil {
return err
}
}
}
return nil
}

0 comments on commit 037e9bd

Please sign in to comment.