diff --git a/CHANGELOG.md b/CHANGELOG.md
index 685f799212..9608c8f2f9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -70,6 +70,8 @@ Main (unreleased)
 
 - Fixed an issue where the `otelcol.processor.interval` could not be used because the debug metrics were not set to default. (@wildum)
 
+- Fixed an issue where `pyroscope.java` did not remove unused JFR files created by previous Alloy instances. (@swar8080)
+
 ### Other changes
 
 - Change the stability of the `livedebugging` feature from "experimental" to "generally available". (@wildum)
diff --git a/internal/component/pyroscope/java/loop.go b/internal/component/pyroscope/java/loop.go
index 6a25e214bb..b3c68a4f51 100644
--- a/internal/component/pyroscope/java/loop.go
+++ b/internal/component/pyroscope/java/loop.go
@@ -7,6 +7,8 @@ import (
 	_ "embed"
 	"fmt"
 	"os"
+	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 	"sync"
@@ -23,7 +25,12 @@ import (
 	gopsutil "github.com/shirou/gopsutil/v3/process"
 )
 
-const spyName = "alloy.java"
+const (
+	spyName       = "alloy.java"
+	processJfrDir = "/tmp"
+)
+
+var jfrFileNamePattern = regexp.MustCompile("^asprof-\\d+-\\d+\\.jfr$")
 
 type profilingLoop struct {
 	logger     log.Logger
@@ -45,6 +52,7 @@ type profilingLoop struct {
 func newProfilingLoop(pid int, target discovery.Target, logger log.Logger, profiler *asprof.Profiler, output *pyroscope.Fanout, cfg ProfilingConfig) *profilingLoop {
 	ctx, cancel := context.WithCancel(context.Background())
 	dist, err := profiler.DistributionForProcess(pid)
+	jfrFileName := fmt.Sprintf("asprof-%d-%d.jfr", os.Getpid(), pid)
 	p := &profilingLoop{
 		logger:   log.With(logger, "pid", pid),
 		output:   output,
@@ -52,7 +60,7 @@ func newProfilingLoop(pid int, target discovery.Target, logger log.Logger, profi
 		target:   target,
 		cancel:   cancel,
 		dist:     dist,
-		jfrFile:  fmt.Sprintf("/tmp/asprof-%d-%d.jfr", os.Getpid(), pid),
+		jfrFile:  filepath.Join(processJfrDir, jfrFileName),
 		cfg:      cfg,
 		profiler: profiler,
 	}
@@ -63,6 +71,16 @@ func newProfilingLoop(pid int, target discovery.Target, logger log.Logger, profi
 		return p
 	}
 
+	p.wg.Add(1)
+	go func() {
+		defer p.wg.Done()
+		// Clean-up files that weren't removed by a previous instance of alloy
+		err := p.cleanupOldJFRFiles(jfrFileName)
+		if err != nil {
+			_ = level.Warn(p.logger).Log("msg", "failed cleaning-up java jfr files created by a previous instance of alloy", "err", err)
+		}
+	}()
+
 	p.wg.Add(1)
 	go func() {
 		defer p.wg.Done()
@@ -275,3 +293,22 @@ func (p *profilingLoop) alive() bool {
 	}
 	return err == nil && exists
 }
+
+func (p *profilingLoop) cleanupOldJFRFiles(myFileName string) error {
+	dir := asprof.ProcessPath(processJfrDir, p.pid)
+	files, err := os.ReadDir(dir)
+	if err != nil {
+		return err
+	}
+
+	for _, file := range files {
+		if !file.IsDir() && jfrFileNamePattern.MatchString(file.Name()) && file.Name() != myFileName {
+			_ = level.Debug(p.logger).Log("msg", "deleting jfr file created by previous alloy process", "file", file.Name())
+			err := os.Remove(filepath.Join(dir, file.Name()))
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}