Merge branch 'main' into spartan0x117/update-memcached-exporter-v0.13.0

.drone/drone.yml

@@ -1384,6 +1384,18 @@ volumes:
     path: /var/run/docker.sock
   name: docker
 ---
+get:
+  name: app-id
+  path: infra/data/ci/agent/githubapp
+kind: secret
+name: app_id
+---
+get:
+  name: app-installation-id
+  path: infra/data/ci/agent/githubapp
+kind: secret
+name: app_installation_id
+---
 get:
   name: username
   path: infra/data/ci/docker_hub
@@ -1432,7 +1444,13 @@ get:
 kind: secret
 name: gpg_public_key
 ---
+get:
+  name: private-key
+  path: infra/data/ci/agent/githubapp
+kind: secret
+name: private_key
+---
 kind: signature
-hmac: cdbbd7e715a892c48356bd5d381e47ffb29940cb24ac898c3e495765bdd982e5
+hmac: 2ff5f7f665aaba388e272316c0b7aee419801f347d882577dbbb84a95e4be672
 
 ...

.drone/util/secrets.jsonnet

@@ -13,6 +13,9 @@ local newSecret(name) = {
   dockerconfigjson: newSecret('dockerconfigjson').getFrom(path='secret/data/common/gcr', name='.dockerconfigjson'),
   gcr_admin: newSecret('gcr_admin').getFrom(path='infra/data/ci/gcr-admin', name='.dockerconfigjson'),
   gh_token: newSecret('gh_token').getFrom(path='infra/data/ci/github/grafanabot', name='pat'),
+  private_key: newSecret('private_key').getFrom(path='infra/data/ci/agent/githubapp', name='private-key'),
+  app_id: newSecret('app_id').getFrom(path='infra/data/ci/agent/githubapp', name='app-id'),
+  app_installation_id: newSecret('app_installation_id').getFrom(path='infra/data/ci/agent/githubapp', name='app-installation-id'),
   gpg_public_key: newSecret('gpg_public_key').getFrom(path='infra/data/ci/packages-publish/gpg', name='public-key'),
   gpg_private_key: newSecret('gpg_private_key').getFrom(path='infra/data/ci/packages-publish/gpg', name='private-key'),
   gpg_passphrase: newSecret('gpg_passphrase').getFrom(path='infra/data/ci/packages-publish/gpg', name='passphrase'),

CHANGELOG.md

@@ -10,13 +10,24 @@ internal API changes are not present.
 Main (unreleased)
 -----------------
 
+> **BREAKING CHANGES**: This release has breaking changes. Please read entries
+> carefully and consult the [upgrade guide][] for specific instructions.
+
+### Breaking changes
+
+- `loki.source.file` component will no longer automatically detect and
+  decompress logs from compressed files. A new configuration block is available
+  to enable decompression explicitly. See the [upgrade guide][] for migration
+  instructions. (@thampiotr)
+
 ### Enhancements
 
 - Better validation of config file with `grafana-agentctl config-check` cmd (@fgouteroux)
 
 - Add [godeltaprof](https://github.com/grafana/godeltaprof) profiling types (`godeltaprof_memory`, `godeltaprof_mutex`, `godeltaprof_block`) to `pyroscope.scrape` component
 
 - Integrations: make `udev` data path configurable in the `node_exporter` integration. (@sduranc)
+- Add `log_format` configuration to eventhandler integration and the `loki.source.kubernetes_events` Flow component. (@sadovnikov)
 
 - Clustering: Enable peer discovery with the go-discover package. (@tpaschalis)
 
@@ -28,6 +39,17 @@ Main (unreleased)
 
 - Allow `loki.source.file` to define the encoding of files. (@tpaschalis)
 
+- Enable graceful termination when receiving SIGTERM/CTRL_SHUTDOWN_EVENT
+  signals. (@tpaschalis)
+
+- Added support for `promtail` configuration conversion in `grafana-agent convert` and `grafana-agent run` commands. (@thampiotr)
+
+- Flow: Add a new stage `non_indexed_labels` to attach non-indexed labels from extracted data to log line entry. (@vlad-diachenko)
+
+- `loki.write` now exposes basic WAL support. (@thepalbi)
+
+- `loki.write` WAL now exposes a last segment reclaimed metric. (@thepalbi)
+
 - Update `memcached_exporter` to `v0.13.0`, which includes bugfixes, new metrics,
   and the option to connect with TLS. (@spartan0x117)
 
@@ -39,10 +61,12 @@ Main (unreleased)
   - `discovery.uyuni` discovers scrape targets from a Uyuni Server. (@sparta0x117)
   - `discovery.eureka` discovers targets from a Eureka Service Registry. (@spartan0x117)
   - `discovery.openstack` - service discovery for OpenStack. (@marctc)
-  - `loki.write` now exposes basic WAL support. (@thepalbi)
+  - `discovery.hetzner` - service discovery for Hetzner Cloud. (@marctc)
 
 ### Bugfixes
 
+- Update to config converter so default relabel `source_labels` are left off the river output. (@erikbaranowski)
+
 - Rename `GrafanaAgentManagement` mixin rules to `GrafanaAgentConfig` and update individual alerts to be more accurate. (@spartan0x117)
 
 - Fix potential goroutine leak in log file tailing in static mode. (@thampiotr)
@@ -51,6 +75,11 @@ Main (unreleased)
 
 - Fix issue on Windows where DNS short names were unresolvable. (@rfratto)
 
+- (Agent static mode) Jaeger remote sampling works again, through a new `jaeger_remote_sampling`
+  entry in the traces config. It is no longer configurable through the jaeger receiver.
+  Support Jaeger remote sampling was removed accidentally in v0.35, and it is now restored, 
+  albeit via a different config entry.
+
 v0.35.2 (2023-07-27)
 --------------------
 
@@ -137,6 +166,10 @@ v0.35.0 (2023-07-18)
 
 - The `remote_sampling` block has been removed from `otelcol.receiver.jaeger`. (@ptodev)
 
+- (Agent static mode) Jaeger remote sampling used to be configured using the Jaeger receiver configuration. 
+  This receiver was updated to a new version, where support for remote sampling in the receiver was removed. 
+  Jaeger remote sampling is available as a separate configuration field starting in v0.35.3. (@ptodev)
+
 ### Deprecations
 
 - `otelcol.exporter.jaeger` has been deprecated and will be removed in Agent v0.38.0. (@ptodev)

cmd/internal/flowmode/cmd_convert.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strings"
 
 	"github.com/spf13/cobra"
 
@@ -68,7 +69,7 @@ output can still be generated.`,
 
 	cmd.Flags().StringVarP(&f.output, "output", "o", f.output, "The filepath and filename where the output is written.")
 	cmd.Flags().StringVarP(&f.report, "report", "r", f.report, "The filepath and filename where the report is written.")
-	cmd.Flags().StringVarP(&f.sourceFormat, "source-format", "f", f.sourceFormat, "The format of the source file. Supported formats: 'prometheus'.")
+	cmd.Flags().StringVarP(&f.sourceFormat, "source-format", "f", f.sourceFormat, fmt.Sprintf("The format of the source file. Supported formats: %s.", supportedFormatsList()))
 	cmd.Flags().BoolVarP(&f.bypassErrors, "bypass-errors", "b", f.bypassErrors, "Enable bypassing errors when converting")
 	return cmd
 }
@@ -165,3 +166,11 @@ func hasErrorLevel(ds convert_diag.Diagnostics, sev convert_diag.Severity) bool
 
 	return false
 }
+
+func supportedFormatsList() string {
+	var ret = make([]string, len(converter.SupportedFormats))
+	for i, f := range converter.SupportedFormats {
+		ret[i] = fmt.Sprintf("%q", f)
+	}
+	return strings.Join(ret, ", ")
+}

cmd/internal/flowmode/cmd_run.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net"
 	"os"
 	"os/signal"
 	"sync"
@@ -24,6 +23,7 @@ import (
 	"github.com/grafana/agent/pkg/flow/tracing"
 	"github.com/grafana/agent/pkg/river/diag"
 	"github.com/grafana/agent/pkg/usagestats"
+	"github.com/grafana/agent/service"
 	httpservice "github.com/grafana/agent/service/http"
 	"github.com/grafana/ckit/peer"
 	"github.com/prometheus/client_golang/prometheus"
@@ -177,31 +177,42 @@ func (fr *flowRun) Run(configFile string) error {
 		}
 	}()
 
-	var httpData httpservice.Data
+	// There's a cyclic dependency between the definition of the Flow controller,
+	// the reload/ready functions, and the HTTP service.
+	//
+	// To work around this, we lazily create variables for the functions the HTTP
+	// service needs and set them after the Flow controller exists.
+	var (
+		reload func() error
+		ready  func() bool
+	)
+
+	httpService := httpservice.New(httpservice.Options{
+		Logger:   log.With(l, "service", "http"),
+		Tracer:   t,
+		Gatherer: prometheus.DefaultGatherer,
+
+		Clusterer:  clusterer,
+		ReadyFunc:  func() bool { return ready() },
+		ReloadFunc: func() error { return reload() },
+
+		HTTPListenAddr:   fr.httpListenAddr,
+		MemoryListenAddr: fr.inMemoryAddr,
+		UIPrefix:         fr.uiPrefix,
+		EnablePProf:      fr.enablePprof,
+	})
 
 	f := flow.New(flow.Options{
-		Logger:         l,
-		Tracer:         t,
-		Clusterer:      clusterer,
-		DataPath:       fr.storagePath,
-		Reg:            reg,
-		HTTPPathPrefix: "/api/v0/component/",
-		HTTPListenAddr: fr.inMemoryAddr,
-
-		// Send requests to fr.inMemoryAddr directly to our in-memory listener.
-		DialFunc: func(ctx context.Context, network, address string) (net.Conn, error) {
-			// NOTE(rfratto): this references a variable because httpData isn't
-			// non-zero by the time we are constructing the Flow controller.
-			//
-			// This is a temporary hack while services are being built out.
-			//
-			// It is not possibl for this to panic, since we will always have the service
-			// constructed by the time anything in Flow invokes this function.
-			return httpData.DialFunc(ctx, network, address)
-		},
+		Logger:    l,
+		Tracer:    t,
+		Clusterer: clusterer,
+		DataPath:  fr.storagePath,
+		Reg:       reg,
+		Services:  []service.Service{httpService},
 	})
 
-	reload := func() error {
+	ready = f.Ready
+	reload = func() error {
 		flowCfg, err := loadFlowFile(configFile, fr.configFormat, fr.configBypassConversionErrors)
 		defer instrumentation.InstrumentLoad(err == nil)
 
@@ -215,22 +226,6 @@ func (fr *flowRun) Run(configFile string) error {
 		return nil
 	}
 
-	httpService := httpservice.New(httpservice.Options{
-		Logger:   log.With(l, "service", "http"),
-		Tracer:   t,
-		Gatherer: prometheus.DefaultGatherer,
-
-		Clusterer:  clusterer,
-		ReadyFunc:  func() bool { return f.Ready() },
-		ReloadFunc: reload,
-
-		HTTPListenAddr:   fr.httpListenAddr,
-		MemoryListenAddr: fr.inMemoryAddr,
-		UIPrefix:         fr.uiPrefix,
-		EnablePProf:      fr.enablePprof,
-	})
-	httpData = httpService.Data().(httpservice.Data)
-
 	// Flow controller
 	{
 		wg.Add(1)
@@ -240,15 +235,6 @@ func (fr *flowRun) Run(configFile string) error {
 		}()
 	}
 
-	// HTTP service
-	{
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			_ = httpService.Run(ctx, f)
-		}()
-	}
-
 	// Report usage of enabled components
 	if !fr.disableReporting {
 		reporter, err := usagestats.NewReporter(l)
@@ -360,7 +346,7 @@ func interruptContext() (context.Context, context.CancelFunc) {
 	go func() {
 		defer cancel()
 		sig := make(chan os.Signal, 1)
-		signal.Notify(sig, os.Interrupt)
+		signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
 		select {
 		case <-sig:
 		case <-ctx.Done():

component/all/all.go

@@ -11,6 +11,7 @@ import (
 	_ "github.com/grafana/agent/component/discovery/eureka"                         // Import discovery.eureka
 	_ "github.com/grafana/agent/component/discovery/file"                           // Import discovery.file
 	_ "github.com/grafana/agent/component/discovery/gce"                            // Import discovery.gce
+	_ "github.com/grafana/agent/component/discovery/hetzner"                        // Import discovery.hetzner
 	_ "github.com/grafana/agent/component/discovery/http"                           // Import discovery.http
 	_ "github.com/grafana/agent/component/discovery/kubelet"                        // Import discovery.kubelet
 	_ "github.com/grafana/agent/component/discovery/kubernetes"                     // Import discovery.kubernetes

component/common/loki/wal/writer.go

@@ -58,6 +58,7 @@ type Writer struct {
 	writeSubscribers     []WriteEventSubscriber
 
 	reclaimedOldSegmentsSpaceCounter *prometheus.CounterVec
+	lastReclaimedSegment             *prometheus.GaugeVec
 
 	closeCleaner chan struct{}
 }
@@ -89,8 +90,16 @@ func NewWriter(walCfg Config, logger log.Logger, reg prometheus.Registerer) (*Wr
 		Help:      "Number of bytes reclaimed from storage.",
 	}, []string{})
 
+	wrt.lastReclaimedSegment = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "loki_write",
+		Subsystem: "wal_writer",
+		Name:      "last_reclaimed_segment",
+		Help:      "Last reclaimed segment number",
+	}, []string{})
+
 	if reg != nil {
 		_ = reg.Register(wrt.reclaimedOldSegmentsSpaceCounter)
+		_ = reg.Register(wrt.lastReclaimedSegment)
 	}
 
 	wrt.start(walCfg.MaxSegmentAge)
@@ -202,6 +211,7 @@ func (wrt *Writer) cleanSegments(maxAge time.Duration) error {
 		for _, subscriber := range wrt.cleanupSubscribers {
 			subscriber.SeriesReset(maxReclaimed)
 		}
+		wrt.lastReclaimedSegment.WithLabelValues().Set(float64(maxReclaimed))
 	}
 	return nil
 }

component/component.go

@@ -60,7 +60,6 @@ package component
 
 import (
 	"context"
-	"net/http"
 )
 
 // The Arguments contains the input fields for a specific component, which is
@@ -115,17 +114,6 @@ type DebugComponent interface {
 	DebugInfo() interface{}
 }
 
-// HTTPComponent is an extension interface for components which contain their own HTTP handlers.
-type HTTPComponent interface {
-	Component
-
-	// Handler should return a valid HTTP handler for the component.
-	// All requests to the component will have the path trimmed such that the component is at the root.
-	// For example, f a request is made to `/component/{id}/metrics`, the component
-	// will receive a request to just `/metrics`.
-	Handler() http.Handler
-}
-
 // ClusteredComponent is an extension interface for components which implement
 // clustering-specific behavior.
 type ClusteredComponent interface {