Merge pull request #127 from vinted/0370_fixes

*: pull various fixes

CHANGELOG.md

@@ -26,6 +26,10 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#7852](https://github.com/thanos-io/thanos/pull/7852) Query Frontend: pass "stats" parameter forward to queriers and fix Prometheus stats merging.
 - [#7832](https://github.com/thanos-io/thanos/pull/7832) Query Frontend: Fix cache keys for dynamic split intervals.
 - [#7885](https://github.com/thanos-io/thanos/pull/7885) Store: Return chunks to the pool after completing a Series call.
+- [#7893](https://github.com/thanos-io/thanos/pull/7893) Sidecar: Fix retrieval of external labels for Prometheus v3.0.0.
+- [#7903](https://github.com/thanos-io/thanos/pull/7903) Query: Fix panic on regex store matchers.
+- [#7915](https://github.com/thanos-io/thanos/pull/7915) Store: Close block series client at the end to not reuse chunk buffer
+- [#7941](https://github.com/thanos-io/thanos/pull/7941) Receive: Fix race condition when adding multiple new tenants, see [issue-7892](https://github.com/thanos-io/thanos/issues/7892).
 
 ### Added
 - [#7763](https://github.com/thanos-io/thanos/pull/7763) Ruler: use native histograms for client latency metrics.

cmd/thanos/config.go

@@ -32,10 +32,15 @@ type grpcConfig struct {
 	maxConnectionAge time.Duration
 }
 
-func (gc *grpcConfig) registerFlag(cmd extkingpin.FlagClause) *grpcConfig {
+func (gc *grpcConfig) registerFlag(cmd extkingpin.FlagClause, optionalListener bool) *grpcConfig {
+	var addr = "0.0.0.0:10901"
+	if optionalListener {
+		addr = ""
+	}
 	cmd.Flag("grpc-address",
 		"Listen ip:port address for gRPC endpoints (StoreAPI). Make sure this address is routable from other components.").
-		Default("0.0.0.0:10901").StringVar(&gc.bindAddress)
+		Default(addr).StringVar(&gc.bindAddress)
+
 	cmd.Flag("grpc-server-tls-cert",
 		"TLS Certificate for gRPC server, leave blank to disable TLS").
 		Default("").StringVar(&gc.tlsSrvCert)

cmd/thanos/query.go

@@ -83,7 +83,7 @@ func registerQuery(app *extkingpin.App) {
 	httpBindAddr, httpGracePeriod, httpTLSConfig := extkingpin.RegisterHTTPFlags(cmd)
 
 	var grpcServerConfig grpcConfig
-	grpcServerConfig.registerFlag(cmd)
+	grpcServerConfig.registerFlag(cmd, false)
 
 	secure := cmd.Flag("grpc-client-tls-secure", "Use TLS when talking to the gRPC server").Default("false").Bool()
 	skipVerify := cmd.Flag("grpc-client-tls-skip-verify", "Disable TLS certificate verification i.e self signed, signed by fake CA").Default("false").Bool()

cmd/thanos/query_frontend.go

@@ -34,9 +34,11 @@ import (
 	"github.com/thanos-io/thanos/pkg/logging"
 	"github.com/thanos-io/thanos/pkg/prober"
 	"github.com/thanos-io/thanos/pkg/queryfrontend"
+	grpcserver "github.com/thanos-io/thanos/pkg/server/grpc"
 	httpserver "github.com/thanos-io/thanos/pkg/server/http"
 	"github.com/thanos-io/thanos/pkg/server/http/middleware"
 	"github.com/thanos-io/thanos/pkg/tenancy"
+	"github.com/thanos-io/thanos/pkg/tls"
 	"github.com/thanos-io/thanos/pkg/tracing"
 )
 
@@ -67,6 +69,9 @@ func registerQueryFrontend(app *extkingpin.App) {
 
 	cfg.http.registerFlag(cmd)
 
+	var grpcServerConfig grpcConfig
+	grpcServerConfig.registerFlag(cmd, true)
+
 	cmd.Flag("web.disable-cors", "Whether to disable CORS headers to be set by Thanos. By default Thanos sets CORS headers to be allowed by all.").
 		Default("false").BoolVar(&cfg.webDisableCORS)
 
@@ -171,7 +176,7 @@ func registerQueryFrontend(app *extkingpin.App) {
 			return errors.Wrap(err, "error while parsing config for request logging")
 		}
 
-		return runQueryFrontend(g, logger, reg, tracer, httpLogOpts, cfg, comp)
+		return runQueryFrontend(g, logger, reg, tracer, httpLogOpts, cfg, comp, grpcServerConfig)
 	})
 }
 
@@ -237,6 +242,7 @@ func runQueryFrontend(
 	httpLogOpts []logging.Option,
 	cfg *queryFrontendConfig,
 	comp component.Component,
+	grpcServerConfig grpcConfig,
 ) error {
 	tenantHeaderProvided := cfg.TenantHeader != "" && cfg.TenantHeader != tenancy.DefaultTenantHeader
 	// If tenant header is set and different from the default tenant header, add it to the list of org id headers.
@@ -328,15 +334,43 @@ func runQueryFrontend(
 	}
 
 	httpProbe := prober.NewHTTP()
+	grpcProbe := prober.NewGRPC()
+
 	statusProber := prober.Combine(
 		httpProbe,
+		grpcProbe,
 		prober.NewInstrumentation(comp, logger, extprom.WrapRegistererWithPrefix("thanos_", reg)),
 	)
 
 	// Configure Request Logging for HTTP calls.
 	logMiddleware := logging.NewHTTPServerMiddleware(logger, httpLogOpts...)
 	ins := extpromhttp.NewTenantInstrumentationMiddleware(cfg.TenantHeader, cfg.DefaultTenant, reg, nil)
 
+	instr := func(f http.HandlerFunc) http.HandlerFunc {
+		hf := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			orgId := extractOrgId(cfg, r)
+			name := "query-frontend"
+			if !cfg.webDisableCORS {
+				api.SetCORS(w)
+			}
+			middleware.RequestID(
+				tracing.HTTPMiddleware(
+					tracer,
+					name,
+					logger,
+					ins.NewHandler(
+						name,
+						logMiddleware.HTTPMiddleware(name, f),
+					),
+					// Cortex frontend middlewares require orgID.
+				),
+			).ServeHTTP(w, r.WithContext(user.InjectOrgID(r.Context(), orgId)))
+		})
+		return hf
+	}
+
+	instrumentedHandler := instr(handler.ServeHTTP)
+
 	// Start metrics HTTP server.
 	{
 		srv := httpserver.New(logger, reg, comp, httpProbe,
@@ -345,29 +379,7 @@ func runQueryFrontend(
 			httpserver.WithTLSConfig(cfg.http.tlsConfig),
 		)
 
-		instr := func(f http.HandlerFunc) http.HandlerFunc {
-			hf := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				orgId := extractOrgId(cfg, r)
-				name := "query-frontend"
-				if !cfg.webDisableCORS {
-					api.SetCORS(w)
-				}
-				middleware.RequestID(
-					tracing.HTTPMiddleware(
-						tracer,
-						name,
-						logger,
-						ins.NewHandler(
-							name,
-							logMiddleware.HTTPMiddleware(name, f),
-						),
-						// Cortex frontend middlewares require orgID.
-					),
-				).ServeHTTP(w, r.WithContext(user.InjectOrgID(r.Context(), orgId)))
-			})
-			return hf
-		}
-		srv.Handle("/", instr(handler.ServeHTTP))
+		srv.Handle("/", instrumentedHandler)
 
 		g.Add(func() error {
 			statusProber.Healthy()
@@ -381,8 +393,37 @@ func runQueryFrontend(
 		})
 	}
 
+	if grpcServerConfig.bindAddress != "" {
+		level.Info(logger).Log("msg", "starting gRPC server", "address", grpcServerConfig.bindAddress)
+		tlsCfg, err := tls.NewServerConfig(log.With(logger, "protocol", "gRPC"), grpcServerConfig.tlsSrvCert, grpcServerConfig.tlsSrvKey, grpcServerConfig.tlsSrvClientCA)
+		if err != nil {
+			return errors.Wrap(err, "setup gRPC server")
+		}
+
+		httpgrpcServer := queryfrontend.NewHTTPGRPCServer(instrumentedHandler)
+
+		s := grpcserver.New(logger, reg, tracer, nil, nil, comp, grpcProbe,
+			grpcserver.WithServer(queryfrontend.RegisterHTTPGRPCServer(httpgrpcServer)),
+			grpcserver.WithListen(grpcServerConfig.bindAddress),
+			grpcserver.WithGracePeriod(grpcServerConfig.gracePeriod),
+			grpcserver.WithMaxConnAge(grpcServerConfig.maxConnectionAge),
+			grpcserver.WithTLSConfig(tlsCfg),
+		)
+
+		g.Add(func() error {
+			statusProber.Ready()
+
+			return s.ListenAndServe()
+		}, func(error) {
+			statusProber.NotReady(err)
+			s.Shutdown(err)
+		})
+	} else {
+		level.Info(logger).Log("msg", "gRPC server is disabled")
+		statusProber.Ready()
+	}
+
 	level.Info(logger).Log("msg", "starting query frontend")
-	statusProber.Ready()
 	return nil
 }
 

cmd/thanos/receive.go

@@ -895,7 +895,7 @@ type receiveConfig struct {
 
 func (rc *receiveConfig) registerFlag(cmd extkingpin.FlagClause) {
 	rc.httpBindAddr, rc.httpGracePeriod, rc.httpTLSConfig = extkingpin.RegisterHTTPFlags(cmd)
-	rc.grpcConfig.registerFlag(cmd)
+	rc.grpcConfig.registerFlag(cmd, false)
 	rc.storeRateLimits.RegisterFlags(cmd)
 
 	cmd.Flag("remote-write.address", "Address to listen on for remote write requests.").

cmd/thanos/rule.go

@@ -120,7 +120,7 @@ type Expression struct {
 
 func (rc *ruleConfig) registerFlag(cmd extkingpin.FlagClause) {
 	rc.http.registerFlag(cmd)
-	rc.grpc.registerFlag(cmd)
+	rc.grpc.registerFlag(cmd, false)
 	rc.web.registerFlag(cmd)
 	rc.shipper.registerFlag(cmd)
 	rc.query.registerFlag(cmd)

cmd/thanos/sidecar.go

@@ -551,7 +551,7 @@ type sidecarConfig struct {
 
 func (sc *sidecarConfig) registerFlag(cmd extkingpin.FlagClause) {
 	sc.http.registerFlag(cmd)
-	sc.grpc.registerFlag(cmd)
+	sc.grpc.registerFlag(cmd, false)
 	sc.prometheus.registerFlag(cmd)
 	sc.tsdb.registerFlag(cmd)
 	sc.reloader.registerFlag(cmd)

cmd/thanos/store.go

@@ -106,7 +106,7 @@ type storeConfig struct {
 
 func (sc *storeConfig) registerFlag(cmd extkingpin.FlagClause) {
 	sc.httpConfig = *sc.httpConfig.registerFlag(cmd)
-	sc.grpcConfig = *sc.grpcConfig.registerFlag(cmd)
+	sc.grpcConfig = *sc.grpcConfig.registerFlag(cmd, false)
 	sc.storeRateLimits.RegisterFlags(cmd)
 
 	cmd.Flag("data-dir", "Local data directory used for caching purposes (index-header, in-mem cache items and meta.jsons). If removed, no data will be lost, just store will have to rebuild the cache. NOTE: Putting raw blocks here will not cause the store to read them. For such use cases use Prometheus + sidecar. Ignored if --no-cache-index-header option is specified.").

docs/sharding.md

@@ -18,7 +18,7 @@ Queries against store gateway which are touching large number of blocks (no matt
 
 # Relabelling
 
-Similar to [promtail](https://grafana.com/docs/loki/latest/send-data/promtail/configuration/#relabel_configs) this config follows native [Prometheus relabel-config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) syntax.
+Similar to [promtail](https://grafana.com/docs/loki/latest/clients/promtail/configuration/#relabel_configs) this config follows native [Prometheus relabel-config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) syntax.
 
 Currently, thanos only supports the following relabel actions:
 

go.mod

@@ -217,7 +217,7 @@ require (
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/gobwas/ws v1.2.1 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
-	github.com/gogo/googleapis v1.4.0 // indirect
+	github.com/gogo/googleapis v1.4.0
 	github.com/google/go-cmp v0.6.0
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect
@@ -285,7 +285,7 @@ replace (
 	// Required by Cortex https://github.com/cortexproject/cortex/pull/3051.
 	github.com/bradfitz/gomemcache => github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab
 
-	github.com/prometheus/prometheus => github.com/vinted/prometheus v1.8.2-0.20241104091239-2b97618294c8
+	github.com/prometheus/prometheus => github.com/vinted/prometheus v0.0.0-20241212114003-53a808bbaf43
 
 	// Pin kuberesolver/v5 to support new grpc version. Need to upgrade kuberesolver version on weaveworks/common.
 	github.com/sercand/kuberesolver/v4 => github.com/sercand/kuberesolver/v5 v5.1.1

go.sum

@@ -2271,8 +2271,8 @@ github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMW
 github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
 github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg=
 github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
-github.com/vinted/prometheus v1.8.2-0.20241104091239-2b97618294c8 h1:yLIYtyb+wGxhvSDYGV7ySkXXMx+S/TQj6dlzsWMG35w=
-github.com/vinted/prometheus v1.8.2-0.20241104091239-2b97618294c8/go.mod h1:n3/PY5be8xgYe+DUCjKdK0eSmDSQBQ6ZOoIUGmO9vEY=
+github.com/vinted/prometheus v0.0.0-20241212114003-53a808bbaf43 h1:3PZq7phU4tTEF8It55AHuM8mAx18edD9guvlWiAHyz8=
+github.com/vinted/prometheus v0.0.0-20241212114003-53a808bbaf43/go.mod h1:n3/PY5be8xgYe+DUCjKdK0eSmDSQBQ6ZOoIUGmO9vEY=
 github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs=
 github.com/vultr/govultr/v2 v2.17.2/go.mod h1:ZFOKGWmgjytfyjeyAdhQlSWwTjh2ig+X49cAp50dzXI=
 github.com/weaveworks/common v0.0.0-20230728070032-dd9e68f319d5 h1:nORobjToZAvi54wcuUXLq+XG2Rsr0XEizy5aHBHvqWQ=