Merge commit 'a768a3b95e65efd0338cb2db598191ecd117f362' into juliendu…

…chesne/upstream-pt3

CHANGELOG.md

@@ -2,9 +2,53 @@
 
 ## unreleased
 
-* [CHANGE] Notifier: Increment the prometheus_notifications_errors_total metric by the number of affected alerts rather than by one per batch of affected alerts. #15428
-* [ENHANCEMENT] OTLP receiver: Convert also metric metadata. #15416
-* [BUGFIX] OTLP receiver: Allow colons in non-standard units. #15710
+## 3.1.0 / 2025-01-02
+
+ * [SECURITY] upgrade golang.org/x/crypto to address reported CVE-2024-45337. #15691
+ * [CHANGE] Notifier: Increment prometheus_notifications_errors_total by the number of affected alerts rather than per batch. #15428
+ * [CHANGE] API: list rules field "groupNextToken:omitempty" renamed to "groupNextToken". #15400
+ * [ENHANCEMENT] OTLP translate: keep identifying attributes in target_info. #15448
+ * [ENHANCEMENT] Paginate rule groups, add infinite scroll to rules within groups. #15677
+ * [ENHANCEMENT] TSDB: Improve calculation of space used by labels. #13880
+ * [ENHANCEMENT] Rules: new metric rule_group_last_rule_duration_sum_seconds. #15672
+ * [ENHANCEMENT] Observability: Export 'go_sync_mutex_wait_total_seconds_total' metric. #15339
+ * [ENHANCEMEN] Remote-Write: optionally use a DNS resolver that picks a random IP. #15329
+ * [PERF] Optimize `l=~".+"` matcher. #15474, #15684
+ * [PERF] TSDB: Cache all symbols for compaction . #15455
+ * [PERF] TSDB: MemPostings: keep a map of label values slices. #15426
+ * [PERF] Remote-Write: Remove interning hook. #15456
+ * [PERF] Scrape: optimize string manipulation for experimental native histograms with custom buckets. #15453
+ * [PERF] TSDB: reduce memory allocations. #15465, #15427
+ * [PERF] Storage: Implement limit in mergeGenericQuerier. #14489
+ * [PERF] TSDB: Optimize inverse matching. #14144
+ * [PERF] Regex: use stack memory for lowercase copy of string. #15210
+ * [PERF] TSDB: When deleting from postings index, pause to unlock and let readers read. #15242
+ * [BUGFIX] Main: Avoid possible segfault at exit. (#15724)
+ * [BUGFIX] Rules: Do not run rules concurrently if uncertain about dependencies. #15560
+ * [BUGFIX] PromQL: Adds test for `absent`, `absent_over_time` and `deriv` func with histograms. #15667
+ * [BUGFIX] PromQL: Fix various bugs related to quoting UTF-8 characters. #15531
+ * [BUGFIX] Scrape: fix nil panic after scrape loop reload. #15563
+ * [BUGFIX] Remote-write: fix panic on repeated log message. #15562
+ * [BUGFIX] Scrape: reload would ignore always_scrape_classic_histograms and convert_classic_histograms_to_nhcb configs. #15489
+ * [BUGFIX] TSDB: fix data corruption in experimental native histograms. #15482
+ * [BUGFIX] PromQL: Ignore histograms in all time related functions. #15479
+ * [BUGFIX] OTLP receiver: Convert metric metadata. #15416
+ * [BUGFIX] PromQL: Fix `resets` function for histograms. #15527
+ * [BUGFIX] PromQL: Fix behaviour of `changes()` for mix of histograms and floats. #15469
+ * [BUGFIX] PromQL: Fix behaviour of some aggregations with histograms. #15432
+ * [BUGFIX] allow quoted exemplar keys in openmetrics text format. #15260
+ * [BUGFIX] TSDB: fixes for rare conditions when loading write-behind-log (WBL). #15380
+ * [BUGFIX] `round()` function did not remove `__name__` label. #15250
+ * [BUGFIX] Promtool: analyze block shows metric name with 0 cardinality. #15438
+ * [BUGFIX] PromQL: Fix `count_values` for histograms. #15422
+ * [BUGFIX] PromQL: fix issues with comparison binary operations with `bool` modifier and native histograms. #15413
+ * [BUGFIX] PromQL: fix incorrect "native histogram ignored in aggregation" annotations. #15414
+ * [BUGFIX] PromQL: Corrects the behaviour of some operator and aggregators with Native Histograms. #15245
+ * [BUGFIX] TSDB: Always return unknown hint for first sample in non-gauge histogram chunk. #15343
+ * [BUGFIX] PromQL: Clamp functions: Ignore any points with native histograms. #15169
+ * [BUGFIX] TSDB: Fix race on stale values in headAppender. #15322
+ * [BUGFIX] UI: Fix selector / series formatting for empty metric names. #15340
+ * [BUGFIX] OTLP receiver: Allow colons in non-standard units. #15710
 
 ## 3.0.1 / 2024-11-28
 
@@ -47,14 +91,14 @@ This release includes new features such as a brand new UI and UTF-8 support enab
 * [CHANGE] PromQL: Range selectors and the lookback delta are now left-open, i.e. a sample coinciding with the lower time limit is excluded rather than included. #13904
 * [CHANGE] Kubernetes SD: Remove support for `discovery.k8s.io/v1beta1` API version of EndpointSlice. This version is no longer served as of Kubernetes v1.25. #14365
 * [CHANGE] Kubernetes SD: Remove support for `networking.k8s.io/v1beta1` API version of Ingress. This version is no longer served as of Kubernetes v1.22. #14365
-* [CHANGE] UTF-8: Enable UTF-8 support by default. Prometheus now allows all UTF-8 characters in metric and label names. The corresponding `utf8-name` feature flag has been removed. #14705
+* [CHANGE] UTF-8: Enable UTF-8 support by default. Prometheus now allows all UTF-8 characters in metric and label names. The corresponding `utf8-name` feature flag has been removed. #14705, #15258
 * [CHANGE] Console: Remove example files for the console feature. Users can continue using the console feature by supplying their own JavaScript and templates. #14807
 * [CHANGE] SD: Enable the new service discovery manager by default. This SD manager does not restart unchanged discoveries upon reloading. This makes reloads faster and reduces pressure on service discoveries' sources. The corresponding `new-service-discovery-manager` feature flag has been removed. #14770
 * [CHANGE] Agent mode has been promoted to stable. The feature flag `agent` has been removed. To run Prometheus in Agent mode, use the new `--agent` cmdline arg instead. #14747
 * [CHANGE] Remove deprecated `remote-write-receiver`,`promql-at-modifier`, and `promql-negative-offset` feature flags. #13456, #14526
 * [CHANGE] Remove deprecated `storage.tsdb.allow-overlapping-blocks`, `alertmanager.timeout`, and `storage.tsdb.retention` flags. #14640, #14643
 * [FEATURE] OTLP receiver: Ability to skip UTF-8 normalization using `otlp.translation_strategy = NoUTF8EscapingWithSuffixes` configuration option. #15384
-* [FEATURE] Support config reload automatically - feature flag `auto-reload-config`. #14769
+* [FEATURE] Support config reload automatically - feature flag `auto-reload-config`. #14769, #15011
 * [ENHANCEMENT] Scraping, rules: handle targets reappearing, or rules moving group, when out-of-order is enabled. #14710
 * [ENHANCEMENT] Tools: add debug printouts to promtool rules unit testing #15196
 * [ENHANCEMENT] Scraping: support Created-Timestamp feature on native histograms. #14694

README.md

@@ -67,7 +67,7 @@ Prometheus will now be reachable at <http://localhost:9090/>.
 
 To build Prometheus from source code, You need:
 
-* Go [version 1.17 or greater](https://golang.org/doc/install).
+* Go [version 1.22 or greater](https://golang.org/doc/install).
 * NodeJS [version 16 or greater](https://nodejs.org/).
 * npm [version 7 or greater](https://www.npmjs.com/).
 

VERSION

@@ -1 +1 @@
-3.0.1
+3.1.0

docs/configuration/configuration.md

@@ -59,6 +59,7 @@ global:
   [ scrape_interval: <duration> | default = 1m ]
 
   # How long until a scrape request times out.
+  # It cannot be greater than the scrape interval.
   [ scrape_timeout: <duration> | default = 10s ]
 
   # The protocols to negotiate during a scrape with the client.
@@ -221,6 +222,7 @@ job_name: <job_name>
 [ scrape_interval: <duration> | default = <global_config.scrape_interval> ]
 
 # Per-scrape timeout when scraping this job.
+# It cannot be greater than the scrape interval.
 [ scrape_timeout: <duration> | default = <global_config.scrape_timeout> ]
 
 # The protocols to negotiate during a scrape with the client.

docs/configuration/template_reference.md

@@ -68,7 +68,7 @@ versions.
 
 | Name          | Arguments     | Returns |    Notes    |
 | ------------- | ------------- | ------- | ----------- |
-| title         | string        | string  | [strings.Title](https://golang.org/pkg/strings/#Title), capitalises first character of each word.|
+| title         | string        | string  | [cases.Title](https://pkg.go.dev/golang.org/x/text/cases#Title), capitalises first character of each word.|
 | toUpper       | string        | string  | [strings.ToUpper](https://golang.org/pkg/strings/#ToUpper), converts all characters to upper case.|
 | toLower       | string        | string  | [strings.ToLower](https://golang.org/pkg/strings/#ToLower), converts all characters to lower case.|
 | stripPort     | string        | string  | [net.SplitHostPort](https://pkg.go.dev/net#SplitHostPort), splits string into host and port, then returns only host.|

docs/http_sd.md

@@ -8,7 +8,7 @@ sort_rank: 7
 Prometheus provides a generic [HTTP Service Discovery](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config),
 that enables it to discover targets over an HTTP endpoint.
 
-The HTTP Service Discovery is complimentary to the supported service
+The HTTP Service Discovery is complementary to the supported service
 discovery mechanisms, and is an alternative to [File-based Service Discovery](https://prometheus.io/docs/guides/file-sd/#use-file-based-service-discovery-to-discover-scrape-targets).
 
 ## Comparison between File-Based SD and HTTP SD

docs/querying/api.md

@@ -1158,6 +1158,8 @@ $ curl http://localhost:9090/api/v1/status/runtimeinfo
   "data": {
     "startTime": "2019-11-02T17:23:59.301361365+01:00",
     "CWD": "/",
+    "hostname" : "DESKTOP-717H17Q",
+    "serverTime": "2025-01-05T18:27:33Z",
     "reloadConfigSuccess": true,
     "lastConfigTime": "2019-11-02T17:23:59+01:00",
     "timeSeriesCount": 873,

model/textparse/nhcbparse.go

@@ -177,61 +177,63 @@ func (p *NHCBParser) CreatedTimestamp() *int64 {
 }
 
 func (p *NHCBParser) Next() (Entry, error) {
-	if p.state == stateEmitting {
-		p.state = stateStart
-		if p.entry == EntrySeries {
-			isNHCB := p.handleClassicHistogramSeries(p.lset)
-			if isNHCB && !p.keepClassicHistograms {
-				// Do not return the classic histogram series if it was converted to NHCB and we are not keeping classic histograms.
-				return p.Next()
+	for {
+		if p.state == stateEmitting {
+			p.state = stateStart
+			if p.entry == EntrySeries {
+				isNHCB := p.handleClassicHistogramSeries(p.lset)
+				if isNHCB && !p.keepClassicHistograms {
+					// Do not return the classic histogram series if it was converted to NHCB and we are not keeping classic histograms.
+					continue
+				}
 			}
+			return p.entry, p.err
 		}
-		return p.entry, p.err
-	}
 
-	p.entry, p.err = p.parser.Next()
-	if p.err != nil {
-		if errors.Is(p.err, io.EOF) && p.processNHCB() {
-			return EntryHistogram, nil
-		}
-		return EntryInvalid, p.err
-	}
-	switch p.entry {
-	case EntrySeries:
-		p.bytes, p.ts, p.value = p.parser.Series()
-		p.metricString = p.parser.Metric(&p.lset)
-		// Check the label set to see if we can continue or need to emit the NHCB.
-		var isNHCB bool
-		if p.compareLabels() {
-			// Labels differ. Check if we can emit the NHCB.
-			if p.processNHCB() {
+		p.entry, p.err = p.parser.Next()
+		if p.err != nil {
+			if errors.Is(p.err, io.EOF) && p.processNHCB() {
 				return EntryHistogram, nil
 			}
-			isNHCB = p.handleClassicHistogramSeries(p.lset)
-		} else {
-			// Labels are the same. Check if after an exponential histogram.
-			if p.lastHistogramExponential {
-				isNHCB = false
-			} else {
+			return EntryInvalid, p.err
+		}
+		switch p.entry {
+		case EntrySeries:
+			p.bytes, p.ts, p.value = p.parser.Series()
+			p.metricString = p.parser.Metric(&p.lset)
+			// Check the label set to see if we can continue or need to emit the NHCB.
+			var isNHCB bool
+			if p.compareLabels() {
+				// Labels differ. Check if we can emit the NHCB.
+				if p.processNHCB() {
+					return EntryHistogram, nil
+				}
 				isNHCB = p.handleClassicHistogramSeries(p.lset)
+			} else {
+				// Labels are the same. Check if after an exponential histogram.
+				if p.lastHistogramExponential {
+					isNHCB = false
+				} else {
+					isNHCB = p.handleClassicHistogramSeries(p.lset)
+				}
+			}
+			if isNHCB && !p.keepClassicHistograms {
+				// Do not return the classic histogram series if it was converted to NHCB and we are not keeping classic histograms.
+				continue
 			}
+			return p.entry, p.err
+		case EntryHistogram:
+			p.bytes, p.ts, p.h, p.fh = p.parser.Histogram()
+			p.metricString = p.parser.Metric(&p.lset)
+			p.storeExponentialLabels()
+		case EntryType:
+			p.bName, p.typ = p.parser.Type()
 		}
-		if isNHCB && !p.keepClassicHistograms {
-			// Do not return the classic histogram series if it was converted to NHCB and we are not keeping classic histograms.
-			return p.Next()
+		if p.processNHCB() {
+			return EntryHistogram, nil
 		}
 		return p.entry, p.err
-	case EntryHistogram:
-		p.bytes, p.ts, p.h, p.fh = p.parser.Histogram()
-		p.metricString = p.parser.Metric(&p.lset)
-		p.storeExponentialLabels()
-	case EntryType:
-		p.bName, p.typ = p.parser.Type()
-	}
-	if p.processNHCB() {
-		return EntryHistogram, nil
 	}
-	return p.entry, p.err
 }
 
 // Return true if labels have changed and we should emit the NHCB.

promql/promqltest/README.md

@@ -22,7 +22,7 @@ Each test file contains a series of commands. There are three kinds of commands:
 
 * `load`
 * `clear`
-* `eval`
+* `eval` (including the variants `eval_fail`, `eval_warn`, `eval_info`, and `eval_ordered`)
 
 Each command is executed in the order given in the file.
 
@@ -50,12 +50,12 @@ load 1m
     my_metric{env="prod"} 5 2+3x2 _ stale {{schema:1 sum:3 count:22 buckets:[5 10 7]}}
 ```
 
-...will create a single series with labels `my_metric{env="prod"}`, with the following points:
+… will create a single series with labels `my_metric{env="prod"}`, with the following points:
 
 * t=0: value is 5
 * t=1m: value is 2
 * t=2m: value is 5
-* t=3m: value is 7
+* t=3m: value is 8
 * t=4m: no point
 * t=5m: stale marker
 * t=6m: native histogram with schema 1, sum -3, count 22 and bucket counts 5, 10 and 7
@@ -74,6 +74,7 @@ When loading a batch of classic histogram float series, you can optionally appen
 ## `eval` command
 
 `eval` runs a query against the test environment and asserts that the result is as expected.
+It requires the query to succeed without any (info or warn) annotations.
 
 Both instant and range queries are supported.
 
@@ -110,11 +111,18 @@ eval range from 0 to 3m step 1m sum by (env) (my_metric)
     {env="test"} 10 20 30 45
 ```
 
-Instant queries also support asserting that the series are returned in exactly the order specified: use `eval_ordered instant ...` instead of `eval instant ...`.
-This is not supported for range queries.
+To assert that a query succeeds with an info or warn annotation, use the
+`eval_info` or `eval_warn` commands, respectively.
 
-It is also possible to test that queries fail: use `eval_fail instant ...` or `eval_fail range ...`.
-`eval_fail` optionally takes an expected error message string or regexp to assert that the error message is as expected.
+Instant queries also support asserting that the series are returned in exactly
+the order specified: use `eval_ordered instant ...` instead of `eval instant
+...`. `eval_ordered` ignores any annotations. The assertion always fails for
+matrix results.
+
+To assert that a query fails, use the `eval_fail` command. `eval_fail` does not
+expect any result lines. Instead, it optionally accepts an expected error
+message string or regular expression to assert that the error message is as
+expected.
 
 For example:
 

promql/promqltest/test.go

@@ -39,6 +39,7 @@ import (
 	"github.com/prometheus/prometheus/promql/parser/posrange"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/util/almost"
+	"github.com/prometheus/prometheus/util/annotations"
 	"github.com/prometheus/prometheus/util/convertnhcb"
 	"github.com/prometheus/prometheus/util/teststorage"
 	"github.com/prometheus/prometheus/util/testutil"
@@ -692,6 +693,24 @@ func (ev *evalCmd) expectMetric(pos int, m labels.Labels, vals ...parser.Sequenc
 	ev.expected[h] = entry{pos: pos, vals: vals}
 }
 
+// checkAnnotations asserts if the annotations match the expectations.
+func (ev *evalCmd) checkAnnotations(expr string, annos annotations.Annotations) error {
+	countWarnings, countInfo := annos.CountWarningsAndInfo()
+	switch {
+	case ev.ordered:
+		// Ignore annotations if testing for order.
+	case !ev.warn && countWarnings > 0:
+		return fmt.Errorf("unexpected warnings evaluating query %q (line %d): %v", expr, ev.line, annos.AsErrors())
+	case ev.warn && countWarnings == 0:
+		return fmt.Errorf("expected warnings evaluating query %q (line %d) but got none", expr, ev.line)
+	case !ev.info && countInfo > 0:
+		return fmt.Errorf("unexpected info annotations evaluating query %q (line %d): %v", expr, ev.line, annos.AsErrors())
+	case ev.info && countInfo == 0:
+		return fmt.Errorf("expected info annotations evaluating query %q (line %d) but got none", expr, ev.line)
+	}
+	return nil
+}
+
 // compareResult compares the result value with the defined expectation.
 func (ev *evalCmd) compareResult(result parser.Value) error {
 	switch val := result.(type) {
@@ -1131,6 +1150,7 @@ func (t *test) execRangeEval(cmd *evalCmd, engine promql.QueryEngine) error {
 	if err != nil {
 		return fmt.Errorf("error creating range query for %q (line %d): %w", cmd.expr, cmd.line, err)
 	}
+	defer q.Close()
 	res := q.Exec(t.context)
 	if res.Err != nil {
 		if cmd.fail {
@@ -1142,18 +1162,9 @@ func (t *test) execRangeEval(cmd *evalCmd, engine promql.QueryEngine) error {
 	if res.Err == nil && cmd.fail {
 		return fmt.Errorf("expected error evaluating query %q (line %d) but got none", cmd.expr, cmd.line)
 	}
-	countWarnings, countInfo := res.Warnings.CountWarningsAndInfo()
-	switch {
-	case !cmd.warn && countWarnings > 0:
-		return fmt.Errorf("unexpected warnings evaluating query %q (line %d): %v", cmd.expr, cmd.line, res.Warnings)
-	case cmd.warn && countWarnings == 0:
-		return fmt.Errorf("expected warnings evaluating query %q (line %d) but got none", cmd.expr, cmd.line)
-	case !cmd.info && countInfo > 0:
-		return fmt.Errorf("unexpected info annotations evaluating query %q (line %d): %v", cmd.expr, cmd.line, res.Warnings)
-	case cmd.info && countInfo == 0:
-		return fmt.Errorf("expected info annotations evaluating query %q (line %d) but got none", cmd.expr, cmd.line)
+	if err := cmd.checkAnnotations(cmd.expr, res.Warnings); err != nil {
+		return err
 	}
-	defer q.Close()
 
 	if err := cmd.compareResult(res.Value); err != nil {
 		return fmt.Errorf("error in %s %s (line %d): %w", cmd, cmd.expr, cmd.line, err)
@@ -1196,16 +1207,8 @@ func (t *test) runInstantQuery(iq atModifierTestCase, cmd *evalCmd, engine promq
 	if res.Err == nil && cmd.fail {
 		return fmt.Errorf("expected error evaluating query %q (line %d) but got none", iq.expr, cmd.line)
 	}
-	countWarnings, countInfo := res.Warnings.CountWarningsAndInfo()
-	switch {
-	case !cmd.warn && countWarnings > 0:
-		return fmt.Errorf("unexpected warnings evaluating query %q (line %d): %v", iq.expr, cmd.line, res.Warnings)
-	case cmd.warn && countWarnings == 0:
-		return fmt.Errorf("expected warnings evaluating query %q (line %d) but got none", iq.expr, cmd.line)
-	case !cmd.info && countInfo > 0:
-		return fmt.Errorf("unexpected info annotations evaluating query %q (line %d): %v", iq.expr, cmd.line, res.Warnings)
-	case cmd.info && countInfo == 0:
-		return fmt.Errorf("expected info annotations evaluating query %q (line %d) but got none", iq.expr, cmd.line)
+	if err := cmd.checkAnnotations(iq.expr, res.Warnings); err != nil {
+		return err
 	}
 	err = cmd.compareResult(res.Value)
 	if err != nil {
@@ -1218,11 +1221,11 @@ func (t *test) runInstantQuery(iq atModifierTestCase, cmd *evalCmd, engine promq
 	if err != nil {
 		return fmt.Errorf("error creating range query for %q (line %d): %w", cmd.expr, cmd.line, err)
 	}
+	defer q.Close()
 	rangeRes := q.Exec(t.context)
 	if rangeRes.Err != nil {
 		return fmt.Errorf("error evaluating query %q (line %d) in range mode: %w", iq.expr, cmd.line, rangeRes.Err)
 	}
-	defer q.Close()
 	if cmd.ordered {
 		// Range queries are always sorted by labels, so skip this test case that expects results in a particular order.
 		return nil