From b2073e6258a517fa0156a867947bbecfb71421e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gy=C3=B6rgy=20Krajcsovits?= Date: Fri, 10 Nov 2023 00:55:27 +0100 Subject: [PATCH 1/2] Fix wrong order of exemplars vs samples MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port of https://github.com/prometheus/prometheus/pull/13113 Signed-off-by: György Krajcsovits --- pkg/metrics/wal/wal.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pkg/metrics/wal/wal.go b/pkg/metrics/wal/wal.go index c3d74773755d..513a43df5da1 100644 --- a/pkg/metrics/wal/wal.go +++ b/pkg/metrics/wal/wal.go @@ -894,24 +894,28 @@ func (a *appender) log() error { buf = buf[:0] } - if len(a.pendingExamplars) > 0 { - buf = encoder.Exemplars(a.pendingExamplars, buf) + if len(a.pendingHistograms) > 0 { + buf = encoder.HistogramSamples(a.pendingHistograms, buf) if err := a.w.wal.Log(buf); err != nil { return err } buf = buf[:0] } - if len(a.pendingHistograms) > 0 { - buf = encoder.HistogramSamples(a.pendingHistograms, buf) + if len(a.pendingFloatHistograms) > 0 { + buf = encoder.FloatHistogramSamples(a.pendingFloatHistograms, buf) if err := a.w.wal.Log(buf); err != nil { return err } buf = buf[:0] } - if len(a.pendingFloatHistograms) > 0 { - buf = encoder.FloatHistogramSamples(a.pendingFloatHistograms, buf) + // Exemplars should be logged after samples (float/native histogram/etc), + // otherwise it might happen that we send the exemplars in a remote write + // batch before the samples, which in turn means the exemplar is rejected + // for missing series, since series are created due to samples. + if len(a.pendingExamplars) > 0 { + buf = encoder.Exemplars(a.pendingExamplars, buf) if err := a.w.wal.Log(buf); err != nil { return err } From 8415e325587ff4fc6dfc1ed71f37d0df88357c8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gy=C3=B6rgy=20Krajcsovits?= Date: Sat, 11 Nov 2023 23:31:55 +0100 Subject: [PATCH 2/2] Update changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: György Krajcsovits --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04afa8db6148..c58ca070e669 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -135,6 +135,10 @@ Main (unreleased) - Updating configuration for `loki.write` no longer drops data. (@thepalbi) +- Fixed a bug in WAL where exemplars were recorded before the first native histogram samples for new series, + resulting in remote write sending the exemplar first and Prometheus failing to ingest it due to missing + series. (@krajorama) + v0.37.4 (2023-11-06) -----------------