Fix wrong order of exemplars vs samples (#5750)

Signed-off-by: György Krajcsovits <[email protected]>

CHANGELOG.md

@@ -139,6 +139,10 @@ Main (unreleased)
 
 - Updating configuration for `loki.write` no longer drops data. (@thepalbi)
 
+- Fixed a bug in WAL where exemplars were recorded before the first native histogram samples for new series,
+  resulting in remote write sending the exemplar first and Prometheus failing to ingest it due to missing
+  series. (@krajorama)
+
 v0.37.4 (2023-11-06)
 -----------------
 

pkg/metrics/wal/wal.go

@@ -894,24 +894,28 @@ func (a *appender) log() error {
 		buf = buf[:0]
 	}
 
-	if len(a.pendingExamplars) > 0 {
-		buf = encoder.Exemplars(a.pendingExamplars, buf)
+	if len(a.pendingHistograms) > 0 {
+		buf = encoder.HistogramSamples(a.pendingHistograms, buf)
 		if err := a.w.wal.Log(buf); err != nil {
 			return err
 		}
 		buf = buf[:0]
 	}
 
-	if len(a.pendingHistograms) > 0 {
-		buf = encoder.HistogramSamples(a.pendingHistograms, buf)
+	if len(a.pendingFloatHistograms) > 0 {
+		buf = encoder.FloatHistogramSamples(a.pendingFloatHistograms, buf)
 		if err := a.w.wal.Log(buf); err != nil {
 			return err
 		}
 		buf = buf[:0]
 	}
 
-	if len(a.pendingFloatHistograms) > 0 {
-		buf = encoder.FloatHistogramSamples(a.pendingFloatHistograms, buf)
+	// Exemplars should be logged after samples (float/native histogram/etc),
+	// otherwise it might happen that we send the exemplars in a remote write
+	// batch before the samples, which in turn means the exemplar is rejected
+	// for missing series, since series are created due to samples.
+	if len(a.pendingExamplars) > 0 {
+		buf = encoder.Exemplars(a.pendingExamplars, buf)
 		if err := a.w.wal.Log(buf); err != nil {
 			return err
 		}