allow '-' rune in datastream namespace

elastic · Sep 25, 2024 · 4ce11ed · 4ce11ed
1 parent f40b9b1
commit 4ce11ed
Show file tree

Hide file tree

Showing 7 changed files with 59 additions and 42 deletions.
diff --git a/input/otlp/logs.go b/input/otlp/logs.go
@@ -195,12 +195,12 @@ func (c *Consumer) convertLogRecord(
 			if event.DataStream == nil {
 				event.DataStream = &modelpb.DataStream{}
 			}
-			event.DataStream.Dataset = sanitizeDataStreamField(v.Str())
+			event.DataStream.Dataset = sanitizeDataStreamDataset(v.Str())
 		case attributeDataStreamNamespace:
 			if event.DataStream == nil {
 				event.DataStream = &modelpb.DataStream{}
 			}
-			event.DataStream.Namespace = sanitizeDataStreamField(v.Str())
+			event.DataStream.Namespace = sanitizeDataStreamNamespace(v.Str())
 		default:
 			setLabel(replaceDots(k), event, v)
 		}

diff --git a/input/otlp/logs_test.go b/input/otlp/logs_test.go
@@ -534,7 +534,8 @@ func processLogEvents(t *testing.T, logs plog.Logs) modelpb.Batch {
 
 func TestConsumerConsumeLogsDataStream(t *testing.T) {
 	randomString := strings.Repeat("abcdefghijklmnopqrstuvwxyz0123456789", 10)
-	maxLen := otlp.MaxDataStreamBytes - len(otlp.DisallowedDataStreamRunes)
+	maxLenNamespace := otlp.MaxDataStreamBytes - len(otlp.DisallowedNamespaceRunes)
+	maxLenDataset := otlp.MaxDataStreamBytes - len(otlp.DisallowedDatasetRunes)
 
 	for _, tc := range []struct {
 		resourceDataStreamDataset   string
@@ -575,10 +576,10 @@ func TestConsumerConsumeLogsDataStream(t *testing.T) {
 		// 1. Replace all disallowed runes with _
 		// 2. Datastream length should not exceed otlp.MaxDataStreamBytes
 		{
-			resourceDataStreamDataset:   otlp.DisallowedDataStreamRunes + randomString,
-			resourceDataStreamNamespace: otlp.DisallowedDataStreamRunes + randomString,
-			expectedDataStreamDataset:   strings.Repeat("_", len(otlp.DisallowedDataStreamRunes)) + randomString[:maxLen],
-			expectedDataStreamNamespace: strings.Repeat("_", len(otlp.DisallowedDataStreamRunes)) + randomString[:maxLen],
+			resourceDataStreamDataset:   otlp.DisallowedDatasetRunes + randomString,
+			resourceDataStreamNamespace: otlp.DisallowedNamespaceRunes + randomString,
+			expectedDataStreamDataset:   strings.Repeat("_", len(otlp.DisallowedDatasetRunes)) + randomString[:maxLenDataset],
+			expectedDataStreamNamespace: strings.Repeat("_", len(otlp.DisallowedNamespaceRunes)) + randomString[:maxLenNamespace],
 		},
 	} {
 		tcName := fmt.Sprintf("%s,%s", tc.expectedDataStreamDataset, tc.expectedDataStreamNamespace)

diff --git a/input/otlp/metadata.go b/input/otlp/metadata.go
@@ -31,9 +31,10 @@ import (
 )
 
 const (
-	AgentNameJaeger           = "Jaeger"
-	MaxDataStreamBytes        = 100
-	DisallowedDataStreamRunes = "-\\/*?\"<>| ,#:"
+	AgentNameJaeger          = "Jaeger"
+	MaxDataStreamBytes       = 100
+	DisallowedNamespaceRunes = "\\/*?\"<>| ,#:"
+	DisallowedDatasetRunes   = "-\\/*?\"<>| ,#:"
 )
 
 var (
@@ -319,12 +320,12 @@ func translateResourceMetadata(resource pcommon.Resource, out *modelpb.APMEvent)
 			if out.DataStream == nil {
 				out.DataStream = &modelpb.DataStream{}
 			}
-			out.DataStream.Dataset = sanitizeDataStreamField(v.Str())
+			out.DataStream.Dataset = sanitizeDataStreamDataset(v.Str())
 		case attributeDataStreamNamespace:
 			if out.DataStream == nil {
 				out.DataStream = &modelpb.DataStream{}
 			}
-			out.DataStream.Namespace = sanitizeDataStreamField(v.Str())
+			out.DataStream.Namespace = sanitizeDataStreamNamespace(v.Str())
 		default:
 			if out.Labels == nil {
 				out.Labels = make(modelpb.Labels)
@@ -462,12 +463,12 @@ func translateScopeMetadata(scope pcommon.InstrumentationScope, out *modelpb.APM
 			if out.DataStream == nil {
 				out.DataStream = &modelpb.DataStream{}
 			}
-			out.DataStream.Dataset = sanitizeDataStreamField(v.Str())
+			out.DataStream.Dataset = sanitizeDataStreamDataset(v.Str())
 		case attributeDataStreamNamespace:
 			if out.DataStream == nil {
 				out.DataStream = &modelpb.DataStream{}
 			}
-			out.DataStream.Namespace = sanitizeDataStreamField(v.Str())
+			out.DataStream.Namespace = sanitizeDataStreamNamespace(v.Str())
 		}
 		return true
 	})
@@ -551,17 +552,30 @@ func setLabel(key string, event *modelpb.APMEvent, v pcommon.Value) {
 
 // Sanitize the datastream fields (dataset, namespace) to apply restrictions
 // as outlined in https://www.elastic.co/guide/en/ecs/current/ecs-data_stream.html
-func sanitizeDataStreamField(field string) string {
-	field = strings.Map(replaceReservedRune, field)
+func sanitizeDataStreamDataset(field string) string {
+	field = strings.Map(replaceReservedRune(DisallowedDatasetRunes), field)
+	if len(field) > MaxDataStreamBytes {
+		return field[:MaxDataStreamBytes]
+	}
+
+	return field
+}
+
+// Sanitize the datastream fields (dataset, namespace) to apply restrictions
+// as outlined in https://www.elastic.co/guide/en/ecs/current/ecs-data_stream.html
+func sanitizeDataStreamNamespace(field string) string {
+	field = strings.Map(replaceReservedRune(DisallowedNamespaceRunes), field)
 	if len(field) > MaxDataStreamBytes {
 		return field[:MaxDataStreamBytes]
 	}
 	return field
 }
 
-func replaceReservedRune(r rune) rune {
-	if strings.ContainsRune(DisallowedDataStreamRunes, r) {
-		return '_'
+func replaceReservedRune(disallowedRunes string) func(r rune) rune {
+	return func(r rune) rune {
+		if strings.ContainsRune(disallowedRunes, r) {
+			return '_'
+		}
+		return unicode.ToLower(r)
 	}
-	return unicode.ToLower(r)
 }
diff --git a/input/otlp/metrics.go b/input/otlp/metrics.go
@@ -180,12 +180,12 @@ func (c *Consumer) handleScopeMetrics(
 					if event.DataStream == nil {
 						event.DataStream = &modelpb.DataStream{}
 					}
-					event.DataStream.Dataset = sanitizeDataStreamField(v.Str())
+					event.DataStream.Dataset = sanitizeDataStreamDataset(v.Str())
 				case attributeDataStreamNamespace:
 					if event.DataStream == nil {
 						event.DataStream = &modelpb.DataStream{}
 					}
-					event.DataStream.Namespace = sanitizeDataStreamField(v.Str())
+					event.DataStream.Namespace = sanitizeDataStreamNamespace(v.Str())
 
 				// The below fields are required by the Processes tab of the
 				// curated Kibana's hostmetrics UI. These fields are

diff --git a/input/otlp/metrics_test.go b/input/otlp/metrics_test.go
@@ -730,7 +730,8 @@ func TestConsumeMetricsExportTimestamp(t *testing.T) {
 
 func TestConsumeMetricsDataStream(t *testing.T) {
 	randomString := strings.Repeat("abcdefghijklmnopqrstuvwxyz0123456789", 10)
-	maxLen := otlp.MaxDataStreamBytes - len(otlp.DisallowedDataStreamRunes)
+	maxLenNamespace := otlp.MaxDataStreamBytes - len(otlp.DisallowedNamespaceRunes)
+	maxLenDataset := otlp.MaxDataStreamBytes - len(otlp.DisallowedDatasetRunes)
 
 	for _, tc := range []struct {
 		resourceDataStreamDataset   string
@@ -771,10 +772,10 @@ func TestConsumeMetricsDataStream(t *testing.T) {
 		// 1. Replace all disallowed runes with _
 		// 2. Datastream length should not exceed otlp.MaxDataStreamBytes
 		{
-			resourceDataStreamDataset:   otlp.DisallowedDataStreamRunes + randomString,
-			resourceDataStreamNamespace: otlp.DisallowedDataStreamRunes + randomString,
-			expectedDataStreamDataset:   strings.Repeat("_", len(otlp.DisallowedDataStreamRunes)) + randomString[:maxLen],
-			expectedDataStreamNamespace: strings.Repeat("_", len(otlp.DisallowedDataStreamRunes)) + randomString[:maxLen],
+			resourceDataStreamDataset:   otlp.DisallowedDatasetRunes + randomString,
+			resourceDataStreamNamespace: otlp.DisallowedNamespaceRunes + randomString,
+			expectedDataStreamDataset:   strings.Repeat("_", len(otlp.DisallowedDatasetRunes)) + randomString[:maxLenDataset],
+			expectedDataStreamNamespace: strings.Repeat("_", len(otlp.DisallowedNamespaceRunes)) + randomString[:maxLenNamespace],
 		},
 	} {
 		tcName := fmt.Sprintf("%s,%s", tc.expectedDataStreamDataset, tc.expectedDataStreamNamespace)

diff --git a/input/otlp/traces.go b/input/otlp/traces.go
@@ -481,12 +481,12 @@ func TranslateTransaction(
 				if event.DataStream == nil {
 					event.DataStream = &modelpb.DataStream{}
 				}
-				event.DataStream.Dataset = sanitizeDataStreamField(stringval)
+				event.DataStream.Dataset = sanitizeDataStreamDataset(stringval)
 			case attributeDataStreamNamespace:
 				if event.DataStream == nil {
 					event.DataStream = &modelpb.DataStream{}
 				}
-				event.DataStream.Namespace = sanitizeDataStreamField(stringval)
+				event.DataStream.Namespace = sanitizeDataStreamNamespace(stringval)
 			default:
 				modelpb.Labels(event.Labels).Set(k, stringval)
 			}
@@ -824,12 +824,12 @@ func TranslateSpan(spanKind ptrace.SpanKind, attributes pcommon.Map, event *mode
 				if event.DataStream == nil {
 					event.DataStream = &modelpb.DataStream{}
 				}
-				event.DataStream.Dataset = sanitizeDataStreamField(stringval)
+				event.DataStream.Dataset = sanitizeDataStreamDataset(stringval)
 			case attributeDataStreamNamespace:
 				if event.DataStream == nil {
 					event.DataStream = &modelpb.DataStream{}
 				}
-				event.DataStream.Namespace = sanitizeDataStreamField(stringval)
+				event.DataStream.Namespace = sanitizeDataStreamNamespace(stringval)
 			default:
 				setLabel(k, event, v)
 			}
@@ -1103,12 +1103,12 @@ func (c *Consumer) convertSpanEvent(
 				if event.DataStream == nil {
 					event.DataStream = &modelpb.DataStream{}
 				}
-				event.DataStream.Dataset = sanitizeDataStreamField(v.Str())
+				event.DataStream.Dataset = sanitizeDataStreamDataset(v.Str())
 			case attributeDataStreamNamespace:
 				if event.DataStream == nil {
 					event.DataStream = &modelpb.DataStream{}
 				}
-				event.DataStream.Namespace = sanitizeDataStreamField(v.Str())
+				event.DataStream.Namespace = sanitizeDataStreamNamespace(v.Str())
 
 			default:
 				setLabel(replaceDots(k), event, v)
@@ -1144,12 +1144,12 @@ func (c *Consumer) convertSpanEvent(
 				if event.DataStream == nil {
 					event.DataStream = &modelpb.DataStream{}
 				}
-				event.DataStream.Dataset = sanitizeDataStreamField(v.Str())
+				event.DataStream.Dataset = sanitizeDataStreamDataset(v.Str())
 			case attributeDataStreamNamespace:
 				if event.DataStream == nil {
 					event.DataStream = &modelpb.DataStream{}
 				}
-				event.DataStream.Namespace = sanitizeDataStreamField(v.Str())
+				event.DataStream.Namespace = sanitizeDataStreamNamespace(v.Str())
 			default:
 				k = replaceDots(k)
 				if isJaeger && k == "message" {
@@ -1202,12 +1202,12 @@ func (c *Consumer) convertJaegerErrorSpanEvent(event ptrace.SpanEvent, apmEvent
 			if apmEvent.DataStream == nil {
 				apmEvent.DataStream = &modelpb.DataStream{}
 			}
-			apmEvent.DataStream.Dataset = sanitizeDataStreamField(v.Str())
+			apmEvent.DataStream.Dataset = sanitizeDataStreamDataset(v.Str())
 		case attributeDataStreamNamespace:
 			if apmEvent.DataStream == nil {
 				apmEvent.DataStream = &modelpb.DataStream{}
 			}
-			apmEvent.DataStream.Namespace = sanitizeDataStreamField(v.Str())
+			apmEvent.DataStream.Namespace = sanitizeDataStreamNamespace(v.Str())
 
 		default:
 			setLabel(replaceDots(k), apmEvent, v)

diff --git a/input/otlp/traces_test.go b/input/otlp/traces_test.go
@@ -874,7 +874,8 @@ func TestSpanNetworkAttributes(t *testing.T) {
 
 func TestSpanDataStream(t *testing.T) {
 	randomString := strings.Repeat("abcdefghijklmnopqrstuvwxyz0123456789", 10)
-	maxLen := otlp.MaxDataStreamBytes - len(otlp.DisallowedDataStreamRunes)
+	maxLenNamespace := otlp.MaxDataStreamBytes - len(otlp.DisallowedNamespaceRunes)
+	maxLenDataset := otlp.MaxDataStreamBytes - len(otlp.DisallowedDatasetRunes)
 
 	for _, tc := range []struct {
 		resourceDataStreamDataset   string
@@ -915,10 +916,10 @@ func TestSpanDataStream(t *testing.T) {
 		// 1. Replace all disallowed runes with _
 		// 2. Datastream length should not exceed otlp.MaxDataStreamBytes
 		{
-			resourceDataStreamDataset:   otlp.DisallowedDataStreamRunes + randomString,
-			resourceDataStreamNamespace: otlp.DisallowedDataStreamRunes + randomString,
-			expectedDataStreamDataset:   strings.Repeat("_", len(otlp.DisallowedDataStreamRunes)) + randomString[:maxLen],
-			expectedDataStreamNamespace: strings.Repeat("_", len(otlp.DisallowedDataStreamRunes)) + randomString[:maxLen],
+			resourceDataStreamDataset:   otlp.DisallowedDatasetRunes + randomString,
+			resourceDataStreamNamespace: otlp.DisallowedNamespaceRunes + randomString,
+			expectedDataStreamDataset:   strings.Repeat("_", len(otlp.DisallowedDatasetRunes)) + randomString[:maxLenDataset],
+			expectedDataStreamNamespace: strings.Repeat("_", len(otlp.DisallowedNamespaceRunes)) + randomString[:maxLenNamespace],
 		},
 	} {
 		for _, isTxn := range []bool{false, true} {