fix(loki.secretfilter): Fix partial masking for short secrets and support multiple allowlists per rule

CHANGELOG.md

@@ -61,6 +61,8 @@ Main (unreleased)
 
 - Update `prometheus.write.queue` library for performance increases in cpu. (@mattdurham)
 
+- Update `loki.secretfilter` to be compatible with the new `[[rules.allowlists]]` gitleaks allowlist format (@romain-gaillard)
+
 ### Bugfixes
 
 - Fixed issue with automemlimit logging bad messages and trying to access cgroup on non-linux builds (@dehaansa)
@@ -73,8 +75,11 @@ Main (unreleased)
 
 - Fixed an issue where the `otelcol.processor.interval` could not be used because the debug metrics were not set to default. (@wildum)
 
+- Fixed an issue where `loki.secretfilter` would crash if the secret was shorter than the `partial_mask` value. (@romain-gaillard)
+
 - Change the log level in the `eventlogmessage` stage of the `loki.process` component from `warn` to `debug`. (@wildum)
 
+
 ### Other changes
 
 - Change the stability of the `livedebugging` feature from "experimental" to "generally available". (@wildum)

docs/sources/reference/components/loki/loki.secretfilter.md

@@ -18,7 +18,7 @@ The detection is based on regular expression patterns, defined in the [Gitleaks
 
 {{< admonition type="caution" >}}
 Personally Identifiable Information (PII) isn't currently in scope and some secrets could remain undetected.
-This component may generate false positives.
+This component may generate false positives or redact too much.
 Don't rely solely on this component to redact sensitive information.
 {{< /admonition >}}
 
@@ -39,17 +39,21 @@ loki.secretfilter "<LABEL>" {
 Name                     | Type                 | Description                                     | Default                          | Required
 -------------------------|----------------------|-------------------------------------------------|----------------------------------|---------
 `forward_to`             | `list(LogsReceiver)` | List of receivers to send log entries to.       |                                  | yes
-`gitleaks_config`        | `string`             | Path to the custom `gitleaks.toml` file.        | Embedded Gitleaks file           | no
-`types`                  | `map(string)`        | Types of secret to look for.                    | All types                        | no
-`redact_with`            | `string`             | String to use to redact secrets.                | `<REDACTED-SECRET:$SECRET_NAME>` | no
-`include_generic`        | `bool`               | Include the generic API key rule.               | `false`                          | no
-`allowlist`              | `map(string)`        | List of regexes to allowlist matching secrets.  | `{}`                             | no
-`partial_mask`           | `number`             | Show the first N characters of the secret.      | `0`                              | no
+`gitleaks_config`        | `string`             | Path to the custom `gitleaks.toml` file.            | Embedded Gitleaks file           | no
+`types`                  | `map(string)`        | Types of secret to look for.                        | All types                        | no
+`redact_with`            | `string`             | String to use to redact secrets.                    | `<REDACTED-SECRET:$SECRET_NAME>` | no
+`include_generic`        | `bool`               | Include the generic API key rule.                   | `false`                          | no
+`allowlist`              | `map(string)`        | List of regexes to allowlist matching secrets.      | `{}`                             | no
+`partial_mask`           | `number`             | Show the first N characters (runes) of the secret.  | `0`                              | no
 
 The `gitleaks_config` argument is the path to the custom `gitleaks.toml` file.
 The Gitleaks configuration file embedded in the component is used if you don't provide the path to a custom configuration file.
 
-The `types` argument is a map of secret types to look for. The values are used as prefixes for the secret types in the Gitleaks configuration. If you don't provide this argument, all types are used.
+{{< admonition type="note" >}}
+This component does not support all the features of the Gitleaks configuration file. Currently, it only supports the regex-based rules, `secretGroup`, and allowlist regexes (`regexTarget` only supports the default value `secret`). Other features such as `keywords`, `entropy`, `paths`, and `stopwords` are not supported. The `extend` feature is also not supported, meaning that a custom configuration file must contain all the rules to use.
+{{< /admonition >}}
+
+The `types` argument is a map of secret types to look for. The values provided are used as prefixes to match rules IDs in the Gitleaks configuration (e.g. providing the type `grafana` will match the rules `grafana-api-key`, `grafana-cloud-api-token`, and `grafana-service-account-token`). If you don't provide this argument, all rules are used.
 
 {{< admonition type="note" >}}
 Configuring this argument with the secret types you want to look for is strongly recommended.
@@ -72,8 +76,10 @@ The `include_generic` argument is a boolean that includes the generic API key ru
 The `allowlist` argument is a map of regular expressions to allow matching secrets.
 A secret will not be redacted if it matches any of the regular expressions. The allowlist in the Gitleaks configuration file is also applied.
 
-The `partial_mask` argument is the number of characters to show from the beginning of the secret before the redact string is added.
+The `partial_mask` argument is the number of characters (runes) to show from the beginning of the secret before the redact string is added.
 If set to `0`, the entire secret is redacted.
+If a secret is not at least 6 characters (runes) long, it will be entirely redacted.
+For short secrets, at most half of the secret (runes) is shown.
 
 ## Blocks
 

internal/component/loki/secretfilter/secretfilter.go

@@ -98,18 +98,24 @@ type Component struct {
 type GitLeaksConfig struct {
 	AllowList struct {
 		Description string
-		Paths       []string
+		Paths       []string // Not used
 		Regexes     []string
 	}
 	Rules []struct {
 		ID          string
-		Description string
+		Description string // Not used
 		Regex       string
-		Keywords    []string
+		Keywords    []string // Not used
 		SecretGroup int
 
+		// Old format, kept for compatibility
 		Allowlist struct {
-			StopWords []string
+			StopWords []string // Not used
+			Regexes   []string
+		}
+		// New format
+		Allowlists []struct {
+			StopWords []string // Not used
 			Regexes   []string
 		}
 	}
@@ -194,16 +200,16 @@ func (c *Component) processEntry(entry loki.Entry) loki.Entry {
 
 			// Check if the secret is in the allowlist
 			var allowRule *AllowRule = nil
-			// First check the rule-specific allowlist
-			for _, a := range r.allowlist {
+			// First check the global allowlist
+			for _, a := range c.AllowList {
 				if a.Regex.MatchString(secret) {
 					allowRule = &a
 					break
 				}
 			}
-			// Then check the global allowlist
+			// Then check the rule-specific allowlists
 			if allowRule == nil {
-				for _, a := range c.AllowList {
+				for _, a := range r.allowlist {
 					if a.Regex.MatchString(secret) {
 						allowRule = &a
 						break
@@ -231,8 +237,21 @@ func (c *Component) redactLine(line string, secret string, ruleName string) stri
 		redactWith = strings.ReplaceAll(redactWith, "$SECRET_NAME", ruleName)
 		redactWith = strings.ReplaceAll(redactWith, "$SECRET_HASH", hashSecret(secret))
 	}
-	if c.args.PartialMask > 0 {
-		redactWith = secret[:c.args.PartialMask] + redactWith
+
+	// If partialMask is set, show the first N characters of the secret
+	partialMask := int(c.args.PartialMask)
+	if partialMask < 0 {
+		partialMask = 0
+	}
+	runesSecret := []rune(secret)
+	// Only do it if the secret is long enough
+	if partialMask > 0 && len(runesSecret) >= 6 {
+		// Show at most half of the secret
+		if partialMask > len(runesSecret)/2 {
+			partialMask = len(runesSecret) / 2
+		}
+		prefix := string(runesSecret[:partialMask])
+		redactWith = prefix + redactWith
 	}
 
 	line = strings.ReplaceAll(line, secret, redactWith)
@@ -306,6 +325,16 @@ func (c *Component) Update(args component.Arguments) error {
 			}
 			allowlist = append(allowlist, AllowRule{Regex: re, Source: fmt.Sprintf("rule %s", rule.ID)})
 		}
+		for _, currAllowList := range rule.Allowlists {
+			for _, r := range currAllowList.Regexes {
+				re, err := regexp.Compile(r)
+				if err != nil {
+					level.Error(c.opts.Logger).Log("msg", "error compiling allowlist regex", "error", err)
+					return err
+				}
+				allowlist = append(allowlist, AllowRule{Regex: re, Source: fmt.Sprintf("rule %s", rule.ID)})
+			}
+		}
 
 		newRule := Rule{
 			name:        rule.ID,
@@ -325,23 +354,23 @@ func (c *Component) Update(args component.Arguments) error {
 	}
 
 	// Compiling global allowlist regexes
-	// From the Gitleaks config
-	for _, r := range gitleaksCfg.AllowList.Regexes {
+	// From the arguments
+	for _, r := range c.args.AllowList {
 		re, err := regexp.Compile(r)
 		if err != nil {
 			level.Error(c.opts.Logger).Log("msg", "error compiling allowlist regex", "error", err)
 			return err
 		}
-		c.AllowList = append(c.AllowList, AllowRule{Regex: re, Source: "gitleaks config"})
+		c.AllowList = append(c.AllowList, AllowRule{Regex: re, Source: "alloy config"})
 	}
-	// From the arguments
-	for _, r := range c.args.AllowList {
+	// From the Gitleaks config
+	for _, r := range gitleaksCfg.AllowList.Regexes {
 		re, err := regexp.Compile(r)
 		if err != nil {
 			level.Error(c.opts.Logger).Log("msg", "error compiling allowlist regex", "error", err)
 			return err
 		}
-		c.AllowList = append(c.AllowList, AllowRule{Regex: re, Source: "alloy config"})
+		c.AllowList = append(c.AllowList, AllowRule{Regex: re, Source: "gitleaks config"})
 	}
 
 	// Add the generic API key rule last if needed