Configurable masking token, perf improvements, refactoring.

src/Agent.Listener/Configuration/FeatureFlagProvider.cs

@@ -25,6 +25,7 @@ public interface IFeatureFlagProvider : IAgentService
         /// <param name="featureFlagName">The name of the feature flag to get the status of.</param>
         /// <param name="traceWriter">Trace writer for output</param>
         /// <returns>The status of the feature flag.</returns>
+        /// <exception cref="VssUnauthorizedException">Thrown if token is not suitable for retrieving feature flag status</exception>
         /// <exception cref="InvalidOperationException">Thrown if agent is not configured</exception>
         public Task<FeatureFlag> GetFeatureFlagAsync(IHostContext context, string featureFlagName, ITraceWriter traceWriter, CancellationToken ctk = default);
 

src/Agent.Sdk/SecretMasking/ILoggedSecretMasker.cs

@@ -3,6 +3,8 @@
 //using Microsoft.TeamFoundation.DistributedTask.Logging;
 using ValueEncoder = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoder;
 using System;
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
 
 namespace Agent.Sdk.SecretMasking
 {
@@ -13,7 +15,7 @@ public interface ILoggedSecretMasker : ISecretMasker
     {
         static int MinSecretLengthLimit { get; }
 
-        void AddRegex(String pattern, string origin);
+        void AddRegex(String pattern, string origin, ISet<string> sniffLiterals = null, RegexOptions regexOptions = 0);
         void AddValue(String value, string origin);
         void AddValueEncoder(ValueEncoder encoder, string origin);
         void SetTrace(ITraceWriter trace);

src/Agent.Sdk/SecretMasking/ISecret.cs

@@ -8,5 +8,5 @@ internal interface ISecret
     /// <summary>
     /// Returns one item (start, length) for each match found in the input string.
     /// </summary>
-    IEnumerable<ReplacementPosition> GetPositions(String input);
+    IEnumerable<Replacement> GetPositions(String input);
 }

src/Agent.Sdk/SecretMasking/ISecretMasker.cs

@@ -3,6 +3,8 @@
 using System;
 using ValueEncoder = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoder;
 using ISecretMaskerVSO = Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker;
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
 
 namespace Agent.Sdk.SecretMasking;
 public interface ISecretMasker : ISecretMaskerVSO

src/Agent.Sdk/SecretMasking/LegacyLoggedSecretMasker.cs

@@ -1,5 +1,9 @@
 using Agent.Sdk.SecretMasking;
 using Microsoft.TeamFoundation.DistributedTask.Logging;
+
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
+
 using ISecretMasker = Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker;
 
 namespace Agent.Sdk.Util.SecretMasking;
@@ -60,7 +64,7 @@ public void AddRegex(string pattern)
     /// </summary>
     /// <param name="pattern"></param>
     /// <param name="origin"></param>
-    public void AddRegex(string pattern, string origin)
+    public void AddRegex(string pattern, string origin, ISet<string> sniffLiterals = null, RegexOptions regexOptions = 0)
     {
         this.Trace($"Setting up regex for origin: {origin}.");
         if (pattern == null)
@@ -138,4 +142,9 @@ Sdk.SecretMasking.ISecretMasker Sdk.SecretMasking.ISecretMasker.Clone()
     {
         return new LegacyLoggedSecretMasker(this._secretMasker.Clone());
     }
+
+    public void AddRegex(string pattern, ISet<string> sniffLiterals, RegexOptions regexOptions, string origin)
+    {
+        throw new System.NotImplementedException();
+    }
 }

src/Agent.Sdk/SecretMasking/LoggedSecretMasker.cs

@@ -4,38 +4,28 @@
 using System;
 using ValueEncoder = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoder;
 using ISecretMaskerVSO = Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker;
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
 
 namespace Agent.Sdk.SecretMasking
 {
     /// <summary>
     /// Extended secret masker service, that allows to log origins of secrets
     /// </summary>
-    public class LoggedSecretMasker : ILoggedSecretMasker
+    public class LoggedSecretMasker : SecretMasker, ILoggedSecretMasker
     {
-        private ISecretMasker _secretMasker;
         private ITraceWriter _trace;
 
-
         private void Trace(string msg)
         {
             this._trace?.Info(msg);
         }
 
-        public LoggedSecretMasker(ISecretMasker secretMasker)
-        {
-            this._secretMasker = secretMasker;
-        }
-
         public void SetTrace(ITraceWriter trace)
         {
             this._trace = trace;
         }
 
-        public void AddValue(string pattern)
-        {
-            this._secretMasker.AddValue(pattern);
-        }
-
         /// <summary>
         /// Overloading of AddValue method with additional logic for logging origin of provided secret
         /// </summary>
@@ -52,17 +42,13 @@ public void AddValue(string value, string origin)
 
             AddValue(value);
         }
-        public void AddRegex(string pattern)
-        {
-            this._secretMasker.AddRegex(pattern);
-        }
 
         /// <summary>
         /// Overloading of AddRegex method with additional logic for logging origin of provided secret
         /// </summary>
         /// <param name="pattern"></param>
         /// <param name="origin"></param>
-        public void AddRegex(string pattern, string origin)
+        public void AddRegex(string pattern, string origin, ISet<string> sniffLiterals, RegexOptions regexOptions)
         {
             this.Trace($"Setting up regex for origin: {origin}.");
             if (pattern == null)
@@ -71,44 +57,38 @@ public void AddRegex(string pattern, string origin)
                 return;
             }
 
-            AddRegex(pattern);
+            AddRegex(pattern, sniffLiterals,regexOptions);
         }
 
         // We don't allow to skip secrets longer than 5 characters.
         // Note: the secret that will be ignored is of length n-1.
         public static int MinSecretLengthLimit => 6;
 
-        public int MinSecretLength
+        public override int MinSecretLength
         {
             get
             {
-                return _secretMasker.MinSecretLength;
+                return base.MinSecretLength;
             }
             set
             {
                 if (value > MinSecretLengthLimit)
                 {
-                    _secretMasker.MinSecretLength = MinSecretLengthLimit;
+                    base.MinSecretLength = MinSecretLengthLimit;
                 }
                 else
                 {
-                    _secretMasker.MinSecretLength = value;
+                    base.MinSecretLength = value;
                 }
             }
         }
 
         public void RemoveShortSecretsFromDictionary()
         {
             this._trace?.Info("Removing short secrets from masking dictionary");
-            _secretMasker.RemoveShortSecretsFromDictionary();
+            base.RemoveShortSecretsFromDictionary();
         }
 
-        public void AddValueEncoder(ValueEncoder encoder)
-        {
-            this._secretMasker.AddValueEncoder(encoder);
-        }
-
-
         /// <summary>
         /// Overloading of AddValueEncoder method with additional logic for logging origin of provided secret
         /// </summary>
@@ -127,16 +107,6 @@ public void AddValueEncoder(ValueEncoder encoder, string origin)
             AddValueEncoder(encoder);
         }
 
-        public ISecretMasker Clone()
-        {
-            return new LoggedSecretMasker(this._secretMasker.Clone());
-        }
-
-        public string MaskSecrets(string input)
-        {
-            return this._secretMasker.MaskSecrets(input);
-        }
-
         ISecretMaskerVSO ISecretMaskerVSO.Clone() => this.Clone();
     }
 }

src/Agent.Sdk/SecretMasking/PatternDescriptor.cs

@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
+
+namespace Microsoft.VisualStudio.Services.Agent
+{
+    public class PatternDescriptor
+    {
+        public string Regex { get; set; }
+
+        public ISet<string> SniffLiterals { get; set; }
+
+        public RegexOptions RegexOptions { get; set; }
+    }
+}

src/Agent.Sdk/SecretMasking/RegexSecret.cs

@@ -1,4 +1,7 @@
-using System;
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
 using System.Collections.Generic;
 using System.Text.RegularExpressions;
 using Microsoft.VisualStudio.Services.Agent.Util;
@@ -7,11 +10,16 @@ namespace Agent.Sdk.SecretMasking;
 
 internal sealed class RegexSecret : ISecret
 {
-    public RegexSecret(String pattern)
+    public RegexSecret(String pattern,
+                       ISet<string> sniffLiterals = null,
+                       RegexOptions regexOptions = RegexOptions.Compiled | RegexOptions.ExplicitCapture)
     {
         ArgUtil.NotNullOrEmpty(pattern, nameof(pattern));
+
         m_pattern = pattern;
-        m_regex = new Regex(pattern, _regexOptions);
+        m_regexOptions = regexOptions;
+        m_sniffLiterals = sniffLiterals;
+        m_regex = new Regex(pattern, regexOptions);
     }
 
     public override Boolean Equals(Object obj)
@@ -21,31 +29,97 @@ public override Boolean Equals(Object obj)
         {
             return false;
         }
-        return String.Equals(m_pattern, item.m_pattern, StringComparison.Ordinal);
+
+        if (!String.Equals(m_pattern, item.m_pattern, StringComparison.Ordinal) ||
+            m_regexOptions != item.m_regexOptions)
+        {
+            return false;
+        }
+
+        if (m_sniffLiterals == null && item.m_sniffLiterals == null)
+        {
+            return true;
+        }
+
+        m_sniffLiterals.Equals(item.m_sniffLiterals);
+
+        if (m_sniffLiterals.Count !=  item.m_sniffLiterals.Count)
+        {
+            return false; 
+        }
+
+        foreach(string sniffLiteral in m_sniffLiterals) 
+        { 
+            if (!item.m_sniffLiterals.Contains(sniffLiteral))
+            {
+                return false;
+            }
+        }
+
+        return true;
     }
 
-    public override int GetHashCode() => m_pattern.GetHashCode();
+    public override int GetHashCode()
+    {
+        int result = 17;
+        unchecked
+        {
+            result = (result * 31) + m_pattern.GetHashCode();
+            result = (result * 31) + m_regexOptions.GetHashCode();
 
-    public IEnumerable<ReplacementPosition> GetPositions(String input)
+            // Use xor for set values to be order-independent.
+            if (m_sniffLiterals != null)
+            {
+                int xor_0 = 0;
+                foreach (var sniffLiteral in m_sniffLiterals)
+                {
+                    xor_0 ^= sniffLiteral.GetHashCode();
+                }
+                result = (result * 31) + xor_0;
+            }
+        }
+
+        return result;
+    }
+
+    public IEnumerable<Replacement> GetPositions(String input)
     {
-        Int32 startIndex = 0;
-        while (startIndex < input.Length)
+        bool runRegexes = m_sniffLiterals == null ? true : false;
+
+        if (m_sniffLiterals != null)
         {
-            var match = m_regex.Match(input, startIndex);
-            if (match.Success)
+            foreach (string sniffLiteral in m_sniffLiterals)
             {
-                startIndex = match.Index + 1;
-                yield return new ReplacementPosition(match.Index, match.Length);
+                if (input.IndexOf(sniffLiteral, StringComparison.Ordinal) != -1)
+                {
+                    runRegexes = true;
+                    break;
+                }
             }
-            else
+        }
+
+        if (runRegexes)
+        {
+            Int32 startIndex = 0;
+            while (startIndex < input.Length)
             {
-                yield break;
+                var match = m_regex.Match(input, startIndex);
+                if (match.Success)
+                {
+                    startIndex = match.Index + 1;
+                    yield return new Replacement(match.Index, match.Length, "+++");
+                }
+                else
+                {
+                    yield break;
+                }
             }
         }
     }
 
     public string Pattern { get { return m_pattern; } }
+    private readonly ISet<string> m_sniffLiterals;
+    private readonly RegexOptions m_regexOptions;
     private readonly String m_pattern;
     private readonly Regex m_regex;
-    private static readonly RegexOptions _regexOptions = RegexOptions.Compiled | RegexOptions.ExplicitCapture;
 }

src/Agent.Sdk/SecretMasking/ReplacementPosition.cs → src/Agent.Sdk/SecretMasking/Replacement.cs

@@ -2,20 +2,22 @@
 
 namespace Agent.Sdk.SecretMasking;
 
-internal sealed class ReplacementPosition
+internal sealed class Replacement
 {
-    public ReplacementPosition(Int32 start, Int32 length)
+    public Replacement(Int32 start, Int32 length, string token = "***")
     {
         Start = start;
         Length = length;
+        Token = token;
     }
 
-    public ReplacementPosition(ReplacementPosition copy)
+    public Replacement(Replacement copy)
     {
+        Token = copy.Token;
         Start = copy.Start;
         Length = copy.Length;
     }
-
+    public string Token { get; private set; }
     public Int32 Start { get; set; }
     public Int32 Length { get; set; }
     public Int32 End