Configurable masking token, perf improvements, refactoring.

src/Agent.Sdk/SecretMasking/ILoggedSecretMasker.cs

@@ -3,6 +3,8 @@
 //using Microsoft.TeamFoundation.DistributedTask.Logging;
 using ValueEncoder = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoder;
 using System;
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
 
 namespace Agent.Sdk.SecretMasking
 {
@@ -13,9 +15,10 @@ public interface ILoggedSecretMasker : ISecretMasker
     {
         static int MinSecretLengthLimit { get; }
 
-        void AddRegex(String pattern, string origin);
+        void AddRegex(String pattern, string origin, string moniker = null, ISet<string> sniffLiterals = null, RegexOptions regexOptions = 0);
         void AddValue(String value, string origin);
         void AddValueEncoder(ValueEncoder encoder, string origin);
         void SetTrace(ITraceWriter trace);
+        IDictionary<string, string> GetTelemetry();
     }
 }

src/Agent.Sdk/SecretMasking/ISecret.cs

@@ -8,5 +8,5 @@ internal interface ISecret
     /// <summary>
     /// Returns one item (start, length) for each match found in the input string.
     /// </summary>
-    IEnumerable<ReplacementPosition> GetPositions(String input);
+    IEnumerable<Replacement> GetReplacements(String input);
 }

src/Agent.Sdk/SecretMasking/LegacyLoggedSecretMasker.cs

@@ -1,5 +1,12 @@
 using Agent.Sdk.SecretMasking;
 using Microsoft.TeamFoundation.DistributedTask.Logging;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Text.RegularExpressions;
+using System.Threading;
+
 using ISecretMasker = Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker;
 
 namespace Agent.Sdk.Util.SecretMasking;
@@ -11,6 +18,7 @@ namespace Agent.Sdk.Util.SecretMasking;
 public class LegacyLoggedSecretMasker : ILoggedSecretMasker
 {
     private ISecretMasker _secretMasker;
+    private double elapsedMaskingTime;
     private ITraceWriter _trace;
 
     private void Trace(string msg)
@@ -60,7 +68,7 @@ public void AddRegex(string pattern)
     /// </summary>
     /// <param name="pattern"></param>
     /// <param name="origin"></param>
-    public void AddRegex(string pattern, string origin)
+    public void AddRegex(string pattern, string origin, string moniker = null, ISet<string> sniffLiterals = null, RegexOptions regexOptions = 0)
     {
         this.Trace($"Setting up regex for origin: {origin}.");
         if (pattern == null)
@@ -129,13 +137,29 @@ public ISecretMasker Clone()
         return new LegacyLoggedSecretMasker(this._secretMasker.Clone());
     }
 
+
+    public double ElapsedMaskingTime => this.elapsedMaskingTime;
+
     public string MaskSecrets(string input)
     {
-        return this._secretMasker.MaskSecrets(input);
+        var stopwatch = Stopwatch.StartNew();
+        string result = this._secretMasker.MaskSecrets(input);
+        Interlocked.Exchange(ref elapsedMaskingTime, stopwatch.ElapsedTicks);
+        return result;
     }
 
     Sdk.SecretMasking.ISecretMasker Sdk.SecretMasking.ISecretMasker.Clone()
     {
         return new LegacyLoggedSecretMasker(this._secretMasker.Clone());
     }
+
+    public IDictionary<string, string> GetTelemetry()
+    {
+        var result = new Dictionary<string, string>
+        {
+            { nameof(ElapsedMaskingTime), elapsedMaskingTime.ToString() },
+        };
+
+        return result;
+    }
 }

src/Agent.Sdk/SecretMasking/LoggedSecretMasker.cs

@@ -4,38 +4,28 @@
 using System;
 using ValueEncoder = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoder;
 using ISecretMaskerVSO = Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker;
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
 
 namespace Agent.Sdk.SecretMasking
 {
     /// <summary>
     /// Extended secret masker service, that allows to log origins of secrets
     /// </summary>
-    public class LoggedSecretMasker : ILoggedSecretMasker
+    public class LoggedSecretMasker : SecretMasker, ILoggedSecretMasker
     {
-        private ISecretMasker _secretMasker;
         private ITraceWriter _trace;
 
-
         private void Trace(string msg)
         {
             this._trace?.Info(msg);
         }
 
-        public LoggedSecretMasker(ISecretMasker secretMasker)
-        {
-            this._secretMasker = secretMasker;
-        }
-
         public void SetTrace(ITraceWriter trace)
         {
             this._trace = trace;
         }
 
-        public void AddValue(string pattern)
-        {
-            this._secretMasker.AddValue(pattern);
-        }
-
         /// <summary>
         /// Overloading of AddValue method with additional logic for logging origin of provided secret
         /// </summary>
@@ -52,17 +42,13 @@ public void AddValue(string value, string origin)
 
             AddValue(value);
         }
-        public void AddRegex(string pattern)
-        {
-            this._secretMasker.AddRegex(pattern);
-        }
 
         /// <summary>
         /// Overloading of AddRegex method with additional logic for logging origin of provided secret
         /// </summary>
         /// <param name="pattern"></param>
         /// <param name="origin"></param>
-        public void AddRegex(string pattern, string origin)
+        public void AddRegex(string pattern, string origin, string moniker, ISet<string> sniffLiterals, RegexOptions regexOptions)
         {
             this.Trace($"Setting up regex for origin: {origin}.");
             if (pattern == null)
@@ -71,44 +57,38 @@ public void AddRegex(string pattern, string origin)
                 return;
             }
 
-            AddRegex(pattern);
+            AddRegex(pattern, moniker, sniffLiterals, regexOptions);
         }
 
         // We don't allow to skip secrets longer than 5 characters.
         // Note: the secret that will be ignored is of length n-1.
         public static int MinSecretLengthLimit => 6;
 
-        public int MinSecretLength
+        public override int MinSecretLength
         {
             get
             {
-                return _secretMasker.MinSecretLength;
+                return base.MinSecretLength;
             }
             set
             {
                 if (value > MinSecretLengthLimit)
                 {
-                    _secretMasker.MinSecretLength = MinSecretLengthLimit;
+                    base.MinSecretLength = MinSecretLengthLimit;
                 }
                 else
                 {
-                    _secretMasker.MinSecretLength = value;
+                    base.MinSecretLength = value;
                 }
             }
         }
 
         public void RemoveShortSecretsFromDictionary()
         {
             this._trace?.Info("Removing short secrets from masking dictionary");
-            _secretMasker.RemoveShortSecretsFromDictionary();
-        }
-
-        public void AddValueEncoder(ValueEncoder encoder)
-        {
-            this._secretMasker.AddValueEncoder(encoder);
+            base.RemoveShortSecretsFromDictionary();
         }
 
-
         /// <summary>
         /// Overloading of AddValueEncoder method with additional logic for logging origin of provided secret
         /// </summary>
@@ -127,16 +107,17 @@ public void AddValueEncoder(ValueEncoder encoder, string origin)
             AddValueEncoder(encoder);
         }
 
-        public ISecretMasker Clone()
-        {
-            return new LoggedSecretMasker(this._secretMasker.Clone());
-        }
+        ISecretMaskerVSO ISecretMaskerVSO.Clone() => this.Clone();
 
-        public string MaskSecrets(string input)
+        public IDictionary<string, string> GetTelemetry()
         {
-            return this._secretMasker.MaskSecrets(input);
-        }
+            var result = new Dictionary<string, string>
+            {
+                { nameof(ElapsedMaskingTime), ElapsedMaskingTime.ToString() },
+            };
 
-        ISecretMaskerVSO ISecretMaskerVSO.Clone() => this.Clone();
+            result["Redactions"] = string.Join(',', ReplacementTokens);
+            return result;
+        }
     }
 }

src/Agent.Sdk/SecretMasking/PatternDescriptor.cs

@@ -0,0 +1,20 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
+
+namespace Microsoft.VisualStudio.Services.Agent
+{
+    public class PatternDescriptor
+    {
+        public string Regex { get; set; }
+
+        public string Moniker { get; set; }
+
+        public ISet<string> SniffLiterals { get; set; }
+
+        public RegexOptions RegexOptions { get; set; }
+    }
+}