diff --git a/src/Agent.Sdk/SecretMasking/ILoggedSecretMasker.cs b/src/Agent.Sdk/SecretMasking/ILoggedSecretMasker.cs index 0847b8259c..bc5d7e4d85 100644 --- a/src/Agent.Sdk/SecretMasking/ILoggedSecretMasker.cs +++ b/src/Agent.Sdk/SecretMasking/ILoggedSecretMasker.cs @@ -3,6 +3,8 @@ //using Microsoft.TeamFoundation.DistributedTask.Logging; using ValueEncoder = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoder; using System; +using System.Collections.Generic; +using System.Text.RegularExpressions; namespace Agent.Sdk.SecretMasking { @@ -13,9 +15,10 @@ public interface ILoggedSecretMasker : ISecretMasker { static int MinSecretLengthLimit { get; } - void AddRegex(String pattern, string origin); + void AddRegex(String pattern, string origin, string moniker = null, ISet sniffLiterals = null, RegexOptions regexOptions = 0); void AddValue(String value, string origin); void AddValueEncoder(ValueEncoder encoder, string origin); void SetTrace(ITraceWriter trace); + IDictionary GetTelemetry(); } } diff --git a/src/Agent.Sdk/SecretMasking/ISecret.cs b/src/Agent.Sdk/SecretMasking/ISecret.cs index b42cf2c900..deb5658f70 100644 --- a/src/Agent.Sdk/SecretMasking/ISecret.cs +++ b/src/Agent.Sdk/SecretMasking/ISecret.cs @@ -8,5 +8,5 @@ internal interface ISecret /// /// Returns one item (start, length) for each match found in the input string. /// - IEnumerable GetPositions(String input); + IEnumerable GetReplacements(String input); } \ No newline at end of file diff --git a/src/Agent.Sdk/SecretMasking/LegacyLoggedSecretMasker.cs b/src/Agent.Sdk/SecretMasking/LegacyLoggedSecretMasker.cs index eb1fbb43c4..2c3fc39b8f 100644 --- a/src/Agent.Sdk/SecretMasking/LegacyLoggedSecretMasker.cs +++ b/src/Agent.Sdk/SecretMasking/LegacyLoggedSecretMasker.cs @@ -1,5 +1,12 @@ using Agent.Sdk.SecretMasking; using Microsoft.TeamFoundation.DistributedTask.Logging; + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text.RegularExpressions; +using System.Threading; + using ISecretMasker = Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker; namespace Agent.Sdk.Util.SecretMasking; @@ -11,6 +18,7 @@ namespace Agent.Sdk.Util.SecretMasking; public class LegacyLoggedSecretMasker : ILoggedSecretMasker { private ISecretMasker _secretMasker; + private double elapsedMaskingTime; private ITraceWriter _trace; private void Trace(string msg) @@ -60,7 +68,7 @@ public void AddRegex(string pattern) /// /// /// - public void AddRegex(string pattern, string origin) + public void AddRegex(string pattern, string origin, string moniker = null, ISet sniffLiterals = null, RegexOptions regexOptions = 0) { this.Trace($"Setting up regex for origin: {origin}."); if (pattern == null) @@ -129,13 +137,29 @@ public ISecretMasker Clone() return new LegacyLoggedSecretMasker(this._secretMasker.Clone()); } + + public double ElapsedMaskingTime => this.elapsedMaskingTime; + public string MaskSecrets(string input) { - return this._secretMasker.MaskSecrets(input); + var stopwatch = Stopwatch.StartNew(); + string result = this._secretMasker.MaskSecrets(input); + Interlocked.Exchange(ref elapsedMaskingTime, stopwatch.ElapsedTicks); + return result; } Sdk.SecretMasking.ISecretMasker Sdk.SecretMasking.ISecretMasker.Clone() { return new LegacyLoggedSecretMasker(this._secretMasker.Clone()); } + + public IDictionary GetTelemetry() + { + var result = new Dictionary + { + { nameof(ElapsedMaskingTime), elapsedMaskingTime.ToString() }, + }; + + return result; + } } \ No newline at end of file diff --git a/src/Agent.Sdk/SecretMasking/LoggedSecretMasker.cs b/src/Agent.Sdk/SecretMasking/LoggedSecretMasker.cs index 9b3c21da3b..bbbeff4370 100644 --- a/src/Agent.Sdk/SecretMasking/LoggedSecretMasker.cs +++ b/src/Agent.Sdk/SecretMasking/LoggedSecretMasker.cs @@ -4,38 +4,28 @@ using System; using ValueEncoder = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoder; using ISecretMaskerVSO = Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker; +using System.Collections.Generic; +using System.Text.RegularExpressions; namespace Agent.Sdk.SecretMasking { /// /// Extended secret masker service, that allows to log origins of secrets /// - public class LoggedSecretMasker : ILoggedSecretMasker + public class LoggedSecretMasker : SecretMasker, ILoggedSecretMasker { - private ISecretMasker _secretMasker; private ITraceWriter _trace; - private void Trace(string msg) { this._trace?.Info(msg); } - public LoggedSecretMasker(ISecretMasker secretMasker) - { - this._secretMasker = secretMasker; - } - public void SetTrace(ITraceWriter trace) { this._trace = trace; } - public void AddValue(string pattern) - { - this._secretMasker.AddValue(pattern); - } - /// /// Overloading of AddValue method with additional logic for logging origin of provided secret /// @@ -52,17 +42,13 @@ public void AddValue(string value, string origin) AddValue(value); } - public void AddRegex(string pattern) - { - this._secretMasker.AddRegex(pattern); - } /// /// Overloading of AddRegex method with additional logic for logging origin of provided secret /// /// /// - public void AddRegex(string pattern, string origin) + public void AddRegex(string pattern, string origin, string moniker, ISet sniffLiterals, RegexOptions regexOptions) { this.Trace($"Setting up regex for origin: {origin}."); if (pattern == null) @@ -71,28 +57,28 @@ public void AddRegex(string pattern, string origin) return; } - AddRegex(pattern); + AddRegex(pattern, moniker, sniffLiterals, regexOptions); } // We don't allow to skip secrets longer than 5 characters. // Note: the secret that will be ignored is of length n-1. public static int MinSecretLengthLimit => 6; - public int MinSecretLength + public override int MinSecretLength { get { - return _secretMasker.MinSecretLength; + return base.MinSecretLength; } set { if (value > MinSecretLengthLimit) { - _secretMasker.MinSecretLength = MinSecretLengthLimit; + base.MinSecretLength = MinSecretLengthLimit; } else { - _secretMasker.MinSecretLength = value; + base.MinSecretLength = value; } } } @@ -100,15 +86,9 @@ public int MinSecretLength public void RemoveShortSecretsFromDictionary() { this._trace?.Info("Removing short secrets from masking dictionary"); - _secretMasker.RemoveShortSecretsFromDictionary(); - } - - public void AddValueEncoder(ValueEncoder encoder) - { - this._secretMasker.AddValueEncoder(encoder); + base.RemoveShortSecretsFromDictionary(); } - /// /// Overloading of AddValueEncoder method with additional logic for logging origin of provided secret /// @@ -127,16 +107,17 @@ public void AddValueEncoder(ValueEncoder encoder, string origin) AddValueEncoder(encoder); } - public ISecretMasker Clone() - { - return new LoggedSecretMasker(this._secretMasker.Clone()); - } + ISecretMaskerVSO ISecretMaskerVSO.Clone() => this.Clone(); - public string MaskSecrets(string input) + public IDictionary GetTelemetry() { - return this._secretMasker.MaskSecrets(input); - } + var result = new Dictionary + { + { nameof(ElapsedMaskingTime), ElapsedMaskingTime.ToString() }, + }; - ISecretMaskerVSO ISecretMaskerVSO.Clone() => this.Clone(); + result["Redactions"] = string.Join(',', ReplacementTokens); + return result; + } } } diff --git a/src/Agent.Sdk/SecretMasking/PatternDescriptor.cs b/src/Agent.Sdk/SecretMasking/PatternDescriptor.cs new file mode 100644 index 0000000000..b1f7cf9a28 --- /dev/null +++ b/src/Agent.Sdk/SecretMasking/PatternDescriptor.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + + +using System.Collections.Generic; +using System.Text.RegularExpressions; + +namespace Microsoft.VisualStudio.Services.Agent +{ + public class PatternDescriptor + { + public string Regex { get; set; } + + public string Moniker { get; set; } + + public ISet SniffLiterals { get; set; } + + public RegexOptions RegexOptions { get; set; } + } +} \ No newline at end of file diff --git a/src/Agent.Sdk/SecretMasking/RegexSecret.cs b/src/Agent.Sdk/SecretMasking/RegexSecret.cs index ef924e746a..10f82fbb69 100644 --- a/src/Agent.Sdk/SecretMasking/RegexSecret.cs +++ b/src/Agent.Sdk/SecretMasking/RegexSecret.cs @@ -1,5 +1,10 @@ -using System; +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; using System.Collections.Generic; +using System.Security.Cryptography; +using System.Text; using System.Text.RegularExpressions; using Microsoft.VisualStudio.Services.Agent.Util; @@ -7,11 +12,18 @@ namespace Agent.Sdk.SecretMasking; internal sealed class RegexSecret : ISecret { - public RegexSecret(String pattern) + public RegexSecret(String pattern, + string moniker = null, + ISet sniffLiterals = null, + RegexOptions regexOptions = RegexOptions.Compiled | RegexOptions.ExplicitCapture) { ArgUtil.NotNullOrEmpty(pattern, nameof(pattern)); - m_pattern = pattern; - m_regex = new Regex(pattern, _regexOptions); + + Pattern = pattern; + Moniker = moniker; + m_regexOptions = regexOptions; + m_sniffLiterals = sniffLiterals; + m_regex = new Regex(pattern, regexOptions); } public override Boolean Equals(Object obj) @@ -21,31 +33,126 @@ public override Boolean Equals(Object obj) { return false; } - return String.Equals(m_pattern, item.m_pattern, StringComparison.Ordinal); + + if (!String.Equals(Pattern, item.Pattern, StringComparison.Ordinal) || + !String.Equals(Moniker, item.Moniker, StringComparison.Ordinal) || + m_regexOptions != item.m_regexOptions) + { + return false; + } + + if (m_sniffLiterals == null && item.m_sniffLiterals == null) + { + return true; + } + + if (m_sniffLiterals.Count != item.m_sniffLiterals.Count) + { + return false; + } + + foreach (string sniffLiteral in m_sniffLiterals) + { + if (!item.m_sniffLiterals.Contains(sniffLiteral)) + { + return false; + } + } + + return true; } - public override int GetHashCode() => m_pattern.GetHashCode(); + public override int GetHashCode() + { + int result = 17; + unchecked + { + result = (result * 31) + Pattern.GetHashCode(); + + if (Moniker != null) + { + result = (result * 31) + Moniker.GetHashCode(); + } - public IEnumerable GetPositions(String input) + result = (result * 31) + m_regexOptions.GetHashCode(); + + // Use xor for set values to be order-independent. + if (m_sniffLiterals != null) + { + int xor_0 = 0; + foreach (var sniffLiteral in m_sniffLiterals) + { + xor_0 ^= sniffLiteral.GetHashCode(); + } + result = (result * 31) + xor_0; + } + } + + return result; + } + + public IEnumerable GetReplacements(String input) { - Int32 startIndex = 0; - while (startIndex < input.Length) + bool runRegexes = m_sniffLiterals == null ? true : false; + + if (m_sniffLiterals != null) { - var match = m_regex.Match(input, startIndex); - if (match.Success) + foreach (string sniffLiteral in m_sniffLiterals) { - startIndex = match.Index + 1; - yield return new ReplacementPosition(match.Index, match.Length); + if (input.IndexOf(sniffLiteral, StringComparison.Ordinal) != -1) + { + runRegexes = true; + break; + } } - else + } + + if (runRegexes) + { + Int32 startIndex = 0; + while (startIndex < input.Length) { - yield break; + var match = m_regex.Match(input, startIndex); + if (match.Success) + { + startIndex = match.Index + 1; + string token = Moniker != null + ? CreateTelemetryForMatch(Moniker, match.Value) + : "+++"; + yield return new Replacement(match.Index, match.Length, token); + } + else + { + yield break; + } } } } - public string Pattern { get { return m_pattern; } } - private readonly String m_pattern; + private string CreateTelemetryForMatch(string moniker, string value) + { + using var sha = SHA256.Create(); + byte[] byteHash = Encoding.UTF8.GetBytes(value); + byte[] checksum = sha.ComputeHash(byteHash); + + string hashedValue = BitConverter.ToString(checksum).Replace("-", string.Empty, StringComparison.Ordinal); + + return $"{moniker}:{hashedValue}"; + } + + internal static string HashString(string value) + { + byte[] byteHash = Encoding.UTF8.GetBytes(value); + byte[] checksum = s_sha256.ComputeHash(byteHash); + return BitConverter.ToString(checksum).Replace("-", string.Empty, StringComparison.Ordinal); + } + + public string Pattern { get; private set; } + public string Moniker { get; private set; } + + private readonly ISet m_sniffLiterals; + private readonly RegexOptions m_regexOptions; private readonly Regex m_regex; - private static readonly RegexOptions _regexOptions = RegexOptions.Compiled | RegexOptions.ExplicitCapture; + + private static readonly SHA256 s_sha256 = SHA256.Create(); } \ No newline at end of file diff --git a/src/Agent.Sdk/SecretMasking/ReplacementPosition.cs b/src/Agent.Sdk/SecretMasking/Replacement.cs similarity index 59% rename from src/Agent.Sdk/SecretMasking/ReplacementPosition.cs rename to src/Agent.Sdk/SecretMasking/Replacement.cs index 8cb90e3c0f..2ddc7cb802 100644 --- a/src/Agent.Sdk/SecretMasking/ReplacementPosition.cs +++ b/src/Agent.Sdk/SecretMasking/Replacement.cs @@ -2,20 +2,22 @@ namespace Agent.Sdk.SecretMasking; -internal sealed class ReplacementPosition +internal sealed class Replacement { - public ReplacementPosition(Int32 start, Int32 length) + public Replacement(Int32 start, Int32 length, String token = "***") { Start = start; Length = length; + Token = token; } - public ReplacementPosition(ReplacementPosition copy) + public Replacement(Replacement copy) { + Token = copy.Token; Start = copy.Start; Length = copy.Length; } - + public String Token { get; private set; } public Int32 Start { get; set; } public Int32 Length { get; set; } public Int32 End diff --git a/src/Agent.Sdk/SecretMasking/SecretMasker.cs b/src/Agent.Sdk/SecretMasking/SecretMasker.cs index fe763ee4f6..082c202fc2 100644 --- a/src/Agent.Sdk/SecretMasking/SecretMasker.cs +++ b/src/Agent.Sdk/SecretMasking/SecretMasker.cs @@ -8,387 +8,78 @@ using ValueEncoder = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoder; using ISecretMaskerVSO = Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker; using System.Text.RegularExpressions; +using System.Diagnostics; namespace Agent.Sdk.SecretMasking; -public sealed class SecretMasker : ISecretMasker, IDisposable - { - - public SecretMasker() - { - MinSecretLength = 0; - m_originalValueSecrets = new HashSet(); - m_regexSecrets = new HashSet(); - m_valueEncoders = new HashSet(); - m_valueSecrets = new HashSet(); - } - - public SecretMasker(int minSecretLength) - { - MinSecretLength = minSecretLength; - m_originalValueSecrets = new HashSet(); - m_regexSecrets = new HashSet(); - m_valueEncoders = new HashSet(); - m_valueSecrets = new HashSet(); - } - - private SecretMasker(SecretMasker copy) - { - // Read section. - try - { - copy.m_lock.EnterReadLock(); - - // Copy the hash sets. - MinSecretLength = copy.MinSecretLength; - m_originalValueSecrets = new HashSet(copy.m_originalValueSecrets); - m_regexSecrets = new HashSet(copy.m_regexSecrets); - m_valueEncoders = new HashSet(copy.m_valueEncoders); - m_valueSecrets = new HashSet(copy.m_valueSecrets); - } - finally - { - if (copy.m_lock.IsReadLockHeld) - { - copy.m_lock.ExitReadLock(); - } - } - } - - /// - /// This property allows to set the minimum length of a secret for masking - /// - public int MinSecretLength { get; set; } +public class SecretMasker : ISecretMasker, IDisposable +{ + public SecretMasker() + { + MinSecretLength = 0; + m_originalValueSecrets = new HashSet(); + m_regexSecrets = new HashSet(); + m_valueEncoders = new HashSet(); + m_valueSecrets = new HashSet(); + ReplacementTokens = new HashSet(); + } + + public SecretMasker(int minSecretLength) + { + MinSecretLength = minSecretLength; + m_originalValueSecrets = new HashSet(); + m_regexSecrets = new HashSet(); + m_valueEncoders = new HashSet(); + m_valueSecrets = new HashSet(); + ReplacementTokens = new HashSet(); + } + + private SecretMasker(SecretMasker copy) + { + // Read section. + try + { + copy.m_lock.EnterReadLock(); + + // Copy the hash sets. + MinSecretLength = copy.MinSecretLength; + m_originalValueSecrets = new HashSet(copy.m_originalValueSecrets); + m_regexSecrets = new HashSet(copy.m_regexSecrets); + m_valueEncoders = new HashSet(copy.m_valueEncoders); + m_valueSecrets = new HashSet(copy.m_valueSecrets); + ReplacementTokens = copy.ReplacementTokens; + } + finally + { + if (copy.m_lock.IsReadLockHeld) + { + copy.m_lock.ExitReadLock(); + } + } + } /// - /// This implementation assumes no more than one thread is adding regexes, values, or encoders at any given time. + /// Total time in ticks spent masking content for the lifetime of this masker instance. + /// + public long ElapsedMaskingTime { get; private set; } + + public ISet ReplacementTokens { get; private set; } + + /// + /// This property allows to set the minimum length of a secret for masking /// + virtual public int MinSecretLength { get; set; } + public void AddRegex(String pattern) - { - // Test for empty. - if (String.IsNullOrEmpty(pattern)) - { - return; - } - - if (pattern.Length < MinSecretLength) - { - return; - } - - // Write section. - try - { - m_lock.EnterWriteLock(); - - // Add the value. - m_regexSecrets.Add(new RegexSecret(pattern)); - } - finally - { - if (m_lock.IsWriteLockHeld) - { - m_lock.ExitWriteLock(); - } - } - } - - - /// - /// This implementation assumes no more than one thread is adding regexes, values, or encoders at any given time. - /// - public void AddValue(String value) - { - // Test for empty. - if (String.IsNullOrEmpty(value)) - { - return; - } - - if (value.Length < MinSecretLength) - { - return; - } - - var valueSecrets = new List(new[] { new ValueSecret(value) }); - - // Read section. - ValueEncoder[] valueEncoders; - try - { - m_lock.EnterReadLock(); - - // Test whether already added. - if (m_originalValueSecrets.Contains(valueSecrets[0])) - { - return; - } - - // Read the value encoders. - valueEncoders = m_valueEncoders.ToArray(); - } - finally - { - if (m_lock.IsReadLockHeld) - { - m_lock.ExitReadLock(); - } - } - - // Compute the encoded values. - foreach (ValueEncoder valueEncoder in valueEncoders) - { - String encodedValue = valueEncoder(value); - if (!String.IsNullOrEmpty(encodedValue) && encodedValue.Length >= MinSecretLength) - { - valueSecrets.Add(new ValueSecret(encodedValue)); - } - } - - // Write section. - try - { - m_lock.EnterWriteLock(); - - // Add the values. - m_originalValueSecrets.Add(valueSecrets[0]); - foreach (ValueSecret valueSecret in valueSecrets) - { - m_valueSecrets.Add(valueSecret); - } - } - finally - { - if (m_lock.IsWriteLockHeld) - { - m_lock.ExitWriteLock(); - } - } - } - - /// - /// This implementation assumes no more than one thread is adding regexes, values, or encoders at any given time. - /// - public void AddValueEncoder(ValueEncoder encoder) - { - ValueSecret[] originalSecrets; - - // Read section. - try - { - m_lock.EnterReadLock(); - - // Test whether already added. - if (m_valueEncoders.Contains(encoder)) - { - return; - } - - // Read the original value secrets. - originalSecrets = m_originalValueSecrets.ToArray(); - } - finally - { - if (m_lock.IsReadLockHeld) - { - m_lock.ExitReadLock(); - } - } - - // Compute the encoded values. - var encodedSecrets = new List(); - foreach (ValueSecret originalSecret in originalSecrets) - { - String encodedValue = encoder(originalSecret.m_value); - if (!String.IsNullOrEmpty(encodedValue) && encodedValue.Length >= MinSecretLength) - { - encodedSecrets.Add(new ValueSecret(encodedValue)); - } - } - - // Write section. - try - { - m_lock.EnterWriteLock(); - - // Add the encoder. - m_valueEncoders.Add(encoder); - - // Add the values. - foreach (ValueSecret encodedSecret in encodedSecrets) - { - m_valueSecrets.Add(encodedSecret); - } - } - finally - { - if (m_lock.IsWriteLockHeld) - { - m_lock.ExitWriteLock(); - } - } - } - - - public ISecretMasker Clone() => new SecretMasker(this); - - public void Dispose() - { - m_lock?.Dispose(); - m_lock = null; - } - - public String MaskSecrets(String input) - { - if (String.IsNullOrEmpty(input)) - { - return String.Empty; - } - - var secretPositions = new List(); - - // Read section. - try - { - m_lock.EnterReadLock(); - - // Get indexes and lengths of all substrings that will be replaced. - foreach (RegexSecret regexSecret in m_regexSecrets) - { - secretPositions.AddRange(regexSecret.GetPositions(input)); - } - - foreach (ValueSecret valueSecret in m_valueSecrets) - { - secretPositions.AddRange(valueSecret.GetPositions(input)); - } - } - finally - { - if (m_lock.IsReadLockHeld) - { - m_lock.ExitReadLock(); - } - } - - // Short-circuit if nothing to replace. - if (secretPositions.Count == 0) - { - return input; - } - - // Merge positions into ranges of characters to replace. - List replacementPositions = new List(); - ReplacementPosition currentReplacement = null; - foreach (ReplacementPosition secretPosition in secretPositions.OrderBy(x => x.Start)) - { - if (currentReplacement == null) - { - currentReplacement = new ReplacementPosition(copy: secretPosition); - replacementPositions.Add(currentReplacement); - } - else - { - if (secretPosition.Start <= currentReplacement.End) - { - // Overlap - currentReplacement.Length = Math.Max(currentReplacement.End, secretPosition.End) - currentReplacement.Start; - } - else - { - // No overlap - currentReplacement = new ReplacementPosition(copy: secretPosition); - replacementPositions.Add(currentReplacement); - } - } - } - - // Replace - var stringBuilder = new StringBuilder(); - Int32 startIndex = 0; - foreach (var replacement in replacementPositions) - { - stringBuilder.Append(input.Substring(startIndex, replacement.Start - startIndex)); - stringBuilder.Append("***"); - startIndex = replacement.Start + replacement.Length; - } - - if (startIndex < input.Length) - { - stringBuilder.Append(input.Substring(startIndex)); - } - - return stringBuilder.ToString(); - } - - /// - /// Removes secrets from the dictionary shorter than the MinSecretLength property. - /// This implementation assumes no more than one thread is adding regexes, values, or encoders at any given time. - /// - public void RemoveShortSecretsFromDictionary() - { - var filteredValueSecrets = new HashSet(); - var filteredRegexSecrets = new HashSet(); - - try - { - m_lock.EnterReadLock(); - - foreach (var secret in m_valueSecrets) - { - if (secret.m_value.Length < MinSecretLength) - { - filteredValueSecrets.Add(secret); - } - } - - foreach (var secret in m_regexSecrets) - { - if (secret.Pattern.Length < MinSecretLength) - { - filteredRegexSecrets.Add(secret); - } - } - } - finally - { - if (m_lock.IsReadLockHeld) - { - m_lock.ExitReadLock(); - } - } - - try - { - m_lock.EnterWriteLock(); - - foreach (var secret in filteredValueSecrets) - { - m_valueSecrets.Remove(secret); - } - - foreach (var secret in filteredRegexSecrets) - { - m_regexSecrets.Remove(secret); - } - - foreach (var secret in filteredValueSecrets) - { - m_originalValueSecrets.Remove(secret); - } - } - finally - { - if (m_lock.IsWriteLockHeld) - { - m_lock.ExitWriteLock(); - } - } - } - - public void AddRegex(string pattern, RegexOptions options) { - RegexSecret regexSecret = new RegexSecret(pattern); + AddRegex(pattern, moniker: null, sniffLiterals: null, RegexOptions.Compiled | RegexOptions.ExplicitCapture); + } + + /// + /// This implementation assumes no more than one thread is adding regexes, values, or encoders at any given time. + /// + public void AddRegex(String pattern, string moniker, ISet sniffLiterals, RegexOptions regexOptions) + { // Test for empty. if (String.IsNullOrEmpty(pattern)) { @@ -406,7 +97,319 @@ public void AddRegex(string pattern, RegexOptions options) m_lock.EnterWriteLock(); // Add the value. - m_regexSecrets.Add(regexSecret); + m_regexSecrets.Add(new RegexSecret(pattern, moniker, sniffLiterals, regexOptions)); + } + finally + { + if (m_lock.IsWriteLockHeld) + { + m_lock.ExitWriteLock(); + } + } + } + + + /// + /// This implementation assumes no more than one thread is adding regexes, values, or encoders at any given time. + /// + public void AddValue(String value) + { + // Test for empty. + if (String.IsNullOrEmpty(value)) + { + return; + } + + if (value.Length < MinSecretLength) + { + return; + } + + var valueSecrets = new List(new[] { new ValueSecret(value) }); + + // Read section. + ValueEncoder[] valueEncoders; + try + { + m_lock.EnterReadLock(); + + // Test whether already added. + if (m_originalValueSecrets.Contains(valueSecrets[0])) + { + return; + } + + // Read the value encoders. + valueEncoders = m_valueEncoders.ToArray(); + } + finally + { + if (m_lock.IsReadLockHeld) + { + m_lock.ExitReadLock(); + } + } + + // Compute the encoded values. + foreach (ValueEncoder valueEncoder in valueEncoders) + { + String encodedValue = valueEncoder(value); + if (!String.IsNullOrEmpty(encodedValue) && encodedValue.Length >= MinSecretLength) + { + valueSecrets.Add(new ValueSecret(encodedValue)); + } + } + + // Write section. + try + { + m_lock.EnterWriteLock(); + + // Add the values. + m_originalValueSecrets.Add(valueSecrets[0]); + foreach (ValueSecret valueSecret in valueSecrets) + { + m_valueSecrets.Add(valueSecret); + } + } + finally + { + if (m_lock.IsWriteLockHeld) + { + m_lock.ExitWriteLock(); + } + } + } + + /// + /// This implementation assumes no more than one thread is adding regexes, values, or encoders at any given time. + /// + public void AddValueEncoder(ValueEncoder encoder) + { + ValueSecret[] originalSecrets; + + // Read section. + try + { + m_lock.EnterReadLock(); + + // Test whether already added. + if (m_valueEncoders.Contains(encoder)) + { + return; + } + + // Read the original value secrets. + originalSecrets = m_originalValueSecrets.ToArray(); + } + finally + { + if (m_lock.IsReadLockHeld) + { + m_lock.ExitReadLock(); + } + } + + // Compute the encoded values. + var encodedSecrets = new List(); + foreach (ValueSecret originalSecret in originalSecrets) + { + String encodedValue = encoder(originalSecret.m_value); + if (!String.IsNullOrEmpty(encodedValue) && encodedValue.Length >= MinSecretLength) + { + encodedSecrets.Add(new ValueSecret(encodedValue)); + } + } + + // Write section. + try + { + m_lock.EnterWriteLock(); + + // Add the encoder. + m_valueEncoders.Add(encoder); + + // Add the values. + foreach (ValueSecret encodedSecret in encodedSecrets) + { + m_valueSecrets.Add(encodedSecret); + } + } + finally + { + if (m_lock.IsWriteLockHeld) + { + m_lock.ExitWriteLock(); + } + } + } + + + public ISecretMasker Clone() => new SecretMasker(this); + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + m_lock?.Dispose(); + m_lock = null; + } + } + + public String MaskSecrets(String input) + { + if (String.IsNullOrEmpty(input)) + { + return String.Empty; + } + + var secretPositions = new List(); + + // Read section. + try + { + m_lock.EnterReadLock(); + + var stopwatch = Stopwatch.StartNew(); + + // Get indexes and lengths of all substrings that will be replaced. + // We also persist the generated replacement tokens for secrets. In + // some cases, this data will be put in the telemetry stream. + foreach (RegexSecret regexSecret in m_regexSecrets) + { + foreach (Replacement replacement in regexSecret.GetReplacements(input)) + { + ReplacementTokens.Add(replacement.Token); + secretPositions.Add(replacement); + } + } + + foreach (ValueSecret valueSecret in m_valueSecrets) + { + secretPositions.AddRange(valueSecret.GetPositions(input)); + } + + ElapsedMaskingTime += stopwatch.ElapsedTicks; + } + finally + { + if (m_lock.IsReadLockHeld) + { + m_lock.ExitReadLock(); + } + } + + // Short-circuit if nothing to replace. + if (secretPositions.Count == 0) + { + return input; + } + + // Merge positions into ranges of characters to replace. + List replacementPositions = new List(); + Replacement currentReplacement = null; + foreach (Replacement secretPosition in secretPositions.OrderBy(x => x.Start)) + { + if (currentReplacement == null) + { + currentReplacement = new Replacement(copy: secretPosition); + replacementPositions.Add(currentReplacement); + } + else + { + if (secretPosition.Start <= currentReplacement.End) + { + // Overlap + currentReplacement.Length = Math.Max(currentReplacement.End, secretPosition.End) - currentReplacement.Start; + } + else + { + // No overlap + currentReplacement = new Replacement(copy: secretPosition); + replacementPositions.Add(currentReplacement); + } + } + } + + // Replace + var stringBuilder = new StringBuilder(); + Int32 startIndex = 0; + foreach (var replacement in replacementPositions) + { + stringBuilder.Append(input.Substring(startIndex, replacement.Start - startIndex)); + stringBuilder.Append(replacement.Token); + startIndex = replacement.Start + replacement.Length; + } + + if (startIndex < input.Length) + { + stringBuilder.Append(input.Substring(startIndex)); + } + + return stringBuilder.ToString(); + } + + /// + /// Removes secrets from the dictionary shorter than the MinSecretLength property. + /// This implementation assumes no more than one thread is adding regexes, values, or encoders at any given time. + /// + public virtual void RemoveShortSecretsFromDictionary() + { + var filteredValueSecrets = new HashSet(); + var filteredRegexSecrets = new HashSet(); + + try + { + m_lock.EnterReadLock(); + + foreach (var secret in m_valueSecrets) + { + if (secret.m_value.Length < MinSecretLength) + { + filteredValueSecrets.Add(secret); + } + } + + foreach (var secret in m_regexSecrets) + { + if (secret.Pattern.Length < MinSecretLength) + { + filteredRegexSecrets.Add(secret); + } + } + } + finally + { + if (m_lock.IsReadLockHeld) + { + m_lock.ExitReadLock(); + } + } + + try + { + m_lock.EnterWriteLock(); + + foreach (var secret in filteredValueSecrets) + { + m_valueSecrets.Remove(secret); + } + + foreach (var secret in filteredRegexSecrets) + { + m_regexSecrets.Remove(secret); + } + + foreach (var secret in filteredValueSecrets) + { + m_originalValueSecrets.Remove(secret); + } } finally { @@ -423,8 +426,8 @@ ISecretMaskerVSO ISecretMaskerVSO.Clone() } private readonly HashSet m_originalValueSecrets; - private readonly HashSet m_regexSecrets; private readonly HashSet m_valueEncoders; private readonly HashSet m_valueSecrets; + private readonly HashSet m_regexSecrets; private ReaderWriterLockSlim m_lock = new ReaderWriterLockSlim(LockRecursionPolicy.NoRecursion); - } \ No newline at end of file +} \ No newline at end of file diff --git a/src/Agent.Sdk/SecretMasking/ValueSecret.cs b/src/Agent.Sdk/SecretMasking/ValueSecret.cs index 5a9e6089a0..7adab65795 100644 --- a/src/Agent.Sdk/SecretMasking/ValueSecret.cs +++ b/src/Agent.Sdk/SecretMasking/ValueSecret.cs @@ -24,7 +24,7 @@ public override Boolean Equals(Object obj) public override Int32 GetHashCode() => m_value.GetHashCode(); - public IEnumerable GetPositions(String input) + public IEnumerable GetPositions(String input) { if (!String.IsNullOrEmpty(input) && !String.IsNullOrEmpty(m_value)) { @@ -36,7 +36,7 @@ public IEnumerable GetPositions(String input) startIndex = input.IndexOf(m_value, startIndex, StringComparison.Ordinal); if (startIndex > -1) { - yield return new ReplacementPosition(startIndex, m_value.Length); + yield return new Replacement(startIndex, m_value.Length); ++startIndex; } } diff --git a/src/Agent.Sdk/Util/WellKnownSecretAliases.cs b/src/Agent.Sdk/Util/WellKnownSecretAliases.cs index 7530449f27..376afaf5cf 100644 --- a/src/Agent.Sdk/Util/WellKnownSecretAliases.cs +++ b/src/Agent.Sdk/Util/WellKnownSecretAliases.cs @@ -28,7 +28,7 @@ public static class WellKnownSecretAliases // Secret regex aliases public static readonly string UrlSecretPattern = "RegexUrlSecretPattern"; - public static readonly string CredScanPatterns = "RegexCredScanPatterns"; + public static readonly string OneESScanPatterns = "Regex1ESScanPatterns"; // Value encoder aliases public static readonly string JsonStringEscape = "ValueEncoderJsonStringEscape"; diff --git a/src/Agent.Worker/JobRunner.cs b/src/Agent.Worker/JobRunner.cs index d363a5f377..a50027e812 100644 --- a/src/Agent.Worker/JobRunner.cs +++ b/src/Agent.Worker/JobRunner.cs @@ -20,6 +20,7 @@ using Newtonsoft.Json.Linq; using Newtonsoft.Json; using Microsoft.VisualStudio.Services.Agent.Worker.Telemetry; +using Agent.Sdk.SecretMasking; namespace Microsoft.VisualStudio.Services.Agent.Worker { @@ -347,6 +348,9 @@ public async Task RunAsync(Pipelines.AgentJobRequestMessage message, finally { Trace.Info("Finalize job."); + + PublishSecretsMaskingTelemetry(jobContext); + await jobExtension.FinalizeJob(jobContext); } @@ -635,5 +639,29 @@ private void PublishTelemetry(IExecutionContext context, string Task_Result, str Trace.Warning($"Unable to publish agent shutdown telemetry data. Exception: {ex}"); } } + private void PublishSecretsMaskingTelemetry(IExecutionContext context) + { + try + { + var masker = HostContext.SecretMasker as LoggedSecretMasker; + if (masker == null) { return; } + + var telemetryData = masker.GetTelemetry(); + telemetryData["JobId"] = context.Variables.System_JobId.ToString(); + + var cmd = new Command("telemetry", "publish"); + cmd.Data = JsonConvert.SerializeObject(telemetryData, Formatting.None); + cmd.Properties.Add("area", "PipelinesTasks"); + cmd.Properties.Add("feature", "SecretsMasking"); + + var publishTelemetryCmd = new TelemetryCommandExtension(); + publishTelemetryCmd.Initialize(HostContext); + publishTelemetryCmd.ProcessCommand(context, cmd); + } + catch (Exception ex) + { + Trace.Warning($"Unable to publish secrets masking telemetry data. Exception: {ex}"); + } + } } } diff --git a/src/Agent.Worker/Release/ContainerFetchEngine/FetchEngine.cs b/src/Agent.Worker/Release/ContainerFetchEngine/FetchEngine.cs index 16c61d9669..7841a1bc53 100644 --- a/src/Agent.Worker/Release/ContainerFetchEngine/FetchEngine.cs +++ b/src/Agent.Worker/Release/ContainerFetchEngine/FetchEngine.cs @@ -383,7 +383,7 @@ private string ConvertToLocalPath(ContainerItem item) return FileSystemManager.JoinPath(RootDestinationDir, localRelativePath); } - public IConatinerFetchEngineLogger ExecutionLogger { get; set; } + public IContainerFetchEngineLogger ExecutionLogger { get; set; } private string RootDestinationDir { get; } private string RootItemPath { get; } diff --git a/src/Agent.Worker/Release/ContainerFetchEngine/HttpRetryOnTimeoutHandler.cs b/src/Agent.Worker/Release/ContainerFetchEngine/HttpRetryOnTimeoutHandler.cs index df2b2fb715..6d1aa3fe04 100644 --- a/src/Agent.Worker/Release/ContainerFetchEngine/HttpRetryOnTimeoutHandler.cs +++ b/src/Agent.Worker/Release/ContainerFetchEngine/HttpRetryOnTimeoutHandler.cs @@ -19,9 +19,9 @@ namespace Microsoft.VisualStudio.Services.Agent.Worker.Release.ContainerFetchEng public class HttpRetryOnTimeoutMessageHandler : DelegatingHandler { private readonly HttpRetryOnTimeoutOptions _retryOptions; - private readonly IConatinerFetchEngineLogger _logger; + private readonly IContainerFetchEngineLogger _logger; - public HttpRetryOnTimeoutMessageHandler(HttpRetryOnTimeoutOptions retryOptions, IConatinerFetchEngineLogger logger) + public HttpRetryOnTimeoutMessageHandler(HttpRetryOnTimeoutOptions retryOptions, IContainerFetchEngineLogger logger) { _retryOptions = retryOptions; _logger = logger; diff --git a/src/Agent.Worker/Release/ContainerFetchEngine/IConatinerFetchEngineLogger.cs b/src/Agent.Worker/Release/ContainerFetchEngine/IContainerFetchEngineLogger.cs similarity index 89% rename from src/Agent.Worker/Release/ContainerFetchEngine/IConatinerFetchEngineLogger.cs rename to src/Agent.Worker/Release/ContainerFetchEngine/IContainerFetchEngineLogger.cs index 17bd9f3136..95a7dc0d27 100644 --- a/src/Agent.Worker/Release/ContainerFetchEngine/IConatinerFetchEngineLogger.cs +++ b/src/Agent.Worker/Release/ContainerFetchEngine/IContainerFetchEngineLogger.cs @@ -5,7 +5,7 @@ namespace Microsoft.VisualStudio.Services.Agent.Worker.Release.ContainerFetchEng { [ServiceLocator(Default = typeof(NullExecutionLogger))] // NOTE: FetchEngine specific interface shouldn't take dependency on Agent code. - public interface IConatinerFetchEngineLogger + public interface IContainerFetchEngineLogger { void Warning(string message); void Output(string message); diff --git a/src/Agent.Worker/Release/ContainerFetchEngine/NullExecutionLogger.cs b/src/Agent.Worker/Release/ContainerFetchEngine/NullExecutionLogger.cs index 4619717b3f..f15d16d6d3 100644 --- a/src/Agent.Worker/Release/ContainerFetchEngine/NullExecutionLogger.cs +++ b/src/Agent.Worker/Release/ContainerFetchEngine/NullExecutionLogger.cs @@ -3,7 +3,7 @@ namespace Microsoft.VisualStudio.Services.Agent.Worker.Release.ContainerFetchEngine { - public class NullExecutionLogger : IConatinerFetchEngineLogger + public class NullExecutionLogger : IContainerFetchEngineLogger { public void Warning(string message) { diff --git a/src/Agent.Worker/Release/ContainerProvider/Helpers/ExecutionLogger.cs b/src/Agent.Worker/Release/ContainerProvider/Helpers/ExecutionLogger.cs index 785a3a68d9..72f5469ab2 100644 --- a/src/Agent.Worker/Release/ContainerProvider/Helpers/ExecutionLogger.cs +++ b/src/Agent.Worker/Release/ContainerProvider/Helpers/ExecutionLogger.cs @@ -5,7 +5,7 @@ namespace Microsoft.VisualStudio.Services.Agent.Worker.Release.ContainerProvider.Helpers { - public class ExecutionLogger : IConatinerFetchEngineLogger + public class ExecutionLogger : IContainerFetchEngineLogger { private readonly IExecutionContext _executionContext; diff --git a/src/Agent.Worker/TaskCommandExtension.cs b/src/Agent.Worker/TaskCommandExtension.cs index 8411b02513..1ccaf3ae6e 100644 --- a/src/Agent.Worker/TaskCommandExtension.cs +++ b/src/Agent.Worker/TaskCommandExtension.cs @@ -44,7 +44,7 @@ public static void AddSecret(IExecutionContext context, string value, string ori { context.GetHostContext().SecretMasker.AddValue(value, origin); - // if DECODE_PERCENTS = false then we need to add decoded value as a secret as well to prevent its exposion in logs + // if DECODE_PERCENTS = false then we need to add decoded value as a secret as well to prevent its exposure in logs var unescapePercents = AgentKnobs.DecodePercents.GetValue(context).AsBoolean(); if (!unescapePercents) { diff --git a/src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.1ES.cs b/src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.1ES.cs new file mode 100644 index 0000000000..f32fee6bcf --- /dev/null +++ b/src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.1ES.cs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Generic; + +namespace Microsoft.VisualStudio.Services.Agent +{ + public static partial class AdditionalMaskingPatterns + { + public static IEnumerable OneESPatterns => oneESPatterns; + + private static IEnumerable oneESPatterns = + new List() + { + // AAD client app, most recent two versions. + new PatternDescriptor + { + Regex = @"(\b[0-9A-Za-z-_~.]{3}7Q~[0-9A-Za-z-_~.]{31}(\b|$))|(\b[0-9A-Za-z-_~.]{3}8Q~[0-9A-Za-z-_~.]{34}(\b|$))", + SniffLiterals = new HashSet(new[]{ "7Q~", "8Q~"}), + Moniker = "SEC101/156.AadClientAppSecret", + }, + + // Prominent Azure provider 256-bit symmetric keys. + new PatternDescriptor + { + Regex = @"\b[0-9A-Za-z+/]{33}(AIoT|\+(ASb|AEh|ARm))[A-P][0-9A-Za-z+/]{5}=(\b|$)", + SniffLiterals = new HashSet(new[]{ "AIoT", "+ASb", "+AEh", "+ARm" }), + Moniker = "SEC102/101.Unclassified32ByteBase64String", + }, + + // Prominent Azure provider 512-bit symmetric keys. + new PatternDescriptor + { + Regex = @"(\b|$)[0-9A-Za-z+/]{76}(APIM|ACDb|\+(ABa|AMC|ASt))[0-9A-Za-z+/]{5}[AQgw]==((\b|$)|$)", + SniffLiterals = new HashSet(new[]{ "APIM", "ACDb", "+ABa", "+AMC", "+ASt", }), + Moniker = "SEC102/102.Unclassified64ByteBase64String", + }, + + // Azure Function key. + new PatternDescriptor + { + Regex = @"\b[0-9A-Za-z_\-]{44}AzFu[0-9A-Za-z\-_]{5}[AQgw]==(\b|$)", + SniffLiterals = new HashSet(new[]{ "AzFu" }), + Moniker = "SEC101/158.AzureFunctionIdentifiableKey", + }, + + // Azure Search keys. + new PatternDescriptor + { + Regex = @"\b[0-9A-Za-z]{42}AzSe[A-D][0-9A-Za-z]{5}(\b|$)", + SniffLiterals = new HashSet(new[]{ "AzSe" }), + Moniker = "SEC101/167.AzureSearchIdentifiableKey", + }, + + // Azure Container Registry keys. + new PatternDescriptor + { + Regex = @"\b[0-9A-Za-z+/]{42}\+ACR[A-D][0-9A-Za-z+/]{5}(\b|$)", + SniffLiterals = new HashSet(new[]{ "+ACR" }), + Moniker = "SEC101/176.AzureContainerRegistryIdentifiableKey", + }, + + // Azure Cache for Redis keys. + new PatternDescriptor + { + Regex = @"\b[0-9A-Za-z]{33}AzCa[A-P][0-9A-Za-z]{5}=(\b|$)", + SniffLiterals = new HashSet(new[]{ "AzCa" }), + Moniker = "SEC101/154.AzureCacheForRedisIdentifiableKey" + }, + + // NuGet API keys. + new PatternDescriptor + { + Regex = @"\boy2[a-p][0-9a-z]{15}[aq][0-9a-z]{11}[eu][bdfhjlnprtvxz357][a-p][0-9a-z]{11}[aeimquy4](\b|$)", + SniffLiterals = new HashSet(new[]{ "oy2" }), + Moniker = "SEC101/031.NuGetApiKey" + }, + + // NPM author keys. + new PatternDescriptor + { + Regex = @"\bnpm_[0-9A-Za-z]{36}(\b|$)", + SniffLiterals = new HashSet(new[]{ "npm_" }), + Moniker = "SEC101/050.NpmAuthorKey" + }, + }; + } +} \ No newline at end of file diff --git a/src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.CredScan.cs b/src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.CredScan.cs deleted file mode 100644 index e786fa36a6..0000000000 --- a/src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.CredScan.cs +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. -// -// THIS FILE IS GENERATED CODE. -// DO NOT EDIT. -// YOUR EDITS MAY BE LOST. -// -// Generated by tools/CredScanRegexes/CredScanRegexes.csproj -// from CredScan version 1.3.30-client-012060001 - -using System.Collections.Generic; - -namespace Microsoft.VisualStudio.Services.Agent -{ - public static partial class AdditionalMaskingRegexes - { - public static IEnumerable CredScanPatterns => credScanPatterns; - - // Each pattern or set of patterns has a comment mentioning - // which CredScan policy it came from. In CredScan, if a pattern - // contains a named group, then that named group is considered the - // sensitive part. - // - // For the agent, we don't want to mask the non-sensitive parts, so - // we wrap lookbehind and lookahead non-match groups around those - // parts which aren't in the named group. - // - // The non-matching parts are pulled out into separate string - // literals to make them easier to manually examine. - private static IEnumerable credScanPatterns = - new List() - { - // AAD client app, most recent two versions. - @"\b" // pre-match - + @"[0-9A-Za-z-_~.]{3}7Q~[0-9A-Za-z-_~.]{31}\b|\b[0-9A-Za-z-_~.]{3}8Q~[0-9A-Za-z-_~.]{34}" // match - + @"\b", // post-match - - // Prominent Azure provider 512-bit symmetric keys. - @"\b" // pre-match - + @"[0-9A-Za-z+/]{76}(APIM|ACDb|\+(ABa|AMC|ASt))[0-9A-Za-z+/]{5}[AQgw]==" // match - + @"", // post-match - // - // Prominent Azure provider 256-bit symmetric keys. - @"\b" // pre-match - + @"[0-9A-Za-z+/]{33}(AIoT|\+(ASb|AEh|ARm))[A-P][0-9A-Za-z+/]{5}=" // match - + @"", // post-match - - // Azure Function key. - @"\b" // pre-match - + @"[0-9A-Za-z_\-]{44}AzFu[0-9A-Za-z\-_]{5}[AQgw]==" // match - + @"", // post-match - - // Azure Search keys. - @"\b" // pre-match - + @"[0-9A-Za-z]{42}AzSe[A-D][0-9A-Za-z]{5}" // match - + @"\b", // post-match - - // Azure Container Registry keys. - @"\b" // pre-match - + @"[0-9A-Za-z+/]{42}\+ACR[A-D][0-9A-Za-z+/]{5}" // match - + @"\b", // post-match - - // Azure Cache for Redis keys. - @"\b" // pre-match - + @"[0-9A-Za-z]{33}AzCa[A-P][0-9A-Za-z]{5}=" // match - + @"", // post-match - - // NuGet API keys. - @"\b" // pre-match - + @"oy2[a-p][0-9a-z]{15}[aq][0-9a-z]{11}[eu][bdfhjlnprtvxz357][a-p][0-9a-z]{11}[aeimquy4]" // match - + @"\b", // post-match - - // NPM author keys. - @"\b" // pre-match - + @"npm_[0-9A-Za-z]{36}" // match - + @"\b", // post-match - }; - } -} diff --git a/src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.cs b/src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.cs index b5a99ed339..9f500d7fd4 100644 --- a/src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.cs +++ b/src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.cs @@ -3,7 +3,7 @@ namespace Microsoft.VisualStudio.Services.Agent { - public static partial class AdditionalMaskingRegexes + public static partial class AdditionalMaskingPatterns { /// /// Regexp for unreserved characters - for more details see https://datatracker.ietf.org/doc/html/rfc3986#section-2.3 diff --git a/src/Microsoft.VisualStudio.Services.Agent/HostContext.cs b/src/Microsoft.VisualStudio.Services.Agent/HostContext.cs index 2cb0cacde0..8f3764a29e 100644 --- a/src/Microsoft.VisualStudio.Services.Agent/HostContext.cs +++ b/src/Microsoft.VisualStudio.Services.Agent/HostContext.cs @@ -22,7 +22,6 @@ using Pipelines = Microsoft.TeamFoundation.DistributedTask.Pipelines; using Agent.Sdk.Util; using Microsoft.TeamFoundation.DistributedTask.Logging; -using SecretMasker = Agent.Sdk.SecretMasking.SecretMasker; using LegacySecretMasker = Microsoft.TeamFoundation.DistributedTask.Logging.SecretMasker; using Agent.Sdk.Util.SecretMasking; @@ -72,7 +71,6 @@ public class HostContext : EventListener, IObserver, IObserv private static int[] _vssHttpCredentialEventIds = new int[] { 11, 13, 14, 15, 16, 17, 18, 20, 21, 22, 27, 29 }; private readonly ConcurrentDictionary _serviceInstances = new ConcurrentDictionary(); protected readonly ConcurrentDictionary ServiceTypes = new ConcurrentDictionary(); - private ILoggedSecretMasker _secretMasker; private readonly ProductInfoHeaderValue _userAgent = new ProductInfoHeaderValue($"VstsAgentCore-{BuildConstants.AgentPackage.PackageName}", BuildConstants.AgentPackage.Version); private CancellationTokenSource _agentShutdownTokenSource = new CancellationTokenSource(); private object _perfLock = new object(); @@ -83,21 +81,27 @@ public class HostContext : EventListener, IObserver, IObserv private AssemblyLoadContext _loadContext; private IDisposable _httpTraceSubscription; private IDisposable _diagListenerSubscription; - private LegacySecretMasker _legacySecretMasker = new LegacySecretMasker(); - private SecretMasker _newSecretMasker = new SecretMasker(); + private IDisposable _disposableSecretMasker; + private ILoggedSecretMasker _loggedSecretMasker; private StartupType _startupType; private string _perfFile; private HostType _hostType; public event EventHandler Unloading; public CancellationToken AgentShutdownToken => _agentShutdownTokenSource.Token; public ShutdownReason AgentShutdownReason { get; private set; } - public ILoggedSecretMasker SecretMasker => _secretMasker; + public ILoggedSecretMasker SecretMasker => _loggedSecretMasker; public ProductInfoHeaderValue UserAgent => _userAgent; public HostContext(HostType hostType, string logFile = null) { - var useNewSecretMasker = AgentKnobs.EnableNewSecretMasker.GetValue(this).AsBoolean(); - _secretMasker = useNewSecretMasker ? new LoggedSecretMasker(_newSecretMasker) : new LegacyLoggedSecretMasker(_legacySecretMasker); + var useNewSecretMasker = AgentKnobs.EnableNewSecretMasker.GetValue(this).AsBoolean(); + + _disposableSecretMasker = useNewSecretMasker ? new LoggedSecretMasker() : new LegacySecretMasker(); + + _loggedSecretMasker = useNewSecretMasker + ? (LoggedSecretMasker)_disposableSecretMasker + : new LegacyLoggedSecretMasker((LegacySecretMasker)_disposableSecretMasker); + // Validate args. if (hostType == HostType.Undefined) { @@ -111,7 +115,9 @@ public HostContext(HostType hostType, string logFile = null) this.SecretMasker.AddValueEncoder(ValueEncoders.JsonStringEscape, $"HostContext_{WellKnownSecretAliases.JsonStringEscape}"); this.SecretMasker.AddValueEncoder(ValueEncoders.UriDataEscape, $"HostContext_{WellKnownSecretAliases.UriDataEscape}"); this.SecretMasker.AddValueEncoder(ValueEncoders.BackslashEscape, $"HostContext_{WellKnownSecretAliases.UriDataEscape}"); - this.SecretMasker.AddRegex(AdditionalMaskingRegexes.UrlSecretPattern, $"HostContext_{WellKnownSecretAliases.UrlSecretPattern}"); + + this.SecretMasker.AddRegex(AdditionalMaskingPatterns.UrlSecretPattern, + $"HostContext_{WellKnownSecretAliases.UrlSecretPattern}"); // Create the trace manager. if (string.IsNullOrEmpty(logFile)) @@ -130,7 +136,7 @@ public HostContext(HostType hostType, string logFile = null) logRetentionDays = _defaultLogRetentionDays; } - // this should give us _diag folder under agent root directory as default value for diagLogDirctory + // this should give us _diag folder under agent root directory as default value for diagLogDirectory string diagLogPath = GetDiagDirectory(_hostType); _traceManager = new TraceManager(new HostTraceListener(diagLogPath, hostType.ToString(), logPageSize, logRetentionDays), this.SecretMasker); @@ -594,10 +600,8 @@ protected virtual void Dispose(bool disposing) _trace = null; _httpTrace?.Dispose(); _httpTrace = null; - _legacySecretMasker?.Dispose(); - _legacySecretMasker = null; - _newSecretMasker?.Dispose(); - _newSecretMasker = null; + _disposableSecretMasker?.Dispose(); + _disposableSecretMasker = null; _agentShutdownTokenSource?.Dispose(); _agentShutdownTokenSource = null; @@ -746,9 +750,13 @@ public static HttpClientHandler CreateHttpClientHandler(this IHostContext contex public static void AddAdditionalMaskingRegexes(this IHostContext context) { ArgUtil.NotNull(context, nameof(context)); - foreach (var pattern in AdditionalMaskingRegexes.CredScanPatterns) + foreach (var pattern in AdditionalMaskingPatterns.OneESPatterns) { - context.SecretMasker.AddRegex(pattern, $"HostContext_{WellKnownSecretAliases.CredScanPatterns}"); + context.SecretMasker.AddRegex(pattern.Regex, + $"HostContext_{WellKnownSecretAliases.OneESScanPatterns}", + pattern.Moniker, + pattern.SniffLiterals, + pattern.RegexOptions); } } } diff --git a/src/Test/L0/HostContextL0.cs b/src/Test/L0/HostContextL0.cs index 47cc14a6c6..2d406669c8 100644 --- a/src/Test/L0/HostContextL0.cs +++ b/src/Test/L0/HostContextL0.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Reflection; using System.Runtime.CompilerServices; @@ -90,45 +91,60 @@ public void UrlSecretsAreMasked(string input, string expected) [Trait("Level", "L0")] [Trait("Category", "Common")] // Some secrets that the scanner SHOULD suppress. - [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddeadAPIMxxxxxQ==", "***")] - [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddeadACDbxxxxxQ==", "***")] - [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddead+ABaxxxxxQ==", "***")] - [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddead+AMCxxxxxQ==", "***")] - [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddead+AStxxxxxQ==", "***")] - [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeadAzFuxdeadQ==", "***")] - [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddeaddeadxxAzSeDeadxx", "***")] - [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddeaddeadde+ACRDeadxx", "***")] - [InlineData("oy2mdeaddeaddeadeadqdeaddeadxxxezodeaddeadwxuq", "***")] - [InlineData("deaddeaddeaddeaddeaddeaddeaddeadxAIoTDeadxx=", "***")] - [InlineData("deaddeaddeaddeaddeaddeaddeaddeadx+ASbDeadxx=", "***")] - [InlineData("deaddeaddeaddeaddeaddeaddeaddeadx+AEhDeadxx=", "***")] - [InlineData("deaddeaddeaddeaddeaddeaddeaddeadx+ARmDeadxx=", "***")] - [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddAzCaDeadxx=", "***")] - [InlineData("xxx8Q~dead.dead.DEAD-DEAD-dead~deadxxxxx", "***")] - [InlineData("npm_deaddeaddeaddeaddeaddeaddeaddeaddead", "***")] - [InlineData("xxx7Q~dead.dead.DEAD-DEAD-dead~deadxx", "***")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddeadAPIMxxxxxQ==", "SEC102/102.Unclassified64ByteBase64String:1DC39072DA446911FE3E87EB697FB22ED6E2F75D7ECE4D0CE7CF4288CE0094D1")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddeadACDbxxxxxQ==", "SEC102/102.Unclassified64ByteBase64String:6AB186D06C8C6FBA25D39806913A70A4D77AB97C526D42B8C8DA6D441DE9F3C5")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddead+ABaxxxxxQ==", "SEC102/102.Unclassified64ByteBase64String:E1BB911668718D50C0C2CE7B9C93A5BB75A17212EA583A8BB060A014058C0802")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddead+AMCxxxxxQ==", "SEC102/102.Unclassified64ByteBase64String:7B3706299058BAC1622245A964D8DBBEF97A0C43C863F2702C4A1AD0413B3FC9")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddead+AStxxxxxQ==", "SEC102/102.Unclassified64ByteBase64String:58FF6B874E1B4014CF17C429A1E235E08466A0199090A0235975A35A87B8D440")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeadAzFuxdeadQ==", "SEC101/158.AzureFunctionIdentifiableKey:FF8E9A7C2A792029814C755C6704D9427F302E954DEF0FD5EE649BF9163E1F24")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddeaddeadxxAzSeDeadxx", "SEC101/167.AzureSearchIdentifiableKey:EAEC92AA13ECA43594A8FEED69D8B7F4696569E990718DCBE1B3872634540670")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddeaddeadde+ACRDeadxx", "SEC101/176.AzureContainerRegistryIdentifiableKey:CE62C55A2D3C220DA0CBFE292B5A6839EC7F747C5B5A7A55A4E5D7D76F1C7D32")] + [InlineData("oy2mdeaddeaddeadeadqdeaddeadxxxezodeaddeadwxuq", "SEC101/031.NuGetApiKey:FC93CD537067C7F452073F24C7043D5F58E11B6F49546316BBE06BAA5747317E")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeadxAIoTDeadxx=", "SEC102/101.Unclassified32ByteBase64String:2B0ADEB74FC9CDA3CD5D1066D85190407C57B8CAF45FCA7D50E26282AD61530C")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeadx+ASbDeadxx=", "SEC102/101.Unclassified32ByteBase64String:83F68F21FC0D7C5990929446509BFF80D604899064CA152D3524BBEECF7F6993")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeadx+AEhDeadxx=", "SEC102/101.Unclassified32ByteBase64String:E636DCD8D5F02304CE4B24DE2344B2D24C4B46BFD062EEF4D7673227720351C9")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeadx+ARmDeadxx=", "SEC102/101.Unclassified32ByteBase64String:9DEFFD24DE5F1DB24292B814B01868BC33E9298DF2BF3318C2B063B4D689A0BC")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddAzCaDeadxx=", "SEC101/154.AzureCacheForRedisIdentifiableKey:29894C9E3F5B60A1477AB08ABAE127152FAA20DD36C162B0FF21F16EF19233E5")] + [InlineData("npm_deaddeaddeaddeaddeaddeaddeaddeaddead", "SEC101/050.NpmAuthorKey:E06C20B8696373D4AEE3057CB1A577DC7A0F7F97BEE352D3C49B48B6328E1CBC")] + [InlineData("xxx8Q~dead.dead.DEAD-DEAD-dead~deadxxxxx", "SEC101/156.AadClientAppSecret:44DB247A273E912A1C3B45AC2732734CEAED00508AB85C3D4E801596CFF5B1D8")] + [InlineData("xxx7Q~dead.dead.DEAD-DEAD-dead~deadxx", "SEC101/156.AadClientAppSecret:23F12851970BB19BD76A448449F16F85BF4AFE915AD14BAFEE635F15021CE6BB")] // Some secrets that the scanner should NOT suppress. - [InlineData("SSdtIGEgY29tcGxldGVseSBpbm5vY3VvdXMgc3RyaW5nLg==", "SSdtIGEgY29tcGxldGVseSBpbm5vY3VvdXMgc3RyaW5nLg==")] - [InlineData("The password is knock knock knock", "The password is knock knock knock")] - public void OtherSecretsAreMasked(string input, string expected) + [InlineData("SSdtIGEgY29tcGxldGVseSBpbm5vY3VvdXMgc3RyaW5nLg==", null)] + [InlineData("The password is knock knock knock", null)] + public void OtherSecretsAreMasked(string input, string expectedValue) { // Arrange. - try + + foreach (string knobValue in new[] { "true", null }) { - Environment.SetEnvironmentVariable("AZP_USE_CREDSCAN_REGEXES", "true"); + string expected = expectedValue; + + // A null value in expected means we expect the input pattern + // to be returned (as it is unmasked). When our knob is null, + // indicating use of the legacy masker, we always expect "***" + // when masking. The new masker emits a redaction token that + // contains a security model id and the hash of the redacted thing. + expected = knobValue == null + ? expected == null ? input : "***" + : expected == null ? input : expected; - using (var _hc = Setup()) + try { - // Act. - var result = _hc.SecretMasker.MaskSecrets(input); + Environment.SetEnvironmentVariable("AZP_ENABLE_NEW_SECRET_MASKER", knobValue); + + using (var _hc = Setup()) + { + // Act. + var result = _hc.SecretMasker.MaskSecrets(input); - // Assert. - Assert.Equal(expected, result); + // Assert. + Assert.Equal(expected, result); + } + } + finally + { + Environment.SetEnvironmentVariable("AZP_ENABLE_NEW_SECRET_MASKER", null); } - } - finally - { - Environment.SetEnvironmentVariable("AZP_USE_CREDSCAN_REGEXES", null); } } diff --git a/src/Test/L0/SecretMaskerTests/LoggedSecretMaskerL0.cs b/src/Test/L0/SecretMaskerTests/LoggedSecretMaskerL0.cs index ba888d7957..e3b3de4e52 100644 --- a/src/Test/L0/SecretMaskerTests/LoggedSecretMaskerL0.cs +++ b/src/Test/L0/SecretMaskerTests/LoggedSecretMaskerL0.cs @@ -22,7 +22,7 @@ public LoggedSecretMaskerL0() [Trait("Category", "SecretMasker")] public void LoggedSecretMasker_MaskingSecrets() { - var lsm = new LoggedSecretMasker(_secretMasker) + using var lsm = new LoggedSecretMasker() { MinSecretLength = 0 }; @@ -39,7 +39,7 @@ public void LoggedSecretMasker_MaskingSecrets() [Trait("Category", "SecretMasker")] public void LoggedSecretMasker_ShortSecret_Removes_From_Dictionary() { - var lsm = new LoggedSecretMasker(_secretMasker) + using var lsm = new LoggedSecretMasker() { MinSecretLength = 0 }; @@ -58,7 +58,7 @@ public void LoggedSecretMasker_ShortSecret_Removes_From_Dictionary() [Trait("Category", "SecretMasker")] public void LoggedSecretMasker_ShortSecret_Removes_From_Dictionary_BoundaryValue() { - var lsm = new LoggedSecretMasker(_secretMasker) + using var lsm = new LoggedSecretMasker() { MinSecretLength = LoggedSecretMasker.MinSecretLengthLimit }; @@ -75,7 +75,7 @@ public void LoggedSecretMasker_ShortSecret_Removes_From_Dictionary_BoundaryValue [Trait("Category", "SecretMasker")] public void LoggedSecretMasker_ShortSecret_Removes_From_Dictionary_BoundaryValue2() { - var lsm = new LoggedSecretMasker(_secretMasker) + using var lsm = new LoggedSecretMasker() { MinSecretLength = LoggedSecretMasker.MinSecretLengthLimit }; @@ -92,7 +92,7 @@ public void LoggedSecretMasker_ShortSecret_Removes_From_Dictionary_BoundaryValue [Trait("Category", "SecretMasker")] public void LoggedSecretMasker_Skipping_ShortSecrets() { - var lsm = new LoggedSecretMasker(_secretMasker) + using var lsm = new LoggedSecretMasker() { MinSecretLength = 3 }; @@ -108,7 +108,7 @@ public void LoggedSecretMasker_Skipping_ShortSecrets() [Trait("Category", "SecretMasker")] public void LoggedSecretMasker_Sets_MinSecretLength_To_MaxValue() { - var lsm = new LoggedSecretMasker(_secretMasker); + using var lsm = new LoggedSecretMasker(); var expectedMinSecretsLengthValue = LoggedSecretMasker.MinSecretLengthLimit; lsm.MinSecretLength = LoggedSecretMasker.MinSecretLengthLimit + 1; @@ -121,7 +121,7 @@ public void LoggedSecretMasker_Sets_MinSecretLength_To_MaxValue() [Trait("Category", "SecretMasker")] public void LoggedSecretMasker_NegativeValue_Passed() { - var lsm = new LoggedSecretMasker(_secretMasker) + using var lsm = new LoggedSecretMasker() { MinSecretLength = -2 }; diff --git a/src/Test/L0/SecretMaskerTests/RegexSecretL0.cs b/src/Test/L0/SecretMaskerTests/RegexSecretL0.cs index 629198ab88..4d020e850e 100644 --- a/src/Test/L0/SecretMaskerTests/RegexSecretL0.cs +++ b/src/Test/L0/SecretMaskerTests/RegexSecretL0.cs @@ -1,6 +1,13 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License.using Agent.Sdk.SecretMasking; using Agent.Sdk.SecretMasking; + +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; + using Xunit; namespace Microsoft.VisualStudio.Services.Agent.Tests; @@ -8,7 +15,7 @@ namespace Microsoft.VisualStudio.Services.Agent.Tests; public class RegexSecretL0 { [Fact] - [Trait("Level","L0")] + [Trait("Level", "L0")] [Trait("Category", "RegexSecret")] public void Equals_ReturnsTrue_WhenPatternsAreEqual() { @@ -22,19 +29,257 @@ public void Equals_ReturnsTrue_WhenPatternsAreEqual() // Assert Assert.True(result); } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "RegexSecret")] + public void Equals_ReturnsFalse_WhenPatternsDiffer() + { + // Arrange + var secret1 = new RegexSecret("abc"); + var secret2 = new RegexSecret("def"); + + // Act + var result = secret1.Equals(secret2); + + // Assert + Assert.False(result); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "RegexSecret")] + public void Equals_ReturnsTrue_WhenMonikersAreEqual() + { + // Arrange + var moniker = "TST1001.TestRule"; + var secret1 = new RegexSecret("abc", moniker); + var secret2 = new RegexSecret("abc", moniker); + + // Act + var result = secret1.Equals(secret2); + + // Assert + Assert.True(result); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "RegexSecret")] + public void Equals_ReturnsFalse_WhenMonikersDiffer() + { + // Arrange + var secret1 = new RegexSecret("abc", "TST1001.TestRule"); + var secret2 = new RegexSecret("abc", "TST1002.TestRule"); + + // Act + var result = secret1.Equals(secret2); + + // Assert + Assert.False(result); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "RegexSecret")] + public void Equals_ReturnsTrue_WhenSniffLiteralsAreEqual() + { + // Arrange + var sniffLiterals = new HashSet(new[] { "sniff" }); + var secret1 = new RegexSecret("abc", sniffLiterals: sniffLiterals); + var secret2 = new RegexSecret("abc", sniffLiterals: sniffLiterals); + + // Act + var result = secret1.Equals(secret2); + + // Assert + Assert.True(result); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "RegexSecret")] + public void Equals_ReturnsFalse_WhenSniffLiteralsDiffer() + { + // Arrange + var secret1 = new RegexSecret("abc", sniffLiterals: new HashSet(new[] { "sniff1" })); + var secret2 = new RegexSecret("abc", sniffLiterals: new HashSet(new[] { "sniff2", "sniff3" })); + + // Act + var result = secret1.Equals(secret2); + + // Assert + Assert.False(result); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "RegexSecret")] + public void Equals_ReturnsTrue_WhenRegexOptionsAreEqual() + { + // Arrange + RegexOptions regexOptions = RegexOptions.IgnoreCase; + var secret1 = new RegexSecret("abc", regexOptions: regexOptions); + var secret2 = new RegexSecret("abc", regexOptions: regexOptions); + + // Act + var result = secret1.Equals(secret2); + + // Assert + Assert.True(result); + } + [Fact] - [Trait("Level","L0")] + [Trait("Level", "L0")] [Trait("Category", "RegexSecret")] - public void GetPositions_ReturnsEmpty_WhenNoMatchesExist() + public void Equals_ReturnsFalse_WhenRegexOptionsDiffer() + { + // Arrange + var secret1 = new RegexSecret("abc", regexOptions: RegexOptions.Multiline); + var secret2 = new RegexSecret("abc", regexOptions: RegexOptions.IgnoreCase); + + // Act + var result = secret1.Equals(secret2); + + // Assert + Assert.False(result); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "RegexSecret")] + public void Equals_ReturnsFalse_WhenComparedToNull() + { + // Arrange + var secret1 = new RegexSecret("abc"); + + // Act + var result = secret1.Equals(null); + + // Assert + Assert.False(result); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "RegexSecret")] + public void GetHashCode_ReturnsUniqueValue_WhenPatternsDiffer() + { + // Arrange + var secret1 = new RegexSecret("abc"); + var secret2 = new RegexSecret("def"); + + // Act + var hashCodeDiffers = secret1.GetHashCode() == secret2.GetHashCode(); + + // Assert + Assert.False(hashCodeDiffers); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "RegexSecret")] + public void GetHashCode_ReturnsUniqueValue_WhenMonikersDiffer() + { + // Arrange + var secret1 = new RegexSecret("abc", "TST1001.TestRule"); + var secret2 = new RegexSecret("abc", "TST1002.TestRule"); + var set = new HashSet(new[] { secret1 }); + + // Act + var hashCodeDiffers = secret1.GetHashCode() == secret2.GetHashCode(); + + // Assert + Assert.False(hashCodeDiffers); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "RegexSecret")] + public void GetHashCode_ReturnsUniqueValue_WhenSniffLiteralsDiffer() + { + // Arrange + var secret1 = new RegexSecret("abc", sniffLiterals: new HashSet(new[] { "sniff1" })); + var secret2 = new RegexSecret("abc", sniffLiterals: new HashSet(new[] { "sniff2" })); + var set = new HashSet(new[] { secret1 }); + + // Act + var hashCodeDiffers = secret1.GetHashCode() == secret2.GetHashCode(); + + // Assert + Assert.False(hashCodeDiffers); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "RegexSecret")] + public void GetHashCode_ReturnsUniqueValue_WhenRegexOptionsDiffer() + { + // Arrange + var secret1 = new RegexSecret("abc", regexOptions: RegexOptions.Multiline); + var secret2 = new RegexSecret("abc", regexOptions: RegexOptions.IgnoreCase); + var set = new HashSet(new[] { secret1 }); + + // Act + var hashCodeDiffers = secret1.GetHashCode() == secret2.GetHashCode(); + + // Assert + Assert.False(hashCodeDiffers); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "RegexSecret")] + public void GetReplacements_ReturnsEmpty_WhenNoMatchesExist() { // Arrange var secret = new RegexSecret("abc"); var input = "defdefdef"; // Act - var positions = secret.GetPositions(input); + var replacements = secret.GetReplacements(input); // Assert - Assert.Empty(positions); + Assert.Empty(replacements); } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "RegexSecret")] + public void GetReplacements_Returns_DefaultRedactionToken() + { + // Arrange + var secret = new RegexSecret("abc"); + var input = "abc"; + + // Act + var replacements = secret.GetReplacements(input); + + // Assert + Assert.NotEmpty(replacements); + Assert.True(replacements.Count() == 1); + Assert.True(replacements.First().Token == "+++"); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "RegexSecret")] + public void GetReplacements_Returns_SecureTelemetryTokenValue_WhenMonikerSpecified() + { + // Arrange + var ruleMoniker = "TST00.TestRule"; + var secret = new RegexSecret("abc", "TST00.TestRule"); + var input = "abc"; + + var hash = RegexSecret.HashString(input); + + // Act + var replacements = secret.GetReplacements(input); + + // Assert + Assert.NotEmpty(replacements); + Assert.True(replacements.Count() == 1); + Assert.True(replacements.First().Token == $"{ruleMoniker}:{hash}"); + } + } \ No newline at end of file diff --git a/src/Test/L0/SecretMaskerTests/SecretMaskerL0.cs b/src/Test/L0/SecretMaskerTests/SecretMaskerL0.cs index 31b529ef3f..86a068d9eb 100644 --- a/src/Test/L0/SecretMaskerTests/SecretMaskerL0.cs +++ b/src/Test/L0/SecretMaskerTests/SecretMaskerL0.cs @@ -5,15 +5,16 @@ using ValueEncoder = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoder; using ValueEncoders = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoders; using Xunit; +using System.Linq; namespace Microsoft.VisualStudio.Services.Agent.Tests { public sealed class SecretMaskerL0 { - private ISecretMasker initSecretMasker() + private SecretMasker InitBasicSecretMasker() { var testSecretMasker = new SecretMasker(); - testSecretMasker.AddRegex(AdditionalMaskingRegexes.UrlSecretPattern); + testSecretMasker.AddRegex(AdditionalMaskingPatterns.UrlSecretPattern); return testSecretMasker; } @@ -23,11 +24,13 @@ private ISecretMasker initSecretMasker() [Trait("Category", "SecretMasker")] public void IsSimpleUrlNotMasked() { - var testSecretMasker = initSecretMasker(); + using var testSecretMasker = InitBasicSecretMasker(); Assert.Equal( "https://simpledomain@example.com", testSecretMasker.MaskSecrets("https://simpledomain@example.com")); + + ValidateTelemetry(testSecretMasker); } [Fact] @@ -35,11 +38,13 @@ public void IsSimpleUrlNotMasked() [Trait("Category", "SecretMasker")] public void IsComplexUrlNotMasked() { - var testSecretMasker = initSecretMasker(); + using var testSecretMasker = InitBasicSecretMasker(); Assert.Equal( "https://url.com:443/~user/foo=bar+42-18?what=this.is.an.example....~~many@¶m=value", testSecretMasker.MaskSecrets("https://url.com:443/~user/foo=bar+42-18?what=this.is.an.example....~~many@¶m=value")); + + ValidateTelemetry(testSecretMasker); } [Fact] @@ -47,10 +52,10 @@ public void IsComplexUrlNotMasked() [Trait("Category", "SecretMasker")] public void IsUserInfoMaskedCorrectly() { - var testSecretMasker = initSecretMasker(); + using var testSecretMasker = InitBasicSecretMasker(); Assert.Equal( - "https://user:***@example.com", + "https://user:+++@example.com", testSecretMasker.MaskSecrets("https://user:pass@example.com")); } @@ -59,11 +64,13 @@ public void IsUserInfoMaskedCorrectly() [Trait("Category", "SecretMasker")] public void IsUserInfoWithSpecialCharactersMaskedCorrectly() { - var testSecretMasker = initSecretMasker(); + using var testSecretMasker = InitBasicSecretMasker(); Assert.Equal( - "https://user:***@example.com", + "https://user:+++@example.com", testSecretMasker.MaskSecrets(@"https://user:pass4';.!&*()=,$-+~@example.com")); + + ValidateTelemetry(testSecretMasker, expectedRedactions: 1); } [Fact] @@ -71,11 +78,13 @@ public void IsUserInfoWithSpecialCharactersMaskedCorrectly() [Trait("Category", "SecretMasker")] public void IsUserInfoWithDigitsInNameMaskedCorrectly() { - var testSecretMasker = initSecretMasker(); + using var testSecretMasker = InitBasicSecretMasker(); Assert.Equal( - "https://username123:***@example.com", + "https://username123:+++@example.com", testSecretMasker.MaskSecrets(@"https://username123:password@example.com")); + + ValidateTelemetry(testSecretMasker, expectedRedactions: 1); } [Fact] @@ -83,23 +92,27 @@ public void IsUserInfoWithDigitsInNameMaskedCorrectly() [Trait("Category", "SecretMasker")] public void IsUserInfoWithLongPasswordAndNameMaskedCorrectly() { - var testSecretMasker = initSecretMasker(); + using var testSecretMasker = InitBasicSecretMasker(); Assert.Equal( - "https://username_loooooooooooooooooooooooooooooooooooooooooong:***@example.com", + "https://username_loooooooooooooooooooooooooooooooooooooooooong:+++@example.com", testSecretMasker.MaskSecrets(@"https://username_loooooooooooooooooooooooooooooooooooooooooong:password_looooooooooooooooooooooooooooooooooooooooooooooooong@example.com")); + + ValidateTelemetry(testSecretMasker, expectedRedactions: 1); } [Fact] [Trait("Level", "L0")] [Trait("Category", "SecretMasker")] - public void IsUserInfoWithEncodedCharactersdInNameMaskedCorrectly() + public void IsUserInfoWithEncodedCharactersInNameMaskedCorrectly() { - var testSecretMasker = initSecretMasker(); + using var testSecretMasker = InitBasicSecretMasker(); Assert.Equal( - "https://username%10%A3%F6:***@example.com", + "https://username%10%A3%F6:+++@example.com", testSecretMasker.MaskSecrets(@"https://username%10%A3%F6:password123@example.com")); + + ValidateTelemetry(testSecretMasker, expectedRedactions: 1); } [Fact] @@ -107,13 +120,15 @@ public void IsUserInfoWithEncodedCharactersdInNameMaskedCorrectly() [Trait("Category", "SecretMasker")] public void IsUserInfoWithEncodedAndEscapedCharactersdInNameMaskedCorrectly() { - var testSecretMasker = initSecretMasker(); + using var testSecretMasker = InitBasicSecretMasker(); Assert.Equal( - "https://username%AZP2510%AZP25A3%AZP25F6:***@example.com", + "https://username%AZP2510%AZP25A3%AZP25F6:+++@example.com", testSecretMasker.MaskSecrets(@"https://username%AZP2510%AZP25A3%AZP25F6:password123@example.com")); + + ValidateTelemetry(testSecretMasker, expectedRedactions: 1); } - + [Fact] [Trait("Level","L0")] [Trait("Category", "SecretMasker")] @@ -140,9 +155,9 @@ public void SecretMaskerTests_CopyConstructor() secretMasker1.AddValueEncoder(x => x.Replace("_", "_masker-1-encoder-3")); // Assert masker 1 values. - Assert.Equal("***", secretMasker1.MaskSecrets("masker-1-regex-1___")); // original regex - Assert.Equal("***", secretMasker1.MaskSecrets("masker-1-regex-2___")); // original regex - Assert.Equal("***", secretMasker1.MaskSecrets("masker-1-regex-3___")); // new regex + Assert.Equal("+++", secretMasker1.MaskSecrets("masker-1-regex-1___")); // original regex + Assert.Equal("+++", secretMasker1.MaskSecrets("masker-1-regex-2___")); // original regex + Assert.Equal("+++", secretMasker1.MaskSecrets("masker-1-regex-3___")); // new regex Assert.Equal("***", secretMasker1.MaskSecrets("masker-1-value-1_")); // original value Assert.Equal("***", secretMasker1.MaskSecrets("masker-1-value-2_")); // original value Assert.Equal("***", secretMasker1.MaskSecrets("masker-1-value-3_")); // new value @@ -157,9 +172,9 @@ public void SecretMaskerTests_CopyConstructor() Assert.Equal("***masker-2-encoder-1", secretMasker1.MaskSecrets("masker-1-value-1_masker-2-encoder-1")); // separate encoder storage from copy // Assert masker 2 values. - Assert.Equal("***", secretMasker2.MaskSecrets("masker-1-regex-1___")); // copied regex - Assert.Equal("***", secretMasker2.MaskSecrets("masker-1-regex-2___")); // copied regex - Assert.Equal("***", secretMasker2.MaskSecrets("masker-2-regex-1___")); // new regex + Assert.Equal("+++", secretMasker2.MaskSecrets("masker-1-regex-1___")); // copied regex + Assert.Equal("+++", secretMasker2.MaskSecrets("masker-1-regex-2___")); // copied regex + Assert.Equal("+++", secretMasker2.MaskSecrets("masker-2-regex-1___")); // new regex Assert.Equal("***", secretMasker2.MaskSecrets("masker-1-value-1_")); // copied value Assert.Equal("***", secretMasker2.MaskSecrets("masker-1-value-2_")); // copied value Assert.Equal("***", secretMasker2.MaskSecrets("masker-2-value-1_")); // new value @@ -430,8 +445,8 @@ public void SecretMaskerTests_ReplacesOverlappingSecrets() var input = "abcdefg"; var result = secretMasker.MaskSecrets(input); - // a naive replacement would replace "def" first, and never find "bcd", resulting in "abc***g" - // or it would replace "bcd" first, and never find "def", resulting in "a***efg" + // a naive replacement would replace "def" first, and never find "bcd", resulting in "abc+++g" + // or it would replace "bcd" first, and never find "def", resulting in "a+++efg" Assert.Equal("a***g", result); } @@ -581,7 +596,7 @@ public void SecretMaskerTests_RemoveShortRegexes() var input = "abcdefgh123"; var result = secretMasker.MaskSecrets(input); - Assert.Equal("abc***3", result); + Assert.Equal("abc+++3", result); } [Fact] @@ -622,5 +637,25 @@ public void SecretMaskerTests_NotAddShortEncodedSecrets() Assert.Equal("ab***cd***", result); } + + private static void ValidateTelemetry(SecretMasker testSecretMasker, int expectedRedactions = 0, bool usesMoniker = false) + { + Assert.True(testSecretMasker.ElapsedMaskingTime > 0); + + if (expectedRedactions > 0) + { + Assert.NotEmpty(testSecretMasker.ReplacementTokens); + + if (!usesMoniker) + { + Assert.True(testSecretMasker.ReplacementTokens.First() == "+++"); + } + else + { + Assert.False(testSecretMasker.ReplacementTokens.First() == "+++"); + } + } + } + } } diff --git a/src/Test/L0/TestHostContext.cs b/src/Test/L0/TestHostContext.cs index dbea7f40a0..67469e38a6 100644 --- a/src/Test/L0/TestHostContext.cs +++ b/src/Test/L0/TestHostContext.cs @@ -65,7 +65,7 @@ public TestHostContext(object testClass, [CallerMemberName] string testName = "" var traceListener = new HostTraceListener(TraceFileName); traceListener.DisableConsoleReporting = true; - _secretMasker = new LoggedSecretMasker(new SecretMasker()); + _secretMasker = new LoggedSecretMasker(); _secretMasker.AddValueEncoder(ValueEncoders.JsonStringEscape); _secretMasker.AddValueEncoder(ValueEncoders.UriDataEscape); _secretMasker.AddValueEncoder(ValueEncoders.BackslashEscape);