-
Notifications
You must be signed in to change notification settings - Fork 864
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Configurable masking token, perf improvements, refactoring. #4586
base: master
Are you sure you want to change the base?
Changes from 4 commits
ec1a87d
de5824a
8de1ab2
d4b65df
a95cc17
a24381f
ba5aa24
8a772aa
f9e3a63
d911e20
1a87083
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
// Copyright (c) Microsoft Corporation. | ||
// Licensed under the MIT License. | ||
|
||
|
||
using System.Collections.Generic; | ||
using System.Text.RegularExpressions; | ||
|
||
namespace Microsoft.VisualStudio.Services.Agent | ||
{ | ||
public class PatternDescriptor | ||
{ | ||
public string Regex { get; set; } | ||
|
||
public ISet<string> SniffLiterals { get; set; } | ||
|
||
public RegexOptions RegexOptions { get; set; } | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,7 @@ | ||
using System; | ||
// Copyright (c) Microsoft Corporation. | ||
// Licensed under the MIT License. | ||
|
||
using System; | ||
using System.Collections.Generic; | ||
using System.Text.RegularExpressions; | ||
using Microsoft.VisualStudio.Services.Agent.Util; | ||
|
@@ -7,11 +10,16 @@ namespace Agent.Sdk.SecretMasking; | |
|
||
internal sealed class RegexSecret : ISecret | ||
{ | ||
public RegexSecret(String pattern) | ||
public RegexSecret(String pattern, | ||
ISet<string> sniffLiterals = null, | ||
RegexOptions regexOptions = RegexOptions.Compiled | RegexOptions.ExplicitCapture) | ||
{ | ||
ArgUtil.NotNullOrEmpty(pattern, nameof(pattern)); | ||
|
||
m_pattern = pattern; | ||
m_regex = new Regex(pattern, _regexOptions); | ||
m_regexOptions = regexOptions; | ||
m_sniffLiterals = sniffLiterals; | ||
m_regex = new Regex(pattern, regexOptions); | ||
} | ||
|
||
public override Boolean Equals(Object obj) | ||
|
@@ -21,31 +29,97 @@ public override Boolean Equals(Object obj) | |
{ | ||
return false; | ||
} | ||
return String.Equals(m_pattern, item.m_pattern, StringComparison.Ordinal); | ||
|
||
if (!String.Equals(m_pattern, item.m_pattern, StringComparison.Ordinal) || | ||
m_regexOptions != item.m_regexOptions) | ||
{ | ||
return false; | ||
} | ||
|
||
if (m_sniffLiterals == null && item.m_sniffLiterals == null) | ||
{ | ||
return true; | ||
} | ||
|
||
m_sniffLiterals.Equals(item.m_sniffLiterals); | ||
|
||
if (m_sniffLiterals.Count != item.m_sniffLiterals.Count) | ||
{ | ||
return false; | ||
} | ||
|
||
foreach(string sniffLiteral in m_sniffLiterals) | ||
{ | ||
if (!item.m_sniffLiterals.Contains(sniffLiteral)) | ||
{ | ||
return false; | ||
} | ||
} | ||
|
||
return true; | ||
} | ||
|
||
public override int GetHashCode() => m_pattern.GetHashCode(); | ||
public override int GetHashCode() | ||
{ | ||
int result = 17; | ||
unchecked | ||
{ | ||
result = (result * 31) + m_pattern.GetHashCode(); | ||
result = (result * 31) + m_regexOptions.GetHashCode(); | ||
|
||
public IEnumerable<ReplacementPosition> GetPositions(String input) | ||
// Use xor for set values to be order-independent. | ||
if (m_sniffLiterals != null) | ||
{ | ||
int xor_0 = 0; | ||
foreach (var sniffLiteral in m_sniffLiterals) | ||
{ | ||
xor_0 ^= sniffLiteral.GetHashCode(); | ||
} | ||
result = (result * 31) + xor_0; | ||
} | ||
} | ||
|
||
return result; | ||
} | ||
|
||
public IEnumerable<Replacement> GetPositions(String input) | ||
{ | ||
Int32 startIndex = 0; | ||
while (startIndex < input.Length) | ||
bool runRegexes = m_sniffLiterals == null ? true : false; | ||
|
||
if (m_sniffLiterals != null) | ||
{ | ||
var match = m_regex.Match(input, startIndex); | ||
if (match.Success) | ||
foreach (string sniffLiteral in m_sniffLiterals) | ||
{ | ||
startIndex = match.Index + 1; | ||
yield return new ReplacementPosition(match.Index, match.Length); | ||
if (input.IndexOf(sniffLiteral, StringComparison.Ordinal) != -1) | ||
{ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. A simple IndexOf check is 15x+ faster than something like RE2 in a typical perf test. And so using it as a first pass filter to decide whether to run a follow-on regex is likely to yield performance gains. These are typically 4o% or more in other scan scenarios 1ES owns that use the technique. |
||
runRegexes = true; | ||
break; | ||
} | ||
} | ||
else | ||
} | ||
|
||
if (runRegexes) | ||
{ | ||
Int32 startIndex = 0; | ||
while (startIndex < input.Length) | ||
{ | ||
yield break; | ||
var match = m_regex.Match(input, startIndex); | ||
if (match.Success) | ||
{ | ||
startIndex = match.Index + 1; | ||
yield return new Replacement(match.Index, match.Length, "+++"); | ||
} | ||
else | ||
{ | ||
yield break; | ||
} | ||
} | ||
} | ||
} | ||
|
||
public string Pattern { get { return m_pattern; } } | ||
private readonly ISet<string> m_sniffLiterals; | ||
private readonly RegexOptions m_regexOptions; | ||
private readonly String m_pattern; | ||
private readonly Regex m_regex; | ||
private static readonly RegexOptions _regexOptions = RegexOptions.Compiled | RegexOptions.ExplicitCapture; | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The logged secret masker (as opposed to the legacy logged secret masker) can extend
SecretMasker
rather than wrapping it (because no otherILoggedSecretMasker
will be wrapped by this class). This change will help us switch over to using aSecretMasker
instance that 1ES provides that lives in a different repo (https://github.com/microsoft/security-utilities).