Skip to content

Commit

Permalink
ValueCluster Collection has to escape wildcards in Like Stements
Browse files Browse the repository at this point in the history
  • Loading branch information
@RNoeldner
RNoeldner committed Apr 14, 2024
1 parent c70f81c commit af992f4
Show file tree
Hide file tree
Showing 6 changed files with 75 additions and 74 deletions.
38 changes: 37 additions & 1 deletion Library/ClassLibraryCSV/StringUtils.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -490,6 +490,42 @@ public static ReadOnlySpan<char> SqlName(this ReadOnlySpan<char> contents) =>
public static string SqlQuote(this string? contents) =>
contents is null || contents.Length == 0 ? string.Empty : contents.Replace("'", "''");

/// <summary>
/// Strings with the right substitution to be used as filter If a pattern in a LIKE clause
/// contains any of these special characters * % [ ], those characters must be escaped in
/// brackets [ ] like this [*], [%], [[] or []].
/// </summary>
/// <param name="inputValue">The input.</param>
/// <returns></returns>
public static string StringEscapeLike(this string? inputValue)
{
if (string.IsNullOrEmpty(inputValue))
return string.Empty;
var returnVal = new StringBuilder(inputValue!.Length);
foreach (var c in inputValue)
{
switch (c)
{
case '%':
case '*':
case '[':
case ']':
returnVal.Append('[' + c + ']');
break;

case '\'':
returnVal.Append("''");
break;

default:
returnVal.Append(c);
break;
}
}

return returnVal.ToString();
}

/// <summary>
/// Handles quotes in SQLs, does not include the outer quotes
/// </summary>
Expand Down Expand Up @@ -524,7 +560,7 @@ public static bool TryGetConstant(this string entry, out string result)
result = entry;
return true;
}

/// <summary>
/// Read the value and determine if this could be a constant value ( surrounded by " or ' ) or
/// if it's a number; if not its assume is a reference to another field
Expand Down
62 changes: 7 additions & 55 deletions Library/WinFormControls/ColumnFilterLogic.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,18 +280,6 @@ public void ApplyFilter()
m_Active = m_FilterExpressionValue.Length > 0 || m_FilterExpressionOperator.Length > 0;
}

/// <summary>
/// Builds the SQL command.
/// </summary>
/// <param name="valueText">The value text.</param>
/// <returns></returns>
//public string BuildSqlCommand(in string valueText)
//{
// if (valueText == OperatorIsNull)
// return string.Format(CultureInfo.InvariantCulture, "({0} IS NULL or {0} = '')", m_DataPropertyNameEscape);
// return string.Format(CultureInfo.InvariantCulture, "{0} = {1}", m_DataPropertyNameEscape, FormatValue(valueText.AsSpan(), m_DataType));
//}

/// <summary>
/// Set the Filter to a value
/// </summary>
Expand Down Expand Up @@ -366,46 +354,10 @@ private static string FormatValue(ReadOnlySpan<char> value, DataTypeEnum targetT
return string.Empty;
}

/// <summary>
/// Strings with the right substitution to be used as filter If a pattern in a LIKE clause
/// contains any of these special characters * % [ ], those characters must be escaped in
/// brackets [ ] like this [*], [%], [[] or []].
/// </summary>
/// <param name="inputValue">The input.</param>
/// <returns></returns>
private static string StringEscapeLike(in string inputValue)
{
if (string.IsNullOrEmpty(inputValue))
return string.Empty;
var returnVal = new StringBuilder(inputValue.Length);
foreach (var c in inputValue)
{
switch (c)
{
case '%':
case '*':
case '[':
case ']':
returnVal.Append('[' + c + ']');
break;

case '\'':
returnVal.Append("''");
break;

default:
returnVal.Append(c);
break;
}
}

return returnVal.ToString();
}

/// <summary>
/// Builds the filter expression for this column for Operator based filter
/// </summary>
/// <returns>A SQL Condition to be used on DataTable</returns>
/// <returns>A SQL Condition to be used on DataTable</returns>
private string BuildFilterExpressionOperator()
{
switch (m_Operator)
Expand All @@ -431,8 +383,8 @@ private string BuildFilterExpressionOperator()
if (!string.IsNullOrEmpty(m_ValueText))
{
if (m_DataType == DataTypeEnum.String)
return string.Format(CultureInfo.InvariantCulture, "{0} LIKE '%{1}%'", m_DataPropertyNameEscape, StringEscapeLike(m_ValueText));
return string.Format(CultureInfo.InvariantCulture, "Convert({0},'System.String') LIKE '%{1}%'", m_DataPropertyNameEscape, StringEscapeLike(m_ValueText));
return string.Format(CultureInfo.InvariantCulture, "{0} LIKE '%{1}%'", m_DataPropertyNameEscape, m_ValueText.StringEscapeLike());
return string.Format(CultureInfo.InvariantCulture, "Convert({0},'System.String') LIKE '%{1}%'", m_DataPropertyNameEscape, m_ValueText.StringEscapeLike());
}
break;

Expand All @@ -444,13 +396,13 @@ private string BuildFilterExpressionOperator()

case cOperatorBegins:
if (m_DataType == DataTypeEnum.String)
return string.Format(CultureInfo.InvariantCulture, "{0} LIKE '{1}%'", m_DataPropertyNameEscape, StringEscapeLike(m_ValueText));
return string.Format(CultureInfo.InvariantCulture, "Convert({0},'System.String') LIKE '{1}%'", m_DataPropertyNameEscape, StringEscapeLike(m_ValueText));
return string.Format(CultureInfo.InvariantCulture, "{0} LIKE '{1}%'", m_DataPropertyNameEscape, m_ValueText.StringEscapeLike());
return string.Format(CultureInfo.InvariantCulture, "Convert({0},'System.String') LIKE '{1}%'", m_DataPropertyNameEscape, m_ValueText.StringEscapeLike());

case cOperatorEnds:
if (m_DataType == DataTypeEnum.String)
return $"{m_DataPropertyNameEscape} LIKE '%{StringEscapeLike(m_ValueText)}'";
return $"Convert({m_DataPropertyNameEscape},'System.String') LIKE '%{StringEscapeLike(m_ValueText)}'";
return $"{m_DataPropertyNameEscape} LIKE '%{m_ValueText.StringEscapeLike()}'";
return $"Convert({m_DataPropertyNameEscape},'System.String') LIKE '%{m_ValueText.StringEscapeLike()}'";

default:
string filterValue;
Expand Down
2 changes: 1 addition & 1 deletion Library/WinFormControls/FromRowsFilter.Designer.cs
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions Library/WinFormControls/FromRowsFilter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ public sealed partial class FromRowsFilter : ResizeForm
private readonly ColumnFilterLogic m_DataGridViewColumnFilter;
private readonly ICollection<object> m_Values;
private readonly int m_MaxCluster;
private readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
private readonly CancellationTokenSource m_CancellationTokenSource = new CancellationTokenSource();

/// <summary>
/// Initializes a new instance of the <see cref="FromRowsFilter" /> class.
Expand Down Expand Up @@ -255,7 +255,7 @@ private void timerRebuild_Tick(object sender, EventArgs e)
{
timerRebuild.Stop();

using var frm = new FormProgress("Filter", false, FontConfig, cancellationTokenSource.Token);
using var frm = new FormProgress("Filter", false, FontConfig, m_CancellationTokenSource.Token);
frm.SetMaximum(100);
frm.Show();
frm.Report(new ProgressInfo("Building clusters", 1));
Expand Down Expand Up @@ -308,7 +308,7 @@ private void ClusterTypeChanged(object sender, EventArgs e)

private void FromRowsFilter_FormClosing(object sender, FormClosingEventArgs e)
{
cancellationTokenSource.Cancel();
m_CancellationTokenSource.Cancel();
}
}
}
19 changes: 11 additions & 8 deletions Library/WinFormControls/ValueClusterCollection.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -856,7 +856,7 @@ private BuildValueClustersResult BuildValueClustersString(in ICollection<string>
$"(SUBSTRING({escapedName},1,1) >= 's' AND SUBSTRING({escapedName},1,1) <= 'z')", countC4, "s", "z"));

var countN = CountPassing(values,
x => x[0] > 48 && x[0] < 57);
x => x[0] >= 48 && x[0] <= 57);
percent += step;
progress?.Report(new ProgressInfo("Range clusters 0-9", percent));
if (countN > 0)
Expand All @@ -872,12 +872,15 @@ private BuildValueClustersResult BuildValueClustersString(in ICollection<string>
AddUnique(new ValueCluster("Special", $"(SUBSTRING({escapedName},1,1) < ' ')", countS, null));

var countP = CountPassing(values,
x => x[0] >= 32 && x[0] < 48 || x[0] >= 58 && x[0] < 65 || x[0] >= 91 && x[0] <= 96 ||
x[0] >= 173 && x[0] <= 176);
x =>
x[0] >= ' ' && x[0] <= '/'
|| x[0] >= ':' && x[0] <= '@'
|| x[0] >= '[' && x[0] <= '`'
|| x[0] >= '{' && x[0] <= '~');
percent += step;
progress?.Report(new ProgressInfo("Range clusters Punctuation", percent));
progress?.Report(new ProgressInfo("Range clusters Punctuation, Marks and Braces", percent));
if (countP > 0)
AddUnique(new ValueCluster("Punctuation",
AddUnique(new ValueCluster("Punctuation & Marks & Braces",
$"((SUBSTRING({escapedName},1,1) >= ' ' AND SUBSTRING({escapedName},1,1) <= '/') " +
$"OR (SUBSTRING({escapedName},1,1) >= ':' AND SUBSTRING({escapedName},1,1) <= '@') " +
$"OR (SUBSTRING({escapedName},1,1) >= '[' AND SUBSTRING({escapedName},1,1) <= '`') " +
Expand Down Expand Up @@ -951,11 +954,11 @@ private BuildValueClustersResult BuildValueClustersString(in ICollection<string>
if (bigger.Count > 0)
{
var sbExcluded = new StringBuilder();
sbExcluded.Append($"{escapedName} LIKE '{text.SqlQuote()}%' AND NOT(");
sbExcluded.Append($"{escapedName} LIKE '{text.StringEscapeLike()}%' AND NOT(");
foreach (var kvp in bigger)
{
countAll -= kvp.Value;
sbExcluded.Append($"{escapedName} = '{kvp.Key.SqlQuote()}' OR");
sbExcluded.Append($"{escapedName} = '{kvp.Key.StringEscapeLike()}' OR");
}

sbExcluded.Length -= 3;
Expand All @@ -972,7 +975,7 @@ private BuildValueClustersResult BuildValueClustersString(in ICollection<string>
percent += step;
progress?.Report(new ProgressInfo("New Clusters", percent));

AddUnique(new ValueCluster($"{text}…", $"({escapedName} LIKE '{text.SqlQuote()}%')", countAll, text));
AddUnique(new ValueCluster($"{text}…", $"({escapedName} LIKE '{text.StringEscapeLike()}%')", countAll, text));
}
}

Expand Down
22 changes: 16 additions & 6 deletions UnitTest/WinFormControlsUnitTest/ValueClusterCollectionTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,18 +53,23 @@ public void BuildValueClusters_StringMaxRestricted()
var fl = GetFilterLogic(0);
Assert.AreEqual(BuildValueClustersResult.ListFilled, fl.ValueClusterCollection.ReBuildValueClusters(
DataTypeEnum.String,
GetColumnData(UnitTestStaticData.Columns.First(x => x.Name == "string").ColumnOrdinal)!, "d1", false, 10),
GetColumnData(UnitTestStaticData.Columns.First(x => x.Name == "string").ColumnOrdinal)!, "string", false, 10),
"Column String");

Assert.IsTrue(fl.ValueClusterCollection.Count>4 && fl.ValueClusterCollection.Count<=10);
Assert.AreEqual(m_Data.Rows.Count, fl.ValueClusterCollection.Select(x => x.Count).Sum(), "Clusters should cover all entries");
// The filter for Specials does not count correct, must be a difference between c# and ADO
foreach (var cluster in fl.ValueClusterCollection.Take(5))
{
m_DataView.RowFilter = cluster.SQLCondition;
Assert.AreEqual(m_DataView.Count, cluster.Count, cluster.SQLCondition);
}
}

[TestMethod]
[Timeout(1000)]
public void BuildValueClusters_StringUnique()
{

var data = new List<object>(40000);
for (int i = 0; i < 20000; i++)
data.Add((i % 15).ToString());
Expand All @@ -84,27 +89,32 @@ public void BuildValueClusters_StringUnique()
"Column String");
Assert.AreEqual(19,fl.ValueClusterCollection.Count);
Assert.AreEqual(data.Count, fl.ValueClusterCollection.Select(x => x.Count).Sum(), "Clusters should cover all entries");

}

[TestMethod]
[Timeout(1000)]
public void BuildValueClusters_StringUseOld()
public void BuildValueClusters_StringUseAlreadyExisting()
{
var fl = GetFilterLogic(0);
const int max1 = 100;
const int max2 = 200;
Assert.AreEqual(BuildValueClustersResult.ListFilled, fl.ValueClusterCollection.ReBuildValueClusters(
DataTypeEnum.String,
GetColumnData(UnitTestStaticData.Columns.First(x => x.Name == "string").ColumnOrdinal)!, "d1", false, max1),
GetColumnData(UnitTestStaticData.Columns.First(x => x.Name == "string").ColumnOrdinal)!, "string", false, max1),
"Column String");

Assert.IsTrue(fl.ValueClusterCollection.Count>4 && fl.ValueClusterCollection.Count<=max1,
$"Expected {4}-{max1} is: {fl.ValueClusterCollection.Count}");
var before = fl.ValueClusterCollection.Count;

foreach (var cluster in fl.ValueClusterCollection)
{
m_DataView.RowFilter = cluster.SQLCondition;
Assert.AreEqual(m_DataView.Count, cluster.Count, cluster.SQLCondition);
}
Assert.AreEqual(BuildValueClustersResult.ListFilled, fl.ValueClusterCollection.ReBuildValueClusters(
DataTypeEnum.String,
GetColumnData(UnitTestStaticData.Columns.First(x => x.Name == "string").ColumnOrdinal)!, "d1", false, max2),
GetColumnData(UnitTestStaticData.Columns.First(x => x.Name == "string").ColumnOrdinal)!, "string", false, max2),
"Column String");
Assert.IsTrue(fl.ValueClusterCollection.Count>=before && fl.ValueClusterCollection.Count<=max2 ,
$"Expected {before}-{max2} is: {fl.ValueClusterCollection.Count}");
Expand Down

0 comments on commit af992f4

Please sign in to comment.