Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ public enum OpKind {
CAPTURE_SIGNED_INTEGER_OR_UNCAPTURED_DASH,
CAPTURE_BRACKETED_WORD_AFTER_SKIP,
SKIP_ANY,
SKIP_ANY_EXCEPT_NEWLINE,
ANCHOR,
OPTIONAL_SEQUENCE
}
Expand Down Expand Up @@ -166,6 +167,7 @@ private static Op opFor(PatternAtom atom) {
Op.captureUntil(
OpKind.CAPTURE_QUOTED_UNTIL_DELIMITER, atom.groupNumber(), atom.delimiter());
case ANY_STAR -> Op.uncaptured(OpKind.SKIP_ANY);
case ANY_STAR_EXCEPT_NEWLINE -> Op.uncaptured(OpKind.SKIP_ANY_EXCEPT_NEWLINE);
case ANCHOR -> Op.uncaptured(OpKind.ANCHOR);
case OPTIONAL_SEQUENCE -> optionalOpFor(atom);
case COMPLEX_ALTERNATION -> null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ public enum Kind {
QUOTED_UNTIL_DELIMITER,
COMPLEX_ALTERNATION,
ANY_STAR,
ANY_STAR_EXCEPT_NEWLINE,
ANCHOR,
OPTIONAL_SEQUENCE,
BRACKETED_WORD_AFTER_SKIP
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -275,11 +275,13 @@ private static PatternAtom atomForQuantifier(
if (delimiter != null) {
return PatternAtom.capturedUntil(groupNumber, groupName, delimiter);
}
if ((charClass.chars.equals(CharSet.ANY)
|| charClass.chars.equals(CharSet.ANY_EXCEPT_NEWLINE))
&& !charClass.negated) {
if (charClass.chars.equals(CharSet.ANY) && !charClass.negated) {
return PatternAtom.captured(PatternAtom.Kind.ANY_STAR, groupNumber, groupName);
}
if (charClass.chars.equals(CharSet.ANY_EXCEPT_NEWLINE) && !charClass.negated) {
return PatternAtom.captured(
PatternAtom.Kind.ANY_STAR_EXCEPT_NEWLINE, groupNumber, groupName);
}
}
}
return null;
Expand Down Expand Up @@ -350,7 +352,7 @@ && atomForGroup(group) == null

private static boolean isTrailingBracketedWordSearch(List<RegexNode> children, int index) {
if (index + 6 >= children.size()) return false;
return isAnyStar(children.get(index))
return isDotAllAnyStar(children.get(index))
&& children.get(index + 1) instanceof LiteralNode spaceBefore
&& spaceBefore.ch == ' '
&& children.get(index + 2) instanceof LiteralNode open
Expand All @@ -362,7 +364,7 @@ && isWordBoundaryWordBoundary(stripNonCapturingGroup(group.child))
&& close.ch == ']'
&& children.get(index + 5) instanceof LiteralNode spaceAfter
&& spaceAfter.ch == ' '
&& isAnyStar(children.get(index + 6));
&& isDotAllAnyStar(children.get(index + 6));
}

private static boolean containsBacktrackingControl(RegexNode node) {
Expand Down Expand Up @@ -430,17 +432,15 @@ public Boolean visitBranchReset(BranchResetNode node) {
});
}

private static boolean isAnyStar(RegexNode node) {
private static boolean isDotAllAnyStar(RegexNode node) {
if (!(node instanceof QuantifierNode quantifier)
|| quantifier.min != 0
|| quantifier.max != -1
|| !quantifier.greedy
|| !(quantifier.child instanceof CharClassNode charClass)
|| charClass.negated) {
return false;
}
return charClass.chars.equals(CharSet.ANY)
|| charClass.chars.equals(CharSet.ANY_EXCEPT_NEWLINE);
|| !quantifier.greedy) return false;
RegexNode child = stripNonCapturingGroup(quantifier.child);
return child instanceof CharClassNode charClass
&& charClass.chars.equals(CharSet.ANY)
&& !charClass.negated;
}

private static RegexNode stripNonCapturingGroup(RegexNode node) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ class PatternCategorizerTest {
@Test
void categorizesLinearDelimitedLogTemplateWithoutGrokNames() throws Exception {
String pattern =
"(?<client>(?:[0-9]{1,3}\\.){3}[0-9]{1,3}|[A-Za-z0-9.-]+) "
"(?s)(?<client>(?:[0-9]{1,3}\\.){3}[0-9]{1,3}|[A-Za-z0-9.-]+) "
+ "(?<ident>\\S+) "
+ "(?<auth>\\S+) "
+ "\\[(?<timestamp>[^\\]]+)\\]\\s+"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ final class LinearTokenSequenceMatcher extends ReggieMatcher {
private final LinearTokenSequencePlan plan;
private final int groupCount;
private final int optionalDepth;
private final ThreadLocal<MatchWorkspace> workspaceHolder;

LinearTokenSequenceMatcher(
String pattern,
Expand All @@ -37,8 +36,6 @@ final class LinearTokenSequenceMatcher extends ReggieMatcher {
this.groupCount = groupCount;
this.nameToIndex = Map.copyOf(nameToIndex);
this.optionalDepth = maxOptionalDepth(plan.ops());
this.workspaceHolder =
ThreadLocal.withInitial(() -> new MatchWorkspace(this.groupCount, this.optionalDepth));
}

@Override
Expand All @@ -49,7 +46,7 @@ boolean embedsNameMap() {
@Override
public boolean matches(String input) {
Objects.requireNonNull(input, "input");
return matchesAt(input, 0, input.length(), workspaceHolder.get(), true);
return matchesAt(input, 0, input.length(), newWorkspace(), true);
}

@Override
Expand All @@ -61,7 +58,7 @@ public boolean find(String input) {
public int findFrom(String input, int start) {
Objects.requireNonNull(input, "input");
if (start < 0 || start > input.length()) return -1;
MatchWorkspace workspace = workspaceHolder.get();
MatchWorkspace workspace = newWorkspace();
for (int pos = start; pos <= input.length(); pos++) {
if (matchesAt(input, pos, input.length(), workspace, false)) return pos;
}
Expand All @@ -71,7 +68,7 @@ public int findFrom(String input, int start) {
@Override
public MatchResult match(String input) {
Objects.requireNonNull(input, "input");
MatchWorkspace workspace = workspaceHolder.get();
MatchWorkspace workspace = newWorkspace();
if (!matchesAt(input, 0, input.length(), workspace, true)) return null;
return toMatchResult(input, workspace);
}
Expand All @@ -80,16 +77,16 @@ public MatchResult match(String input) {
public boolean matchesBounded(CharSequence input, int start, int end) {
Objects.requireNonNull(input, "input");
if (!isValidRegion(input, start, end)) return false;
return matchesAt(input, start, end, workspaceHolder.get(), true);
return matchesAt(input, start, end, newWorkspace(), true);
}

@Override
public MatchResult matchBounded(CharSequence input, int start, int end) {
Objects.requireNonNull(input, "input");
if (!isValidRegion(input, start, end)) return null;
MatchWorkspace workspace = workspaceHolder.get();
MatchWorkspace workspace = newWorkspace();
if (!matchesAt(input, start, end, workspace, true)) return null;
return toMatchResult(input.subSequence(0, end).toString(), workspace);
return toMatchResult(input.toString(), workspace);
}

@Override
Expand All @@ -101,7 +98,7 @@ public MatchResult findMatch(String input) {
public MatchResult findMatchFrom(String input, int start) {
Objects.requireNonNull(input, "input");
if (start < 0 || start > input.length()) return null;
MatchWorkspace workspace = workspaceHolder.get();
MatchWorkspace workspace = newWorkspace();
for (int pos = start; pos <= input.length(); pos++) {
if (!matchesAt(input, pos, input.length(), workspace, false)) {
continue;
Expand All @@ -125,22 +122,19 @@ boolean matchIntoBounded(
if (groupStarts.length <= groupCount || groupEnds.length <= groupCount) {
throw new IndexOutOfBoundsException("group arrays too small for " + groupCount + " groups");
}
MatchWorkspace workspace = workspaceHolder.get();
MatchWorkspace workspace = newWorkspace();
if (!matchesAt(input, start, end, workspace, true)) return false;
System.arraycopy(workspace.starts, 0, groupStarts, 0, groupCount + 1);
System.arraycopy(workspace.ends, 0, groupEnds, 0, groupCount + 1);
return true;
}

private MatchResult toMatchResult(String input, MatchWorkspace workspace) {
// workspace is reused across calls on the same thread; MatchResultImpl/NamedMatchResultImpl
// hold onto the arrays they're given, so a defensive copy is required here.
int[] starts = Arrays.copyOf(workspace.starts, workspace.starts.length);
int[] ends = Arrays.copyOf(workspace.ends, workspace.ends.length);
if (!nameToIndex.isEmpty()) {
return new NamedMatchResultImpl(input, starts, ends, groupCount, nameToIndex);
return new NamedMatchResultImpl(
input, workspace.starts, workspace.ends, groupCount, nameToIndex);
}
return new MatchResultImpl(input, starts, ends, groupCount, nameToIndex);
return new MatchResultImpl(input, workspace.starts, workspace.ends, groupCount, nameToIndex);
}

private static void validateRegion(CharSequence input, int start, int end) {
Expand All @@ -153,6 +147,10 @@ private static boolean isValidRegion(CharSequence input, int start, int end) {
return start >= 0 && end >= start && end <= input.length();
}

private MatchWorkspace newWorkspace() {
return new MatchWorkspace(groupCount, optionalDepth);
}

private boolean matchesAt(
CharSequence input, int offset, int regionEnd, MatchWorkspace workspace, boolean fullMatch) {
int[] starts = workspace.starts;
Expand Down Expand Up @@ -221,6 +219,8 @@ private int apply(
case CAPTURE_BRACKETED_WORD_AFTER_SKIP ->
captureBracketedWordAfterSkip(input, pos, regionEnd, op.groupNumber(), starts, ends);
case SKIP_ANY -> lastOp ? consumeToEnd(input, pos, regionEnd) : -1;
case SKIP_ANY_EXCEPT_NEWLINE ->
lastOp ? consumeToEndExceptNewline(input, pos, regionEnd) : -1;
case ANCHOR -> pos;
case OPTIONAL_SEQUENCE ->
applyOptional(op, input, pos, regionEnd, starts, ends, workspace, optionalDepth);
Expand All @@ -230,7 +230,7 @@ private int apply(
private static int captureNonSpace(
CharSequence input, int pos, int regionEnd, int group, int[] starts, int[] ends) {
int start = pos;
while (pos < regionEnd && !Character.isWhitespace(input.charAt(pos))) pos++;
while (pos < regionEnd && !isJdkWhitespace(input.charAt(pos))) pos++;
if (pos == start) return -1;
set(starts, ends, group, start, pos);
return pos;
Expand Down Expand Up @@ -336,7 +336,7 @@ private static int captureQuotedUntil(
if (end == regionEnd) return -1;
if (nonSpace) {
for (int i = start; i < end; i++) {
if (Character.isWhitespace(input.charAt(i))) return -1;
if (isJdkWhitespace(input.charAt(i))) return -1;
}
}
set(starts, ends, group, start, end);
Expand All @@ -357,7 +357,7 @@ private static int captureBracketedWordAfterSkip(
int wordEnd = -1;
for (int index = pos; index < regionEnd; index++) {
char ch = input.charAt(index);
if (ch == '[') {
if (ch == '[' && index > pos && input.charAt(index - 1) == ' ') {
open = index;
wordEnd = index + 1;
continue;
Expand All @@ -369,7 +369,7 @@ private static int captureBracketedWordAfterSkip(
if (wordEnd == index
&& wordEnd > open + 1
&& index + 1 < regionEnd
&& Character.isWhitespace(input.charAt(index + 1))) {
&& input.charAt(index + 1) == ' ') {
lastStart = open + 1;
lastEnd = index;
}
Expand Down Expand Up @@ -495,39 +495,26 @@ private static final class MatchWorkspace {

private static int skipWhitespace(CharSequence input, int pos, int regionEnd) {
int start = pos;
while (pos < regionEnd && Character.isWhitespace(input.charAt(pos))) pos++;
while (pos < regionEnd && isJdkWhitespace(input.charAt(pos))) pos++;
return pos == start ? -1 : pos;
}

private static boolean startsWith(CharSequence input, int pos, int regionEnd, String prefix) {
if (pos < 0 || pos + prefix.length() > regionEnd) return false;
if (input instanceof String s) {
return s.startsWith(prefix, pos);
}
for (int i = 0; i < prefix.length(); i++) {
if (input.charAt(pos + i) != prefix.charAt(i)) return false;
}
return true;
}

private static int findChar(CharSequence input, int pos, int regionEnd, char target) {
if (input instanceof String s) {
int idx = s.indexOf(target, pos);
return idx >= 0 && idx < regionEnd ? idx : regionEnd;
}
for (int i = pos; i < regionEnd; i++) {
if (input.charAt(i) == target) return i;
}
return regionEnd;
}

private static int findLastLiteral(CharSequence input, int start, int end, String literal) {
if (input instanceof String s) {
int fromIndex = end - literal.length();
if (fromIndex < start) return -1;
int idx = s.lastIndexOf(literal, fromIndex);
return idx >= start ? idx : -1;
}
int last = -1;
for (int pos = start; pos + literal.length() <= end; pos++) {
if (startsWith(input, pos, end, literal)) last = pos;
Expand All @@ -536,18 +523,19 @@ private static int findLastLiteral(CharSequence input, int start, int end, Strin
}

private static int consumeToEnd(CharSequence input, int pos, int regionEnd) {
// String#charAt has no observable side effects, so the validation walk below — which exists
// to surface CharSequence implementations that throw or misbehave on out-of-range access —
// is unnecessary overhead for the common String case.
if (input instanceof String) {
return regionEnd;
}
while (pos < regionEnd) {
input.charAt(pos++);
}
return regionEnd;
}

private static int consumeToEndExceptNewline(CharSequence input, int pos, int regionEnd) {
while (pos < regionEnd) {
if (input.charAt(pos++) == '\n') return -1;
}
return regionEnd;
}

private static void set(int[] starts, int[] ends, int group, int start, int end) {
if (group > 0) {
starts[group] = start;
Expand All @@ -568,7 +556,7 @@ private static boolean isIpOrHost(CharSequence input, int start, int end) {
private static boolean isNonSpace(CharSequence input, int start, int end) {
if (end <= start) return false;
for (int i = start; i < end; i++) {
if (Character.isWhitespace(input.charAt(i))) return false;
if (isJdkWhitespace(input.charAt(i))) return false;
}
return true;
}
Expand All @@ -591,6 +579,10 @@ private static int scanDecimal(CharSequence input, int pos, int limit, boolean s
return pos;
}

private static boolean isJdkWhitespace(char ch) {
return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\f' || ch == '\r';
}
Comment thread
jbachorik marked this conversation as resolved.

private static boolean isDigit(char ch) {
return ch >= '0' && ch <= '9';
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -817,17 +817,26 @@ private static ReggieMatcher tryCompileLinearTokenSequence(
String pattern, RegexNode ast, Map<String, Integer> nameMap) {
return LinearTokenSequencePlan.from(PatternCategorizer.categorize(ast))
.filter(plan -> plan.coversCaptureIndexes(nameMap.values()))
.filter(RuntimeCompiler::isRuntimeExecutableLinearTokenSequence)
.filter(plan -> isRuntimeExecutableLinearTokenSequence(pattern, plan))
.map(plan -> new LinearTokenSequenceMatcher(pattern, plan, countGroups(pattern), nameMap))
.map(m -> m.embedsNameMap() ? m : new NameEnrichingMatcher(m))
.orElse(null);
}

private static boolean isRuntimeExecutableLinearTokenSequence(LinearTokenSequencePlan plan) {
private static boolean isRuntimeExecutableLinearTokenSequence(
String pattern, LinearTokenSequencePlan plan) {
boolean requiresDotAll = false;
for (int i = 0; i < plan.ops().size(); i++) {
LinearTokenSequencePlan.Op op = plan.ops().get(i);
if (op.kind() == LinearTokenSequencePlan.OpKind.ANCHOR) return false;
if (op.kind() == LinearTokenSequencePlan.OpKind.SKIP_ANY && i != plan.ops().size() - 1) {
if (op.kind() == LinearTokenSequencePlan.OpKind.SKIP_ANY_EXCEPT_NEWLINE) return false;
if (op.kind() == LinearTokenSequencePlan.OpKind.SKIP_ANY
|| op.kind() == LinearTokenSequencePlan.OpKind.CAPTURE_BRACKETED_WORD_AFTER_SKIP) {
requiresDotAll = true;
}
if ((op.kind() == LinearTokenSequencePlan.OpKind.SKIP_ANY
|| op.kind() == LinearTokenSequencePlan.OpKind.SKIP_ANY_EXCEPT_NEWLINE)
&& i != plan.ops().size() - 1) {
return false;
}
if (op.kind() == LinearTokenSequencePlan.OpKind.OPTIONAL_SEQUENCE
Expand All @@ -836,7 +845,7 @@ && canOptionalPresentBranchStealFollowingInput(op, plan.ops().get(i + 1))) {
return false;
}
}
return true;
return !requiresDotAll || pattern.startsWith("(?s)");
}

private static boolean canOptionalPresentBranchStealFollowingInput(
Expand Down
Loading
Loading