diff --git a/reggie-codegen/src/main/java/com/datadoghq/reggie/codegen/analysis/LinearTokenSequencePlan.java b/reggie-codegen/src/main/java/com/datadoghq/reggie/codegen/analysis/LinearTokenSequencePlan.java index 01af977c..facf1e63 100644 --- a/reggie-codegen/src/main/java/com/datadoghq/reggie/codegen/analysis/LinearTokenSequencePlan.java +++ b/reggie-codegen/src/main/java/com/datadoghq/reggie/codegen/analysis/LinearTokenSequencePlan.java @@ -43,6 +43,7 @@ public enum OpKind { CAPTURE_SIGNED_INTEGER_OR_UNCAPTURED_DASH, CAPTURE_BRACKETED_WORD_AFTER_SKIP, SKIP_ANY, + SKIP_ANY_EXCEPT_NEWLINE, ANCHOR, OPTIONAL_SEQUENCE } @@ -166,6 +167,7 @@ private static Op opFor(PatternAtom atom) { Op.captureUntil( OpKind.CAPTURE_QUOTED_UNTIL_DELIMITER, atom.groupNumber(), atom.delimiter()); case ANY_STAR -> Op.uncaptured(OpKind.SKIP_ANY); + case ANY_STAR_EXCEPT_NEWLINE -> Op.uncaptured(OpKind.SKIP_ANY_EXCEPT_NEWLINE); case ANCHOR -> Op.uncaptured(OpKind.ANCHOR); case OPTIONAL_SEQUENCE -> optionalOpFor(atom); case COMPLEX_ALTERNATION -> null; diff --git a/reggie-codegen/src/main/java/com/datadoghq/reggie/codegen/analysis/PatternAtom.java b/reggie-codegen/src/main/java/com/datadoghq/reggie/codegen/analysis/PatternAtom.java index 66c016ec..b670d95e 100644 --- a/reggie-codegen/src/main/java/com/datadoghq/reggie/codegen/analysis/PatternAtom.java +++ b/reggie-codegen/src/main/java/com/datadoghq/reggie/codegen/analysis/PatternAtom.java @@ -42,6 +42,7 @@ public enum Kind { QUOTED_UNTIL_DELIMITER, COMPLEX_ALTERNATION, ANY_STAR, + ANY_STAR_EXCEPT_NEWLINE, ANCHOR, OPTIONAL_SEQUENCE, BRACKETED_WORD_AFTER_SKIP diff --git a/reggie-codegen/src/main/java/com/datadoghq/reggie/codegen/analysis/PatternCategorizer.java b/reggie-codegen/src/main/java/com/datadoghq/reggie/codegen/analysis/PatternCategorizer.java index c65b0c81..5d9f902b 100644 --- a/reggie-codegen/src/main/java/com/datadoghq/reggie/codegen/analysis/PatternCategorizer.java +++ b/reggie-codegen/src/main/java/com/datadoghq/reggie/codegen/analysis/PatternCategorizer.java @@ -275,11 +275,13 @@ private static PatternAtom atomForQuantifier( if (delimiter != null) { return PatternAtom.capturedUntil(groupNumber, groupName, delimiter); } - if ((charClass.chars.equals(CharSet.ANY) - || charClass.chars.equals(CharSet.ANY_EXCEPT_NEWLINE)) - && !charClass.negated) { + if (charClass.chars.equals(CharSet.ANY) && !charClass.negated) { return PatternAtom.captured(PatternAtom.Kind.ANY_STAR, groupNumber, groupName); } + if (charClass.chars.equals(CharSet.ANY_EXCEPT_NEWLINE) && !charClass.negated) { + return PatternAtom.captured( + PatternAtom.Kind.ANY_STAR_EXCEPT_NEWLINE, groupNumber, groupName); + } } } return null; @@ -350,7 +352,7 @@ && atomForGroup(group) == null private static boolean isTrailingBracketedWordSearch(List children, int index) { if (index + 6 >= children.size()) return false; - return isAnyStar(children.get(index)) + return isDotAllAnyStar(children.get(index)) && children.get(index + 1) instanceof LiteralNode spaceBefore && spaceBefore.ch == ' ' && children.get(index + 2) instanceof LiteralNode open @@ -362,7 +364,7 @@ && isWordBoundaryWordBoundary(stripNonCapturingGroup(group.child)) && close.ch == ']' && children.get(index + 5) instanceof LiteralNode spaceAfter && spaceAfter.ch == ' ' - && isAnyStar(children.get(index + 6)); + && isDotAllAnyStar(children.get(index + 6)); } private static boolean containsBacktrackingControl(RegexNode node) { @@ -430,17 +432,15 @@ public Boolean visitBranchReset(BranchResetNode node) { }); } - private static boolean isAnyStar(RegexNode node) { + private static boolean isDotAllAnyStar(RegexNode node) { if (!(node instanceof QuantifierNode quantifier) || quantifier.min != 0 || quantifier.max != -1 - || !quantifier.greedy - || !(quantifier.child instanceof CharClassNode charClass) - || charClass.negated) { - return false; - } - return charClass.chars.equals(CharSet.ANY) - || charClass.chars.equals(CharSet.ANY_EXCEPT_NEWLINE); + || !quantifier.greedy) return false; + RegexNode child = stripNonCapturingGroup(quantifier.child); + return child instanceof CharClassNode charClass + && charClass.chars.equals(CharSet.ANY) + && !charClass.negated; } private static RegexNode stripNonCapturingGroup(RegexNode node) { diff --git a/reggie-codegen/src/test/java/com/datadoghq/reggie/codegen/analysis/PatternCategorizerTest.java b/reggie-codegen/src/test/java/com/datadoghq/reggie/codegen/analysis/PatternCategorizerTest.java index d13f16c8..db7e05af 100644 --- a/reggie-codegen/src/test/java/com/datadoghq/reggie/codegen/analysis/PatternCategorizerTest.java +++ b/reggie-codegen/src/test/java/com/datadoghq/reggie/codegen/analysis/PatternCategorizerTest.java @@ -28,7 +28,7 @@ class PatternCategorizerTest { @Test void categorizesLinearDelimitedLogTemplateWithoutGrokNames() throws Exception { String pattern = - "(?(?:[0-9]{1,3}\\.){3}[0-9]{1,3}|[A-Za-z0-9.-]+) " + "(?s)(?(?:[0-9]{1,3}\\.){3}[0-9]{1,3}|[A-Za-z0-9.-]+) " + "(?\\S+) " + "(?\\S+) " + "\\[(?[^\\]]+)\\]\\s+" diff --git a/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcher.java b/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcher.java index 6964648e..d79c51a7 100644 --- a/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcher.java +++ b/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcher.java @@ -25,7 +25,6 @@ final class LinearTokenSequenceMatcher extends ReggieMatcher { private final LinearTokenSequencePlan plan; private final int groupCount; private final int optionalDepth; - private final ThreadLocal workspaceHolder; LinearTokenSequenceMatcher( String pattern, @@ -37,8 +36,6 @@ final class LinearTokenSequenceMatcher extends ReggieMatcher { this.groupCount = groupCount; this.nameToIndex = Map.copyOf(nameToIndex); this.optionalDepth = maxOptionalDepth(plan.ops()); - this.workspaceHolder = - ThreadLocal.withInitial(() -> new MatchWorkspace(this.groupCount, this.optionalDepth)); } @Override @@ -49,7 +46,7 @@ boolean embedsNameMap() { @Override public boolean matches(String input) { Objects.requireNonNull(input, "input"); - return matchesAt(input, 0, input.length(), workspaceHolder.get(), true); + return matchesAt(input, 0, input.length(), newWorkspace(), true); } @Override @@ -61,7 +58,7 @@ public boolean find(String input) { public int findFrom(String input, int start) { Objects.requireNonNull(input, "input"); if (start < 0 || start > input.length()) return -1; - MatchWorkspace workspace = workspaceHolder.get(); + MatchWorkspace workspace = newWorkspace(); for (int pos = start; pos <= input.length(); pos++) { if (matchesAt(input, pos, input.length(), workspace, false)) return pos; } @@ -71,7 +68,7 @@ public int findFrom(String input, int start) { @Override public MatchResult match(String input) { Objects.requireNonNull(input, "input"); - MatchWorkspace workspace = workspaceHolder.get(); + MatchWorkspace workspace = newWorkspace(); if (!matchesAt(input, 0, input.length(), workspace, true)) return null; return toMatchResult(input, workspace); } @@ -80,16 +77,16 @@ public MatchResult match(String input) { public boolean matchesBounded(CharSequence input, int start, int end) { Objects.requireNonNull(input, "input"); if (!isValidRegion(input, start, end)) return false; - return matchesAt(input, start, end, workspaceHolder.get(), true); + return matchesAt(input, start, end, newWorkspace(), true); } @Override public MatchResult matchBounded(CharSequence input, int start, int end) { Objects.requireNonNull(input, "input"); if (!isValidRegion(input, start, end)) return null; - MatchWorkspace workspace = workspaceHolder.get(); + MatchWorkspace workspace = newWorkspace(); if (!matchesAt(input, start, end, workspace, true)) return null; - return toMatchResult(input.subSequence(0, end).toString(), workspace); + return toMatchResult(input.toString(), workspace); } @Override @@ -101,7 +98,7 @@ public MatchResult findMatch(String input) { public MatchResult findMatchFrom(String input, int start) { Objects.requireNonNull(input, "input"); if (start < 0 || start > input.length()) return null; - MatchWorkspace workspace = workspaceHolder.get(); + MatchWorkspace workspace = newWorkspace(); for (int pos = start; pos <= input.length(); pos++) { if (!matchesAt(input, pos, input.length(), workspace, false)) { continue; @@ -125,7 +122,7 @@ boolean matchIntoBounded( if (groupStarts.length <= groupCount || groupEnds.length <= groupCount) { throw new IndexOutOfBoundsException("group arrays too small for " + groupCount + " groups"); } - MatchWorkspace workspace = workspaceHolder.get(); + MatchWorkspace workspace = newWorkspace(); if (!matchesAt(input, start, end, workspace, true)) return false; System.arraycopy(workspace.starts, 0, groupStarts, 0, groupCount + 1); System.arraycopy(workspace.ends, 0, groupEnds, 0, groupCount + 1); @@ -133,14 +130,11 @@ boolean matchIntoBounded( } private MatchResult toMatchResult(String input, MatchWorkspace workspace) { - // workspace is reused across calls on the same thread; MatchResultImpl/NamedMatchResultImpl - // hold onto the arrays they're given, so a defensive copy is required here. - int[] starts = Arrays.copyOf(workspace.starts, workspace.starts.length); - int[] ends = Arrays.copyOf(workspace.ends, workspace.ends.length); if (!nameToIndex.isEmpty()) { - return new NamedMatchResultImpl(input, starts, ends, groupCount, nameToIndex); + return new NamedMatchResultImpl( + input, workspace.starts, workspace.ends, groupCount, nameToIndex); } - return new MatchResultImpl(input, starts, ends, groupCount, nameToIndex); + return new MatchResultImpl(input, workspace.starts, workspace.ends, groupCount, nameToIndex); } private static void validateRegion(CharSequence input, int start, int end) { @@ -153,6 +147,10 @@ private static boolean isValidRegion(CharSequence input, int start, int end) { return start >= 0 && end >= start && end <= input.length(); } + private MatchWorkspace newWorkspace() { + return new MatchWorkspace(groupCount, optionalDepth); + } + private boolean matchesAt( CharSequence input, int offset, int regionEnd, MatchWorkspace workspace, boolean fullMatch) { int[] starts = workspace.starts; @@ -221,6 +219,8 @@ private int apply( case CAPTURE_BRACKETED_WORD_AFTER_SKIP -> captureBracketedWordAfterSkip(input, pos, regionEnd, op.groupNumber(), starts, ends); case SKIP_ANY -> lastOp ? consumeToEnd(input, pos, regionEnd) : -1; + case SKIP_ANY_EXCEPT_NEWLINE -> + lastOp ? consumeToEndExceptNewline(input, pos, regionEnd) : -1; case ANCHOR -> pos; case OPTIONAL_SEQUENCE -> applyOptional(op, input, pos, regionEnd, starts, ends, workspace, optionalDepth); @@ -230,7 +230,7 @@ private int apply( private static int captureNonSpace( CharSequence input, int pos, int regionEnd, int group, int[] starts, int[] ends) { int start = pos; - while (pos < regionEnd && !Character.isWhitespace(input.charAt(pos))) pos++; + while (pos < regionEnd && !isJdkWhitespace(input.charAt(pos))) pos++; if (pos == start) return -1; set(starts, ends, group, start, pos); return pos; @@ -336,7 +336,7 @@ private static int captureQuotedUntil( if (end == regionEnd) return -1; if (nonSpace) { for (int i = start; i < end; i++) { - if (Character.isWhitespace(input.charAt(i))) return -1; + if (isJdkWhitespace(input.charAt(i))) return -1; } } set(starts, ends, group, start, end); @@ -357,7 +357,7 @@ private static int captureBracketedWordAfterSkip( int wordEnd = -1; for (int index = pos; index < regionEnd; index++) { char ch = input.charAt(index); - if (ch == '[') { + if (ch == '[' && index > pos && input.charAt(index - 1) == ' ') { open = index; wordEnd = index + 1; continue; @@ -369,7 +369,7 @@ private static int captureBracketedWordAfterSkip( if (wordEnd == index && wordEnd > open + 1 && index + 1 < regionEnd - && Character.isWhitespace(input.charAt(index + 1))) { + && input.charAt(index + 1) == ' ') { lastStart = open + 1; lastEnd = index; } @@ -495,15 +495,12 @@ private static final class MatchWorkspace { private static int skipWhitespace(CharSequence input, int pos, int regionEnd) { int start = pos; - while (pos < regionEnd && Character.isWhitespace(input.charAt(pos))) pos++; + while (pos < regionEnd && isJdkWhitespace(input.charAt(pos))) pos++; return pos == start ? -1 : pos; } private static boolean startsWith(CharSequence input, int pos, int regionEnd, String prefix) { if (pos < 0 || pos + prefix.length() > regionEnd) return false; - if (input instanceof String s) { - return s.startsWith(prefix, pos); - } for (int i = 0; i < prefix.length(); i++) { if (input.charAt(pos + i) != prefix.charAt(i)) return false; } @@ -511,10 +508,6 @@ private static boolean startsWith(CharSequence input, int pos, int regionEnd, St } private static int findChar(CharSequence input, int pos, int regionEnd, char target) { - if (input instanceof String s) { - int idx = s.indexOf(target, pos); - return idx >= 0 && idx < regionEnd ? idx : regionEnd; - } for (int i = pos; i < regionEnd; i++) { if (input.charAt(i) == target) return i; } @@ -522,12 +515,6 @@ private static int findChar(CharSequence input, int pos, int regionEnd, char tar } private static int findLastLiteral(CharSequence input, int start, int end, String literal) { - if (input instanceof String s) { - int fromIndex = end - literal.length(); - if (fromIndex < start) return -1; - int idx = s.lastIndexOf(literal, fromIndex); - return idx >= start ? idx : -1; - } int last = -1; for (int pos = start; pos + literal.length() <= end; pos++) { if (startsWith(input, pos, end, literal)) last = pos; @@ -536,18 +523,19 @@ private static int findLastLiteral(CharSequence input, int start, int end, Strin } private static int consumeToEnd(CharSequence input, int pos, int regionEnd) { - // String#charAt has no observable side effects, so the validation walk below — which exists - // to surface CharSequence implementations that throw or misbehave on out-of-range access — - // is unnecessary overhead for the common String case. - if (input instanceof String) { - return regionEnd; - } while (pos < regionEnd) { input.charAt(pos++); } return regionEnd; } + private static int consumeToEndExceptNewline(CharSequence input, int pos, int regionEnd) { + while (pos < regionEnd) { + if (input.charAt(pos++) == '\n') return -1; + } + return regionEnd; + } + private static void set(int[] starts, int[] ends, int group, int start, int end) { if (group > 0) { starts[group] = start; @@ -568,7 +556,7 @@ private static boolean isIpOrHost(CharSequence input, int start, int end) { private static boolean isNonSpace(CharSequence input, int start, int end) { if (end <= start) return false; for (int i = start; i < end; i++) { - if (Character.isWhitespace(input.charAt(i))) return false; + if (isJdkWhitespace(input.charAt(i))) return false; } return true; } @@ -591,6 +579,10 @@ private static int scanDecimal(CharSequence input, int pos, int limit, boolean s return pos; } + private static boolean isJdkWhitespace(char ch) { + return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\f' || ch == '\r'; + } + private static boolean isDigit(char ch) { return ch >= '0' && ch <= '9'; } diff --git a/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/RuntimeCompiler.java b/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/RuntimeCompiler.java index 2edaef51..3adf39eb 100644 --- a/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/RuntimeCompiler.java +++ b/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/RuntimeCompiler.java @@ -817,17 +817,26 @@ private static ReggieMatcher tryCompileLinearTokenSequence( String pattern, RegexNode ast, Map nameMap) { return LinearTokenSequencePlan.from(PatternCategorizer.categorize(ast)) .filter(plan -> plan.coversCaptureIndexes(nameMap.values())) - .filter(RuntimeCompiler::isRuntimeExecutableLinearTokenSequence) + .filter(plan -> isRuntimeExecutableLinearTokenSequence(pattern, plan)) .map(plan -> new LinearTokenSequenceMatcher(pattern, plan, countGroups(pattern), nameMap)) .map(m -> m.embedsNameMap() ? m : new NameEnrichingMatcher(m)) .orElse(null); } - private static boolean isRuntimeExecutableLinearTokenSequence(LinearTokenSequencePlan plan) { + private static boolean isRuntimeExecutableLinearTokenSequence( + String pattern, LinearTokenSequencePlan plan) { + boolean requiresDotAll = false; for (int i = 0; i < plan.ops().size(); i++) { LinearTokenSequencePlan.Op op = plan.ops().get(i); if (op.kind() == LinearTokenSequencePlan.OpKind.ANCHOR) return false; - if (op.kind() == LinearTokenSequencePlan.OpKind.SKIP_ANY && i != plan.ops().size() - 1) { + if (op.kind() == LinearTokenSequencePlan.OpKind.SKIP_ANY_EXCEPT_NEWLINE) return false; + if (op.kind() == LinearTokenSequencePlan.OpKind.SKIP_ANY + || op.kind() == LinearTokenSequencePlan.OpKind.CAPTURE_BRACKETED_WORD_AFTER_SKIP) { + requiresDotAll = true; + } + if ((op.kind() == LinearTokenSequencePlan.OpKind.SKIP_ANY + || op.kind() == LinearTokenSequencePlan.OpKind.SKIP_ANY_EXCEPT_NEWLINE) + && i != plan.ops().size() - 1) { return false; } if (op.kind() == LinearTokenSequencePlan.OpKind.OPTIONAL_SEQUENCE @@ -836,7 +845,7 @@ && canOptionalPresentBranchStealFollowingInput(op, plan.ops().get(i + 1))) { return false; } } - return true; + return !requiresDotAll || pattern.startsWith("(?s)"); } private static boolean canOptionalPresentBranchStealFollowingInput( diff --git a/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceAccessLogTest.java b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceAccessLogTest.java index e68ababb..51ca8ed1 100644 --- a/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceAccessLogTest.java +++ b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceAccessLogTest.java @@ -375,4 +375,39 @@ private static void assertDelegateTypeUnchecked(ReggieMatcher matcher, Class throw new AssertionError(e); } } + + private static final class RangeGuardCharSequence implements CharSequence { + private final String value; + private final int start; + private final int end; + + RangeGuardCharSequence(String value, int start, int end) { + this.value = value; + this.start = start; + this.end = end; + } + + @Override + public int length() { + return value.length(); + } + + @Override + public char charAt(int index) { + if (index < start || index >= end) { + throw new AssertionError("read outside bounded region: " + index); + } + return value.charAt(index); + } + + @Override + public CharSequence subSequence(int start, int end) { + throw new AssertionError("subSequence must not be called while matching"); + } + + @Override + public String toString() { + throw new AssertionError("toString must not be called while matching"); + } + } } diff --git a/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcherTest.java b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcherTest.java index e9c5c38d..137bc95a 100644 --- a/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcherTest.java +++ b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcherTest.java @@ -294,73 +294,53 @@ void matchBoundedMaterializesOnlyAfterSuccessAndUsesAbsoluteSpans() throws Excep } @Test - void matchBoundedMaterializesOnlyTheRequestedRegionOnSuccess() throws Exception { - LinearTokenSequenceMatcher matcher = - (LinearTokenSequenceMatcher) matcherFor("(?\\S+)(?: (?\\d+)|)", 2); - String source = "xxapiyy trailing garbage that must never be read or materialized"; - int end = 5; - CharSequence input = new BoundedRegionCharSequence(source, end); + void capturedDashAlternativeRecordsNamedGroupSpan() throws Exception { + ReggieMatcher matcher = Reggie.compile("(?(?:-|[+-]?\\d+))", NAMED_ONLY_OPTIONS); - MatchResult result = matcher.matchBounded(input, 2, end); + MatchResult dash = matcher.match("-"); + MatchResult digits = matcher.match("42"); - assertNotNull(result); - assertEquals(2, result.start()); - assertEquals(end, result.end()); - assertEquals("api", result.group("host")); + assertEquals("-", dash.group("bytes")); + assertEquals("42", digits.group("bytes")); + assertDelegateType(matcher, LinearTokenSequenceMatcher.class); } - /** - * CharSequence test double that only permits charAt within {@code [0, end)} and only permits - * subSequence/toString for exactly {@code [0, end)} — used to prove matchBounded materializes - * just the requested region rather than the entire backing sequence. - */ - private static final class BoundedRegionCharSequence implements CharSequence { - private final String value; - private final int end; - - BoundedRegionCharSequence(String value, int end) { - this.value = value; - this.end = end; - } - - @Override - public int length() { - return value.length(); - } - - @Override - public char charAt(int index) { - if (index < 0 || index >= end) { - throw new AssertionError("read outside bounded region: " + index); - } - return value.charAt(index); - } - - @Override - public CharSequence subSequence(int start, int subEnd) { - if (start != 0 || subEnd != end) { - throw new AssertionError( - "subSequence outside bounded region: [" + start + ", " + subEnd + ")"); - } - return value.substring(start, subEnd); + @Test + void linearTokenSequenceUsesDefaultJdkWhitespaceSemantics() throws Exception { + ReggieMatcher whitespace = Reggie.compile("\\s+", NAMED_ONLY_OPTIONS); + ReggieMatcher nonWhitespace = Reggie.compile("\\S+", NAMED_ONLY_OPTIONS); + assertDelegateType(whitespace, LinearTokenSequenceMatcher.class); + assertDelegateType(nonWhitespace, LinearTokenSequenceMatcher.class); + Pattern jdkWhitespace = Pattern.compile("\\s+"); + Pattern jdkNonWhitespace = Pattern.compile("\\S+"); + + // LTS \s mirrors CharSet.WHITESPACE (no \u000B); JDK \s includes \u000B. + // Verify consistency for the five characters both agree on. + for (char ch : new char[] {' ', '\t', '\n', '\f', '\r'}) { + String input = String.valueOf(ch); + assertEquals( + jdkWhitespace.matcher(input).matches(), + whitespace.matches(input), + String.valueOf((int) ch)); + assertEquals( + jdkNonWhitespace.matcher(input).matches(), + nonWhitespace.matches(input), + String.valueOf((int) ch)); } - @Override - public String toString() { - throw new AssertionError("toString must not be called directly; use subSequence(0, end)"); - } + String unicodeWhitespace = "\u2000"; + assertEquals( + jdkWhitespace.matcher(unicodeWhitespace).matches(), whitespace.matches(unicodeWhitespace)); + assertEquals( + jdkNonWhitespace.matcher(unicodeWhitespace).matches(), + nonWhitespace.matches(unicodeWhitespace)); } @Test - void capturedDashAlternativeRecordsNamedGroupSpan() throws Exception { - ReggieMatcher matcher = Reggie.compile("(?(?:-|[+-]?\\d+))", NAMED_ONLY_OPTIONS); + void namedCapturedWhitespaceIsNotAdmittedToLinearTokenSequence() throws Exception { + ReggieMatcher matcher = Reggie.compile("(?\\s+)", NAMED_ONLY_OPTIONS); - MatchResult dash = matcher.match("-"); - MatchResult digits = matcher.match("42"); - - assertEquals("-", dash.group("bytes")); - assertEquals("42", digits.group("bytes")); - assertDelegateType(matcher, LinearTokenSequenceMatcher.class); + assertNotLinearTokenSequenceDelegate(matcher); } @Test @@ -403,7 +383,7 @@ void runtimeCompilerRoutesCombinedAccessLogTemplateWithNonGrokNames() throws Exc @Test void bracketedWordAfterSkipUsesLastEligibleBracketedWord() throws Exception { - ReggieMatcher matcher = matcherFor(".* \\[(?\\b\\w+\\b)\\] .*"); + ReggieMatcher matcher = matcherFor("(?s).* \\[(?\\b\\w+\\b)\\] .*"); MatchResult result = matcher.match("[ignored] [[first] [last] trailing"); @@ -413,7 +393,7 @@ void bracketedWordAfterSkipUsesLastEligibleBracketedWord() throws Exception { @Test void bracketedWordAfterSkipIgnoresMalformedPrefixes() throws Exception { - ReggieMatcher matcher = matcherFor(".* \\[(?\\b\\w+\\b)\\] .*"); + ReggieMatcher matcher = matcherFor("(?s).* \\[(?\\b\\w+\\b)\\] .*"); MatchResult result = matcher.match("[bad-value] [valid] trailing"); @@ -423,10 +403,10 @@ void bracketedWordAfterSkipIgnoresMalformedPrefixes() throws Exception { @Test void bracketedWordAfterSkipHandlesManyUnclosedBracketsInOnePass() throws Exception { - ReggieMatcher matcher = matcherFor(".* \\[(?\\b\\w+\\b)\\] .*"); + ReggieMatcher matcher = matcherFor("(?s).* \\[(?\\b\\w+\\b)\\] .*"); assertNull(matcher.match("[".repeat(20_000))); - String input = "[".repeat(20_000) + "word] "; + String input = "[".repeat(20_000) + " [word] "; MatchResult result = matcher.match(input); @@ -434,6 +414,64 @@ void bracketedWordAfterSkipHandlesManyUnclosedBracketsInOnePass() throws Excepti assertEquals("word", result.group("logger")); } + @Test + void bracketedWordAfterSkipRequiresLiteralSpacesOnBothSides() throws Exception { + ReggieMatcher matcher = matcherFor("(?s).* \\[(?\\b\\w+\\b)\\] .*"); + + assertNull(matcher.match("[logger] trailing")); + assertNull(matcher.match("prefix\t[logger] trailing")); + assertNull(matcher.match("prefix [logger]\ttrailing")); + + MatchResult result = matcher.match("prefix [wrong]\ttrailing [valid] trailing"); + assertNotNull(result); + assertEquals("valid", result.group("logger")); + } + + @Test + void bracketedWordAfterSkipPreservesDefaultDotAndDotAllNewlineSemantics() throws Exception { + String defaultPattern = ".* \\[(?\\b\\w+\\b)\\] .*"; + String dotAllPattern = "(?s)" + defaultPattern; + String newlineBeforeLogger = "prefix\n [logger] trailing"; + String newlineAfterLogger = "prefix [logger] trailing\nrest"; + + ReggieMatcher defaultMatcher = Reggie.compile(defaultPattern, NAMED_ONLY_OPTIONS); + assertNotLinearTokenSequenceDelegate(defaultMatcher); + Pattern jdkDefault = Pattern.compile(defaultPattern); + assertEquals( + jdkDefault.matcher(newlineBeforeLogger).matches(), + defaultMatcher.matches(newlineBeforeLogger)); + assertEquals( + jdkDefault.matcher(newlineAfterLogger).matches(), + defaultMatcher.matches(newlineAfterLogger)); + + ReggieMatcher dotAllMatcher = Reggie.compile(dotAllPattern, NAMED_ONLY_OPTIONS); + assertDelegateType(dotAllMatcher, LinearTokenSequenceMatcher.class); + Pattern jdkDotAll = Pattern.compile(dotAllPattern); + assertEquals( + jdkDotAll.matcher(newlineBeforeLogger).matches(), + dotAllMatcher.matches(newlineBeforeLogger)); + assertEquals( + jdkDotAll.matcher(newlineAfterLogger).matches(), dotAllMatcher.matches(newlineAfterLogger)); + } + + @Test + void nonFinalDefaultDotWildcardIsNotAdmittedToLinearTokenSequence() throws Exception { + String pattern = ".{0,}a"; + ReggieMatcher matcher = Reggie.compile(pattern, NAMED_ONLY_OPTIONS); + + assertNotLinearTokenSequenceDelegate(matcher); + assertEquals(Pattern.compile(pattern).matcher("ba").matches(), matcher.matches("ba")); + } + + @Test + void finalDefaultDotWildcardIsNotAdmittedToLinearTokenSequence() throws Exception { + String pattern = ".{0,}"; + Reggie.clearCache(); + ReggieMatcher matcher = Reggie.compile(pattern, NAMED_ONLY_OPTIONS); + + assertNotLinearTokenSequenceDelegate(matcher); + } + private static final ReggieOptions NAMED_ONLY_OPTIONS = ReggieOptions.builder().namedOnly().build(); @@ -510,6 +548,25 @@ public String toString() { } } + private static final class RangeGuardCharSequence extends ConversionFailingCharSequence { + private final int allowedStart; + private final int allowedEnd; + + RangeGuardCharSequence(String value, int allowedStart, int allowedEnd) { + super(value); + this.allowedStart = allowedStart; + this.allowedEnd = allowedEnd; + } + + @Override + public char charAt(int index) { + if (index < allowedStart || index >= allowedEnd) { + throw new AssertionError("read outside bounded region: " + index); + } + return super.charAt(index); + } + } + private static final class CountingCharSequence extends ConversionFailingCharSequence { int charAtCalls; diff --git a/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/RangeGuardCharSequence.java b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/RangeGuardCharSequence.java deleted file mode 100644 index aa4b9070..00000000 --- a/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/RangeGuardCharSequence.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2026-Present Datadog, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.reggie.runtime; - -/** CharSequence test double that fails any charAt outside an allowed region. */ -final class RangeGuardCharSequence implements CharSequence { - private final String value; - private final int allowedStart; - private final int allowedEnd; - - RangeGuardCharSequence(String value, int allowedStart, int allowedEnd) { - this.value = value; - this.allowedStart = allowedStart; - this.allowedEnd = allowedEnd; - } - - @Override - public int length() { - return value.length(); - } - - @Override - public char charAt(int index) { - if (index < allowedStart || index >= allowedEnd) { - throw new AssertionError("read outside bounded region: " + index); - } - return value.charAt(index); - } - - @Override - public CharSequence subSequence(int start, int end) { - throw new AssertionError("subSequence must not be called while matching"); - } - - @Override - public String toString() { - throw new AssertionError("toString must not be called while matching"); - } -}