diff --git a/reggie-codegen/src/main/java/com/datadoghq/reggie/codegen/analysis/LinearTokenSequencePlan.java b/reggie-codegen/src/main/java/com/datadoghq/reggie/codegen/analysis/LinearTokenSequencePlan.java index a9961f41..01af977c 100644 --- a/reggie-codegen/src/main/java/com/datadoghq/reggie/codegen/analysis/LinearTokenSequencePlan.java +++ b/reggie-codegen/src/main/java/com/datadoghq/reggie/codegen/analysis/LinearTokenSequencePlan.java @@ -16,8 +16,12 @@ package com.datadoghq.reggie.codegen.analysis; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Optional; +import java.util.Set; /** Executable, deterministic plan for a categorized linear-token-sequence regex. */ public record LinearTokenSequencePlan(List ops, int groupCount) { @@ -74,6 +78,37 @@ static Op optional(List children) { ops = List.copyOf(ops); } + /** + * Returns whether this plan has a capture-producing operation for every requested group index. + * + *

The check recurses into optional sequences because an optional capture remains observable + * when its present branch matches. It deliberately does not use {@link #groupCount()}: that value + * is a top-level construction detail and may omit captures nested in an optional sequence. + */ + public boolean coversCaptureIndexes(Collection requiredIndexes) { + Objects.requireNonNull(requiredIndexes, "requiredIndexes"); + Set remaining = new HashSet<>(); + for (Integer index : requiredIndexes) { + if (index == null || index <= 0) { + throw new IllegalArgumentException("capture group indexes must be positive: " + index); + } + remaining.add(index); + } + removeCoveredIndexes(ops, remaining); + return remaining.isEmpty(); + } + + private static void removeCoveredIndexes(List ops, Set remaining) { + for (Op op : ops) { + if (op.groupNumber() > 0) { + remaining.remove(op.groupNumber()); + } + if (!op.children().isEmpty()) { + removeCoveredIndexes(op.children(), remaining); + } + } + } + /** Converts categorizer atoms into a closed, executable linear-token-sequence plan. */ public static Optional from(PatternCategorization categorization) { if (!categorization.isLinearTokenSequence()) return Optional.empty(); diff --git a/reggie-codegen/src/test/java/com/datadoghq/reggie/codegen/analysis/LinearTokenSequencePlanTest.java b/reggie-codegen/src/test/java/com/datadoghq/reggie/codegen/analysis/LinearTokenSequencePlanTest.java index 6a082614..aa31a28e 100644 --- a/reggie-codegen/src/test/java/com/datadoghq/reggie/codegen/analysis/LinearTokenSequencePlanTest.java +++ b/reggie-codegen/src/test/java/com/datadoghq/reggie/codegen/analysis/LinearTokenSequencePlanTest.java @@ -16,6 +16,8 @@ package com.datadoghq.reggie.codegen.analysis; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import com.datadoghq.reggie.codegen.ast.RegexNode; @@ -63,6 +65,41 @@ void failsClosedForGeneralRegexCategories() throws Exception { assertTrue(LinearTokenSequencePlan.from(categorization).isEmpty()); } + @Test + void coverageIncludesCapturesNestedInOptionalSequences() throws Exception { + LinearTokenSequencePlan plan = planFor("(?:|(?\\S+))"); + + assertEquals(0, plan.groupCount()); + assertTrue(plan.coversCaptureIndexes(List.of(1))); + assertFalse(plan.coversCaptureIndexes(List.of(2))); + } + + @Test + void coverageRecursesThroughMultipleNestedOptionalLevels() throws Exception { + LinearTokenSequencePlan plan = planFor("(?:(?:(?:(?\\d+))|)|)"); + + assertTrue(plan.coversCaptureIndexes(List.of(1))); + assertFalse(plan.coversCaptureIndexes(List.of(2))); + } + + @Test + void coverageRequiresEveryNamedCaptureNotOnlyTheLargestGroupNumber() throws Exception { + RegexParser parser = new RegexParser(); + RegexNode ast = parser.parse("(?(?:-|(?[+-]?\\d+)))"); + LinearTokenSequencePlan plan = + LinearTokenSequencePlan.from(PatternCategorizer.categorize(ast)).orElseThrow(); + + assertTrue(plan.coversCaptureIndexes(List.of(2))); + assertFalse(plan.coversCaptureIndexes(parser.getGroupNameMap().values())); + } + + @Test + void coverageRejectsNonPositiveGroupIndexes() throws Exception { + LinearTokenSequencePlan plan = planFor("(?\\S+)"); + + assertThrows(IllegalArgumentException.class, () -> plan.coversCaptureIndexes(List.of(0))); + } + private static LinearTokenSequencePlan planFor(String pattern) throws Exception { return LinearTokenSequencePlan.from(categorize(pattern)).orElseThrow(); } diff --git a/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcher.java b/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcher.java index addb33fd..a08dd8c3 100644 --- a/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcher.java +++ b/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcher.java @@ -24,10 +24,12 @@ final class LinearTokenSequenceMatcher extends ReggieMatcher { private final LinearTokenSequencePlan plan; private final int groupCount; - private final int[] scratchStarts; - private final int[] scratchEnds; - private final int[][] optionalScratchStarts; - private final int[][] optionalScratchEnds; + private final int optionalDepth; + private final ThreadLocal workspace; + + private MatchWorkspace workspace() { + return workspace.get(); + } LinearTokenSequenceMatcher( String pattern, @@ -38,11 +40,8 @@ final class LinearTokenSequenceMatcher extends ReggieMatcher { this.plan = plan; this.groupCount = groupCount; this.nameToIndex = Map.copyOf(nameToIndex); - this.scratchStarts = new int[groupCount + 1]; - this.scratchEnds = new int[groupCount + 1]; - int optionalDepth = maxOptionalDepth(plan.ops()); - this.optionalScratchStarts = new int[optionalDepth][groupCount + 1]; - this.optionalScratchEnds = new int[optionalDepth][groupCount + 1]; + this.optionalDepth = maxOptionalDepth(plan.ops()); + this.workspace = ThreadLocal.withInitial(() -> new MatchWorkspace(groupCount, optionalDepth)); } @Override @@ -52,7 +51,8 @@ boolean embedsNameMap() { @Override public boolean matches(String input) { - return matchInto(input, scratchStarts, scratchEnds); + Objects.requireNonNull(input, "input"); + return matchesAt(input, 0, workspace(), true); } @Override @@ -64,17 +64,20 @@ public boolean find(String input) { public int findFrom(String input, int start) { Objects.requireNonNull(input, "input"); if (start < 0 || start > input.length()) return -1; + MatchWorkspace ws = workspace(); for (int pos = start; pos <= input.length(); pos++) { - if (matchesAt(input, pos, scratchStarts, scratchEnds, false)) return pos; + if (matchesAt(input, pos, ws, false)) return pos; } return -1; } @Override public MatchResult match(String input) { - int[] starts = new int[groupCount + 1]; - int[] ends = new int[groupCount + 1]; - if (!matchInto(input, starts, ends)) return null; + Objects.requireNonNull(input, "input"); + MatchWorkspace ws = workspace(); + if (!matchesAt(input, 0, ws, true)) return null; + int[] starts = Arrays.copyOf(ws.starts, groupCount + 1); + int[] ends = Arrays.copyOf(ws.ends, groupCount + 1); if (!nameToIndex.isEmpty()) { return new NamedMatchResultImpl(input, starts, ends, groupCount, nameToIndex); } @@ -104,15 +107,21 @@ public MatchResult findMatch(String input) { @Override public MatchResult findMatchFrom(String input, int start) { - int pos = findFrom(input, start); - if (pos < 0) return null; - int[] starts = new int[groupCount + 1]; - int[] ends = new int[groupCount + 1]; - if (!matchesAt(input, pos, starts, ends, false)) return null; - if (!nameToIndex.isEmpty()) { - return new NamedMatchResultImpl(input, starts, ends, groupCount, nameToIndex); + Objects.requireNonNull(input, "input"); + if (start < 0 || start > input.length()) return null; + MatchWorkspace ws = workspace(); + for (int pos = start; pos <= input.length(); pos++) { + if (!matchesAt(input, pos, ws, false)) { + continue; + } + int[] starts = Arrays.copyOf(ws.starts, groupCount + 1); + int[] ends = Arrays.copyOf(ws.ends, groupCount + 1); + if (!nameToIndex.isEmpty()) { + return new NamedMatchResultImpl(input, starts, ends, groupCount, nameToIndex); + } + return new MatchResultImpl(input, starts, ends, groupCount, nameToIndex); } - return new MatchResultImpl(input, starts, ends, groupCount, nameToIndex); + return null; } @Override @@ -123,13 +132,16 @@ public boolean matchInto(String input, int[] groupStarts, int[] groupEnds) { if (groupStarts.length <= groupCount || groupEnds.length <= groupCount) { throw new IndexOutOfBoundsException("group arrays too small for " + groupCount + " groups"); } - if (!matchesAt(input, 0, scratchStarts, scratchEnds, true)) return false; - System.arraycopy(scratchStarts, 0, groupStarts, 0, groupCount + 1); - System.arraycopy(scratchEnds, 0, groupEnds, 0, groupCount + 1); + MatchWorkspace ws = workspace(); + if (!matchesAt(input, 0, ws, true)) return false; + System.arraycopy(ws.starts, 0, groupStarts, 0, groupCount + 1); + System.arraycopy(ws.ends, 0, groupEnds, 0, groupCount + 1); return true; } - private boolean matchesAt(String input, int offset, int[] starts, int[] ends, boolean fullMatch) { + private boolean matchesAt(String input, int offset, MatchWorkspace workspace, boolean fullMatch) { + int[] starts = workspace.starts; + int[] ends = workspace.ends; Arrays.fill(starts, -1); Arrays.fill(ends, -1); starts[0] = offset; @@ -144,7 +156,7 @@ private boolean matchesAt(String input, int offset, int[] starts, int[] ends, bo i++; continue; } - pos = apply(op, input, pos, starts, ends, i == plan.ops().size() - 1, 0); + pos = apply(op, input, pos, starts, ends, i == plan.ops().size() - 1, workspace, 0); if (pos < 0) return false; } if (fullMatch && pos != input.length()) return false; @@ -159,6 +171,7 @@ private int apply( int[] starts, int[] ends, boolean lastOp, + MatchWorkspace workspace, int optionalDepth) { return switch (op.kind()) { case LITERAL -> startsWith(input, pos, op.literal()) ? pos + op.literal().length() : -1; @@ -187,7 +200,8 @@ private int apply( captureBracketedWordAfterSkip(input, pos, op.groupNumber(), starts, ends); case SKIP_ANY -> lastOp ? input.length() : -1; case ANCHOR -> pos; - case OPTIONAL_SEQUENCE -> applyOptional(op, input, pos, starts, ends, optionalDepth); + case OPTIONAL_SEQUENCE -> + applyOptional(op, input, pos, starts, ends, workspace, optionalDepth); }; } @@ -292,24 +306,35 @@ private static int captureIpOrHost(String input, int pos, int group, int[] start private static int captureBracketedWordAfterSkip( String input, int pos, int group, int[] starts, int[] ends) { - int search = pos; int lastStart = -1; int lastEnd = -1; - while (search < input.length()) { - int open = input.indexOf('[', search); - if (open < 0) break; - int close = input.indexOf(']', open + 1); - if (close < 0) break; - int wordEnd = open + 1; - while (wordEnd < close && isWord(input.charAt(wordEnd))) wordEnd++; - if (wordEnd == close - && wordEnd > open + 1 - && close + 1 < input.length() - && Character.isWhitespace(input.charAt(close + 1))) { - lastStart = open + 1; - lastEnd = close; + int open = -1; + int wordEnd = -1; + for (int index = pos; index < input.length(); index++) { + char ch = input.charAt(index); + if (ch == '[') { + open = index; + wordEnd = index + 1; + continue; + } + if (open < 0) { + continue; + } + if (ch == ']') { + if (wordEnd == index + && wordEnd > open + 1 + && index + 1 < input.length() + && Character.isWhitespace(input.charAt(index + 1))) { + lastStart = open + 1; + lastEnd = index; + } + open = -1; + wordEnd = -1; + } else if (wordEnd == index && isWord(ch)) { + wordEnd++; + } else { + wordEnd = -1; } - search = open + 1; } if (lastStart < 0) return -1; set(starts, ends, group, lastStart, lastEnd); @@ -365,9 +390,10 @@ private int applyOptional( int pos, int[] starts, int[] ends, + MatchWorkspace workspace, int optionalDepth) { - int[] savedStarts = optionalScratchStarts[optionalDepth]; - int[] savedEnds = optionalScratchEnds[optionalDepth]; + int[] savedStarts = workspace.optionalStarts[optionalDepth]; + int[] savedEnds = workspace.optionalEnds[optionalDepth]; System.arraycopy(starts, 0, savedStarts, 0, starts.length); System.arraycopy(ends, 0, savedEnds, 0, ends.length); int next = pos; @@ -380,6 +406,7 @@ private int applyOptional( starts, ends, i == op.children().size() - 1, + workspace, optionalDepth + 1); if (next < 0) { System.arraycopy(savedStarts, 0, starts, 0, starts.length); @@ -404,6 +431,20 @@ private static int maxOptionalDepth(Iterable ops) { return max; } + private static final class MatchWorkspace { + final int[] starts; + final int[] ends; + final int[][] optionalStarts; + final int[][] optionalEnds; + + MatchWorkspace(int groupCount, int optionalDepth) { + starts = new int[groupCount + 1]; + ends = new int[groupCount + 1]; + optionalStarts = new int[optionalDepth][groupCount + 1]; + optionalEnds = new int[optionalDepth][groupCount + 1]; + } + } + private static int skipWhitespace(String input, int pos) { int start = pos; while (pos < input.length() && Character.isWhitespace(input.charAt(pos))) pos++; diff --git a/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/RuntimeCompiler.java b/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/RuntimeCompiler.java index d5b39f56..2edaef51 100644 --- a/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/RuntimeCompiler.java +++ b/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/RuntimeCompiler.java @@ -816,6 +816,7 @@ private static ReggieMatcher compileInternal( private static ReggieMatcher tryCompileLinearTokenSequence( String pattern, RegexNode ast, Map nameMap) { return LinearTokenSequencePlan.from(PatternCategorizer.categorize(ast)) + .filter(plan -> plan.coversCaptureIndexes(nameMap.values())) .filter(RuntimeCompiler::isRuntimeExecutableLinearTokenSequence) .map(plan -> new LinearTokenSequenceMatcher(pattern, plan, countGroups(pattern), nameMap)) .map(m -> m.embedsNameMap() ? m : new NameEnrichingMatcher(m)) diff --git a/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcherConcurrencyTest.java b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcherConcurrencyTest.java new file mode 100644 index 00000000..3ef24482 --- /dev/null +++ b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcherConcurrencyTest.java @@ -0,0 +1,242 @@ +/* + * Copyright 2026-Present Datadog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datadoghq.reggie.runtime; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.datadoghq.reggie.Reggie; +import com.datadoghq.reggie.ReggieOptions; +import com.datadoghq.reggie.codegen.parsing.RegexParser; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Map; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +class LinearTokenSequenceMatcherConcurrencyTest { + private static final ReggieOptions NAMED_ONLY = ReggieOptions.builder().namedOnly().build(); + private static final String WITH_OPTIONAL_CAPTURES = + "10.202.82.195 - - [15/Mar/2019:19:45:35 -0700] \"POST /config?x=y HTTP/1.1\" " + + "200 17888"; + private static final String WITHOUT_OPTIONAL_CAPTURES = + "2001:db8::1 - - [15/Mar/2019:19:45:35 -0700] \"/health\" 200 -"; + + @AfterEach + void clearCache() { + Reggie.clearCache(); + } + + @Test + void cachedNamedOnlyLinearTokenSequenceMatcherIsSafeForConcurrentUse() throws Exception { + Reggie.clearCache(); + String pattern = testResource("logs-grok-pattern-1.regex"); + Map groupNumbers = groupNumbers(pattern); + ReggieMatcher shared = Reggie.compile(pattern, NAMED_ONLY); + ReggieMatcher second = Reggie.compile(pattern, NAMED_ONLY); + int groupCount = shared.match(WITH_OPTIONAL_CAPTURES).groupCount(); + + assertSame(shared, second); + assertDelegateType(shared, LinearTokenSequenceMatcher.class); + + int threads = 16; + int iterations = 1_000; + CountDownLatch ready = new CountDownLatch(threads); + CountDownLatch start = new CountDownLatch(1); + CountDownLatch done = new CountDownLatch(threads); + ConcurrentLinkedQueue failures = new ConcurrentLinkedQueue<>(); + + try (ExecutorService executor = Executors.newFixedThreadPool(threads)) { + for (int thread = 0; thread < threads; thread++) { + executor.execute( + () -> { + ready.countDown(); + try { + start.await(); + for (int iteration = 0; iteration < iterations; iteration++) { + assertMatchWithOptionalCaptures(shared, groupNumbers); + assertMatchWithoutOptionalCaptures(shared, groupNumbers); + assertFailedMatchLeavesArraysUntouched(shared, groupCount); + assertTrue(shared.find("noise " + WITH_OPTIONAL_CAPTURES)); + assertFalse(shared.find("noise malformed access log")); + } + } catch (Throwable failure) { + failures.add(failure); + } finally { + done.countDown(); + } + }); + } + + assertTrue(ready.await(10, TimeUnit.SECONDS), "workers did not become ready"); + start.countDown(); + assertTrue(done.await(30, TimeUnit.SECONDS), "workers did not finish"); + } + assertTrue(failures.isEmpty(), () -> "concurrent LTS failure: " + failures.peek()); + } + + @Test + void cachedLinearTokenSequenceMatcherSafelyRollsBackNestedOptionalSequences() throws Exception { + String pattern = "(?:a(?:b|)|)c"; + Reggie.clearCache(); + ReggieMatcher shared = Reggie.compile(pattern, NAMED_ONLY); + + assertSame(shared, Reggie.compile(pattern, NAMED_ONLY)); + assertDelegateType(shared, LinearTokenSequenceMatcher.class); + + int threads = 16; + int iterations = 1_000; + CountDownLatch ready = new CountDownLatch(threads); + CountDownLatch start = new CountDownLatch(1); + CountDownLatch done = new CountDownLatch(threads); + ConcurrentLinkedQueue failures = new ConcurrentLinkedQueue<>(); + + try (ExecutorService executor = Executors.newFixedThreadPool(threads)) { + for (int thread = 0; thread < threads; thread++) { + executor.execute( + () -> { + ready.countDown(); + try { + start.await(); + for (int iteration = 0; iteration < iterations; iteration++) { + assertTrue(shared.matches("abc")); + assertTrue(shared.matches("ac")); + assertTrue(shared.matches("c")); + assertFalse(shared.matches("ab")); + assertNotNull(shared.match("abc")); + assertNotNull(shared.match("ac")); + assertNotNull(shared.match("c")); + assertNull(shared.match("ab")); + assertTrue(shared.find("noise abc")); + assertFalse(shared.find("noise ab")); + + int[] starts = {17}; + int[] ends = {19}; + assertFalse(shared.matchInto("ab", starts, ends)); + assertEquals(17, starts[0]); + assertEquals(19, ends[0]); + } + } catch (Throwable failure) { + failures.add(failure); + } finally { + done.countDown(); + } + }); + } + + assertTrue(ready.await(10, TimeUnit.SECONDS), "workers did not become ready"); + start.countDown(); + assertTrue(done.await(30, TimeUnit.SECONDS), "workers did not finish"); + } + assertTrue( + failures.isEmpty(), () -> "concurrent nested-optional LTS failure: " + failures.peek()); + } + + private static void assertMatchWithOptionalCaptures( + ReggieMatcher matcher, Map groupNumbers) { + assertTrue(matcher.matches(WITH_OPTIONAL_CAPTURES)); + MatchResult result = matcher.match(WITH_OPTIONAL_CAPTURES); + assertNotNull(result); + assertEquals("POST", result.group("grok4")); + assertEquals("/config?x=y", result.group("grok5")); + assertEquals("1.1", result.group("grok6")); + assertEquals("200", result.group("grok7")); + + int[] starts = new int[result.groupCount() + 1]; + int[] ends = new int[result.groupCount() + 1]; + assertTrue(matcher.matchInto(WITH_OPTIONAL_CAPTURES, starts, ends)); + assertEquals( + "POST", + WITH_OPTIONAL_CAPTURES.substring( + starts[groupNumbers.get("grok4")], ends[groupNumbers.get("grok4")])); + assertEquals( + "1.1", + WITH_OPTIONAL_CAPTURES.substring( + starts[groupNumbers.get("grok6")], ends[groupNumbers.get("grok6")])); + } + + private static void assertMatchWithoutOptionalCaptures( + ReggieMatcher matcher, Map groupNumbers) { + assertTrue(matcher.matches(WITHOUT_OPTIONAL_CAPTURES)); + MatchResult result = matcher.match(WITHOUT_OPTIONAL_CAPTURES); + assertNotNull(result); + assertNull(result.group("grok4")); + assertEquals("/health", result.group("grok5")); + assertNull(result.group("grok6")); + assertEquals("200", result.group("grok7")); + + int[] starts = new int[result.groupCount() + 1]; + int[] ends = new int[result.groupCount() + 1]; + assertTrue(matcher.matchInto(WITHOUT_OPTIONAL_CAPTURES, starts, ends)); + assertEquals(-1, starts[groupNumbers.get("grok4")]); + assertEquals(-1, ends[groupNumbers.get("grok4")]); + assertEquals(-1, starts[groupNumbers.get("grok6")]); + assertEquals(-1, ends[groupNumbers.get("grok6")]); + } + + private static void assertFailedMatchLeavesArraysUntouched( + ReggieMatcher matcher, int groupCount) { + String input = "not an access log"; + assertFalse(matcher.matches(input)); + assertNull(matcher.match(input)); + int[] starts = new int[groupCount + 1]; + int[] ends = new int[groupCount + 1]; + Arrays.fill(starts, 17); + Arrays.fill(ends, 19); + assertFalse(matcher.matchInto(input, starts, ends)); + assertTrue(Arrays.stream(starts).allMatch(value -> value == 17)); + assertTrue(Arrays.stream(ends).allMatch(value -> value == 19)); + } + + private static Map groupNumbers(String pattern) throws Exception { + RegexParser parser = new RegexParser(); + parser.parse(pattern); + return parser.getGroupNameMap(); + } + + private static String testResource(String name) throws IOException { + String resource = "/com/datadoghq/reggie/runtime/" + name; + try (InputStream input = + LinearTokenSequenceMatcherConcurrencyTest.class.getResourceAsStream(resource)) { + if (input == null) { + throw new IOException("missing test resource: " + resource); + } + return new String(input.readAllBytes(), StandardCharsets.UTF_8); + } + } + + private static void assertDelegateType(ReggieMatcher matcher, Class expectedType) + throws Exception { + if (matcher.getClass() == expectedType) { + return; + } + Field delegate = matcher.getClass().getDeclaredField("delegate"); + delegate.setAccessible(true); + assertEquals(expectedType, delegate.get(matcher).getClass()); + } +} diff --git a/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcherTest.java b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcherTest.java index 195bc4fd..07578216 100644 --- a/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcherTest.java +++ b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/LinearTokenSequenceMatcherTest.java @@ -32,6 +32,8 @@ import java.lang.reflect.Field; import java.util.Arrays; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.junit.jupiter.api.Test; class LinearTokenSequenceMatcherTest { @@ -158,6 +160,33 @@ void namedOnlyProjectionRunsBeforeLinearTokenRouting() throws Exception { assertDelegateType(matcher, LinearTokenSequenceMatcher.class); } + @Test + void namedOnlyRoutingRejectsPlansMissingAnObservableNamedCapture() throws Exception { + String pattern = "(?(?:-|(?[+-]?\\d+)))"; + RegexParser parser = new RegexParser(); + RegexNode ast = parser.parse(pattern); + LinearTokenSequencePlan plan = + LinearTokenSequencePlan.from(PatternCategorizer.categorize(ast)).orElseThrow(); + assertFalse(plan.coversCaptureIndexes(parser.getGroupNameMap().values())); + + ReggieMatcher matcher = Reggie.compile(pattern, NAMED_ONLY_OPTIONS); + assertNotLinearTokenSequenceDelegate(matcher); + + Pattern jdkPattern = Pattern.compile(pattern); + for (String input : new String[] {"-", "42"}) { + Matcher jdk = jdkPattern.matcher(input); + assertTrue(jdk.matches(), input); + + MatchResult actual = matcher.match(input); + assertNotNull(actual, input); + for (int group : parser.getGroupNameMap().values()) { + assertEquals(jdk.start(group), actual.start(group), "start group " + group + ": " + input); + assertEquals(jdk.end(group), actual.end(group), "end group " + group + ": " + input); + assertEquals(jdk.group(group), actual.group(group), "value group " + group + ": " + input); + } + } + } + @Test void capturedDashAlternativeRecordsNamedGroupSpan() throws Exception { ReggieMatcher matcher = Reggie.compile("(?(?:-|[+-]?\\d+))", NAMED_ONLY_OPTIONS); @@ -208,6 +237,39 @@ void runtimeCompilerRoutesCombinedAccessLogTemplateWithNonGrokNames() throws Exc assertDelegateType(matcher, LinearTokenSequenceMatcher.class); } + @Test + void bracketedWordAfterSkipUsesLastEligibleBracketedWord() throws Exception { + ReggieMatcher matcher = matcherFor(".* \\[(?\\b\\w+\\b)\\] .*"); + + MatchResult result = matcher.match("[ignored] [[first] [last] trailing"); + + assertNotNull(result); + assertEquals("last", result.group("logger")); + } + + @Test + void bracketedWordAfterSkipIgnoresMalformedPrefixes() throws Exception { + ReggieMatcher matcher = matcherFor(".* \\[(?\\b\\w+\\b)\\] .*"); + + MatchResult result = matcher.match("[bad-value] [valid] trailing"); + + assertNotNull(result); + assertEquals("valid", result.group("logger")); + } + + @Test + void bracketedWordAfterSkipHandlesManyUnclosedBracketsInOnePass() throws Exception { + ReggieMatcher matcher = matcherFor(".* \\[(?\\b\\w+\\b)\\] .*"); + assertNull(matcher.match("[".repeat(20_000))); + + String input = "[".repeat(20_000) + "word] "; + + MatchResult result = matcher.match(input); + + assertNotNull(result); + assertEquals("word", result.group("logger")); + } + private static final ReggieOptions NAMED_ONLY_OPTIONS = ReggieOptions.builder().namedOnly().build(); @@ -221,6 +283,19 @@ private static void assertDelegateType(ReggieMatcher matcher, Class expectedT assertEquals(expectedType, delegate.get(matcher).getClass()); } + private static void assertNotLinearTokenSequenceDelegate(ReggieMatcher matcher) throws Exception { + if (matcher.getClass() == LinearTokenSequenceMatcher.class) { + throw new AssertionError("matcher unexpectedly used LinearTokenSequenceMatcher"); + } + try { + Field delegate = matcher.getClass().getDeclaredField("delegate"); + delegate.setAccessible(true); + assertNotEquals(LinearTokenSequenceMatcher.class, delegate.get(matcher).getClass()); + } catch (NoSuchFieldException ignored) { + // A non-wrapper matcher cannot be an LTS matcher because the direct type was checked above. + } + } + private static ReggieMatcher matcherFor(String pattern) throws Exception { RegexParser parser = new RegexParser(); RegexNode ast = parser.parse(pattern);