From 362ae27faeb99a6fc2b8c47729f33f90b65317de Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 14 Aug 2026 15:42:16 +0200 Subject: [PATCH] feat: add native linear token admission --- .../reggie/runtime/RuntimeCompiler.java | 119 ++++++++----- .../runtime/NamedOnlyLtsAdmissionTest.java | 159 ++++++++++++++++++ 2 files changed, 236 insertions(+), 42 deletions(-) create mode 100644 reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/NamedOnlyLtsAdmissionTest.java diff --git a/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/RuntimeCompiler.java b/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/RuntimeCompiler.java index 7ad09b4e..2ddc2581 100644 --- a/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/RuntimeCompiler.java +++ b/reggie-runtime/src/main/java/com/datadoghq/reggie/runtime/RuntimeCompiler.java @@ -75,7 +75,6 @@ import java.io.PrintWriter; import java.lang.invoke.MethodHandles; import java.lang.reflect.Constructor; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -930,6 +929,55 @@ private static ReggieMatcher compileInternal( } } + enum NamedOnlyLtsRejection { + UNSUPPORTED_FLAGS, + SOURCE_INLINE_MODIFIER, + PARSE_FAILURE, + PLAN_UNAVAILABLE, + MISSING_NAMED_CAPTURE, + PROFILE_INELIGIBLE + } + + record NamedOnlyLtsCompilation( + LinearTokenSequenceMatcher matcher, NamedOnlyLtsRejection rejection) { + NamedOnlyLtsCompilation { + if ((matcher == null) == (rejection == null)) { + throw new IllegalArgumentException("exactly one of matcher or rejection is required"); + } + } + + static NamedOnlyLtsCompilation admitted(LinearTokenSequenceMatcher matcher) { + return new NamedOnlyLtsCompilation(matcher, null); + } + + static NamedOnlyLtsCompilation rejected(NamedOnlyLtsRejection rejection) { + return new NamedOnlyLtsCompilation(null, rejection); + } + } + + static NamedOnlyLtsCompilation tryCompileNamedOnlyLinearTokenSequence(String source, int flags) { + if (flags != 0 && flags != ReggieFlags.DOTALL) { + return NamedOnlyLtsCompilation.rejected(NamedOnlyLtsRejection.UNSUPPORTED_FLAGS); + } + if (hasSourceInlineModifier(source)) { + return NamedOnlyLtsCompilation.rejected(NamedOnlyLtsRejection.SOURCE_INLINE_MODIFIER); + } + boolean dotAll = flags == ReggieFlags.DOTALL; + String parsePattern = dotAll ? "(?s)" + source : source; + try { + RegexParser parser = new RegexParser(); + RegexNode ast = + CaptureProjection.preserveNamedAndSemanticCaptures(parser.parse(parsePattern)); + Map nameMap = parser.getGroupNameMap(); + return admitNamedOnlyLinearTokenSequence( + source, ast, nameMap, new LinearTokenSequenceAdmission(true, dotAll)); + } catch (RegexParser.ParseException e) { + return NamedOnlyLtsCompilation.rejected(NamedOnlyLtsRejection.PARSE_FAILURE); + } catch (UnsupportedOperationException | IllegalStateException e) { + return NamedOnlyLtsCompilation.rejected(NamedOnlyLtsRejection.PLAN_UNAVAILABLE); + } + } + private static ReggieMatcher tryCompileLinearTokenSequence( String pattern, RegexNode ast, @@ -938,24 +986,34 @@ private static ReggieMatcher tryCompileLinearTokenSequence( if (!admission.isEligible()) { return null; } - return LinearTokenSequencePlan.from(PatternCategorizer.categorize(ast)) - .filter(plan -> plan.coversCaptureIndexes(nameMap.values())) - .filter(plan -> isRuntimeExecutableLinearTokenSequence(admission.isDotAll(), plan)) - .map(plan -> new LinearTokenSequenceMatcher(pattern, plan, countGroups(pattern), nameMap)) - .map(m -> m.embedsNameMap() ? m : new NameEnrichingMatcher(m)) - .orElse(null); + return admitNamedOnlyLinearTokenSequence(pattern, ast, nameMap, admission).matcher(); } - private static boolean isRuntimeExecutableLinearTokenSequence( - boolean dotAll, LinearTokenSequencePlan plan) { - return validateOps(dotAll, plan.ops(), true); + private static NamedOnlyLtsCompilation admitNamedOnlyLinearTokenSequence( + String pattern, + RegexNode ast, + Map nameMap, + LinearTokenSequenceAdmission admission) { + LinearTokenSequencePlan plan = + LinearTokenSequencePlan.from(PatternCategorizer.categorize(ast)).orElse(null); + if (plan == null) { + return NamedOnlyLtsCompilation.rejected(NamedOnlyLtsRejection.PLAN_UNAVAILABLE); + } + if (!plan.coversCaptureIndexes(nameMap.values())) { + return NamedOnlyLtsCompilation.rejected(NamedOnlyLtsRejection.MISSING_NAMED_CAPTURE); + } + if (!isRuntimeExecutableLinearTokenSequence(admission.isDotAll(), plan)) { + return NamedOnlyLtsCompilation.rejected(NamedOnlyLtsRejection.PROFILE_INELIGIBLE); + } + return NamedOnlyLtsCompilation.admitted( + new LinearTokenSequenceMatcher(pattern, plan, countGroups(pattern), nameMap)); } - private static boolean validateOps( - boolean dotAll, List ops, boolean isTopLevel) { + private static boolean isRuntimeExecutableLinearTokenSequence( + boolean dotAll, LinearTokenSequencePlan plan) { boolean requiresDotAll = false; - for (int i = 0; i < ops.size(); i++) { - LinearTokenSequencePlan.Op op = ops.get(i); + for (int i = 0; i < plan.ops().size(); i++) { + LinearTokenSequencePlan.Op op = plan.ops().get(i); if (op.kind() == LinearTokenSequencePlan.OpKind.ANCHOR) return false; if (op.kind() == LinearTokenSequencePlan.OpKind.SKIP_ANY_EXCEPT_NEWLINE) return false; if (op.kind() == LinearTokenSequencePlan.OpKind.SKIP_ANY @@ -964,41 +1022,18 @@ private static boolean validateOps( } if ((op.kind() == LinearTokenSequencePlan.OpKind.SKIP_ANY || op.kind() == LinearTokenSequencePlan.OpKind.SKIP_ANY_EXCEPT_NEWLINE) - && i != ops.size() - 1) { + && i != plan.ops().size() - 1) { return false; } - if (op.kind() == LinearTokenSequencePlan.OpKind.OPTIONAL_SEQUENCE) { - if (!validateOps(dotAll, op.children(), false)) return false; - // An optional sequence containing a wildcard (SKIP_ANY or SKIP_ANY_EXCEPT_NEWLINE) - // cannot backtrack: the present branch greedily consumes all remaining input, - // so any ops following the optional sequence would fail. Reject when there are - // more ops after the optional. - if (isTopLevel && i + 1 < ops.size() && containsWildcard(op)) { - return false; - } - if (isTopLevel - && i + 1 < ops.size() - && canOptionalPresentBranchStealFollowingInput(op, ops.get(i + 1))) { - return false; - } + if (op.kind() == LinearTokenSequencePlan.OpKind.OPTIONAL_SEQUENCE + && i + 1 < plan.ops().size() + && canOptionalPresentBranchStealFollowingInput(op, plan.ops().get(i + 1))) { + return false; } } return !requiresDotAll || dotAll; } - private static boolean containsWildcard(LinearTokenSequencePlan.Op optional) { - for (LinearTokenSequencePlan.Op child : optional.children()) { - if (child.kind() == LinearTokenSequencePlan.OpKind.SKIP_ANY - || child.kind() == LinearTokenSequencePlan.OpKind.SKIP_ANY_EXCEPT_NEWLINE) { - return true; - } - if (!child.children().isEmpty() && containsWildcard(child)) { - return true; - } - } - return false; - } - private static boolean canOptionalPresentBranchStealFollowingInput( LinearTokenSequencePlan.Op optional, LinearTokenSequencePlan.Op next) { if (optional.children().isEmpty()) return false; diff --git a/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/NamedOnlyLtsAdmissionTest.java b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/NamedOnlyLtsAdmissionTest.java new file mode 100644 index 00000000..5e03580c --- /dev/null +++ b/reggie-runtime/src/test/java/com/datadoghq/reggie/runtime/NamedOnlyLtsAdmissionTest.java @@ -0,0 +1,159 @@ +/* + * Copyright 2026-Present Datadog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datadoghq.reggie.runtime; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.datadoghq.reggie.Reggie; +import com.datadoghq.reggie.ReggieFlags; +import com.datadoghq.reggie.ReggieOptions; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +class NamedOnlyLtsAdmissionTest { + + @AfterEach + void clearCache() { + RuntimeCompiler.clearCache(); + } + + @Test + void admitsOnlyNativeLtsWithoutTouchingLegacyCaches() { + int patternCacheSize = RuntimeCompiler.cacheSize(); + int structuralCacheSize = RuntimeCompiler.structuralCacheSize(); + + RuntimeCompiler.NamedOnlyLtsCompilation compilation = + RuntimeCompiler.tryCompileNamedOnlyLinearTokenSequence( + ".* \\[(?\\b\\w+\\b)\\] .*", ReggieFlags.DOTALL); + + assertNotNull(compilation.matcher()); + assertNull(compilation.rejection()); + assertEquals(".* \\[(?\\b\\w+\\b)\\] .*", compilation.matcher().pattern()); + assertEquals("nginx", compilation.matcher().match("before\n [nginx] after").group("logger")); + assertEquals(patternCacheSize, RuntimeCompiler.cacheSize()); + assertEquals(structuralCacheSize, RuntimeCompiler.structuralCacheSize()); + } + + @Test + void returnsStructuredRejectionsBeforeLegacyCompilation() { + int patternCacheSize = RuntimeCompiler.cacheSize(); + int structuralCacheSize = RuntimeCompiler.structuralCacheSize(); + assertRejected( + "(?s)(?\\S+)", 0, RuntimeCompiler.NamedOnlyLtsRejection.SOURCE_INLINE_MODIFIER); + assertRejected( + "(?s:(?\\S+))", 0, RuntimeCompiler.NamedOnlyLtsRejection.SOURCE_INLINE_MODIFIER); + assertRejected( + "(?\\S+)", + ReggieFlags.CASE_INSENSITIVE, + RuntimeCompiler.NamedOnlyLtsRejection.UNSUPPORTED_FLAGS); + assertRejected( + "(?\\S+)", + java.util.regex.Pattern.DOTALL, + RuntimeCompiler.NamedOnlyLtsRejection.UNSUPPORTED_FLAGS); + assertRejected( + ".* \\[(?\\b\\w+\\b)\\] .*", + 0, + RuntimeCompiler.NamedOnlyLtsRejection.PROFILE_INELIGIBLE); + assertRejected("(?[a-z]+)", 0, RuntimeCompiler.NamedOnlyLtsRejection.PLAN_UNAVAILABLE); + assertRejected( + "(?(?:-|(?[+-]?\\d+)))", + 0, + RuntimeCompiler.NamedOnlyLtsRejection.MISSING_NAMED_CAPTURE); + assertRejected("(?", 0, RuntimeCompiler.NamedOnlyLtsRejection.PARSE_FAILURE); + assertEquals(patternCacheSize, RuntimeCompiler.cacheSize()); + assertEquals(structuralCacheSize, RuntimeCompiler.structuralCacheSize()); + } + + @Test + void repeatedAdmissionsReturnIndependentMatchers() { + RuntimeCompiler.NamedOnlyLtsCompilation first = + RuntimeCompiler.tryCompileNamedOnlyLinearTokenSequence("(?\\S+)", 0); + RuntimeCompiler.NamedOnlyLtsCompilation second = + RuntimeCompiler.tryCompileNamedOnlyLinearTokenSequence("(?\\S+)", 0); + + assertNotNull(first.matcher()); + assertNotNull(second.matcher()); + assertNotSame(first.matcher(), second.matcher()); + assertEquals("first", first.matcher().match("first").group("value")); + assertEquals("second", second.matcher().match("second").group("value")); + } + + @Test + void preservesOriginalNamedIndexWhileProjectingAnUnnamedCapture() { + String pattern = "(x)(?\\S+)"; + RuntimeCompiler.NamedOnlyLtsCompilation direct = + RuntimeCompiler.tryCompileNamedOnlyLinearTokenSequence(pattern, 0); + ReggieMatcher legacy = Reggie.compile(pattern, ReggieOptions.builder().namedOnly().build()); + + MatchResult directResult = direct.matcher().match("xvalue"); + MatchResult legacyResult = legacy.match("xvalue"); + assertEquals(2, directResult.groupCount()); + assertNull(directResult.group(1)); + assertEquals("value", directResult.group(2)); + assertEquals(directResult.group("value"), legacyResult.group("value")); + assertEquals(directResult.start(2), legacyResult.start(2)); + assertEquals(directResult.end(2), legacyResult.end(2)); + } + + @Test + void admittedMatchersRemainIndependentUnderConcurrentUse() throws Exception { + LinearTokenSequenceMatcher shared = + RuntimeCompiler.tryCompileNamedOnlyLinearTokenSequence("(?\\S+)", 0).matcher(); + int threads = 4; + CountDownLatch done = new CountDownLatch(threads); + ConcurrentLinkedQueue failures = new ConcurrentLinkedQueue<>(); + try (ExecutorService executor = Executors.newFixedThreadPool(threads)) { + for (int thread = 0; thread < threads; thread++) { + int id = thread; + executor.execute( + () -> { + try { + LinearTokenSequenceMatcher independent = + RuntimeCompiler.tryCompileNamedOnlyLinearTokenSequence("(?\\S+)", 0) + .matcher(); + for (int iteration = 0; iteration < 100; iteration++) { + assertEquals("shared", shared.match("shared").group("value")); + assertEquals("value" + id, independent.match("value" + id).group("value")); + } + } catch (Throwable failure) { + failures.add(failure); + } finally { + done.countDown(); + } + }); + } + assertTrue(done.await(10, TimeUnit.SECONDS), "workers did not finish"); + } + assertTrue(failures.isEmpty(), () -> "concurrent admission failure: " + failures.peek()); + } + + private static void assertRejected( + String source, int flags, RuntimeCompiler.NamedOnlyLtsRejection expected) { + RuntimeCompiler.NamedOnlyLtsCompilation compilation = + RuntimeCompiler.tryCompileNamedOnlyLinearTokenSequence(source, flags); + assertNull(compilation.matcher()); + assertEquals(expected, compilation.rejection()); + } +}