Add markdown support for javadocs and modernize highlighting a bit. (#15568)

Author: dweiss

build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/RenderJavadocPlugin.java

@@ -566,12 +566,17 @@ public void render() throws IOException {
             getTaskResources(),
             "table_padding.css",
             "custom_styles.css",
-            "prettify/prettify.css");
+            "prettify/prettify.css",
+            "prettify/prism.css");
 
         // append prettify to scripts
         Provider<RegularFile> customScript = getOutputDir().file("script-files/lucene-script.js");
         concat(
-            customScript, getTaskResources().dir("prettify"), "prettify.js", "inject-javadocs.js");
+            customScript,
+            getTaskResources().dir("prettify"),
+            "prettify.js",
+            "inject-javadocs.js",
+            "prism.js");
 
         opts.add(List.of("--add-script", customScript.get().getAsFile().toString()));
         opts.add(List.of("--add-stylesheet", customCss.get().getAsFile().toString()));

build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/spotless/ParentGoogleJavaFormatTask.java

@@ -76,7 +76,7 @@ abstract class ParentGoogleJavaFormatTask extends DefaultTask {
   @Internal
   protected abstract RegularFileProperty getFileStateCache();
 
-  private Function<File, String> fileToKey;
+  private final Function<File, String> fileToKey;
 
   public ParentGoogleJavaFormatTask(ProjectLayout layout, String gjfTask) {
     getOutputChangeListFile()

gradle/documentation/render-javadoc/prettify/prism.css

@@ -15,7 +15,7 @@ errorprone = "2.45.0"
 # markdown documentation generation
 flexmark = "0.64.8"
 # @keep This is GJF version for java source code formatting
-googleJavaFormat = "1.33.0"
+googleJavaFormat = "1.33.0-markdown"
 # gradle build support
 groovy = "5.0.3"
 # test assertions
@@ -72,7 +72,7 @@ flexmark-ext-abbreviation = { module = "com.vladsch.flexmark:flexmark-ext-abbrev
 flexmark-ext-attributes = { module = "com.vladsch.flexmark:flexmark-ext-attributes", version.ref = "flexmark" }
 flexmark-ext-autolink = { module = "com.vladsch.flexmark:flexmark-ext-autolink", version.ref = "flexmark" }
 flexmark-ext-tables = { module = "com.vladsch.flexmark:flexmark-ext-tables", version.ref = "flexmark" }
-gjf = { module = "com.google.googlejavaformat:google-java-format", version.ref = "googleJavaFormat" }
+gjf = { module = "com.carrotsearch.googlejavaformat:google-java-format", version.ref = "googleJavaFormat" }
 groovy = { module = "org.apache.groovy:groovy-all", version.ref = "groovy" }
 hamcrest = { module = "org.hamcrest:hamcrest", version.ref = "hamcrest" }
 icu4j = { module = "com.ibm.icu:icu4j", version.ref = "icu4j" }

gradle/documentation/render-javadoc/prettify/prism.js

@@ -71,6 +71,8 @@ New Features
 Improvements
 ---------------------
 
+* GITHUB#15568: Add markdown support for javadocs and modernize highlighting a bit. (Dawid Weiss)
+
 * GITHUB#14972: Require named thread factory in ThreadPoolExecutor constructors. (Dawid Weiss)
 
 * GITHUB#14597: Rewrite expressions compiler to use Java 24+ Classfile library instead of ASM.

gradle/libs.versions.toml

@@ -23,14 +23,14 @@
 /**
  * Factory for {@link org.apache.lucene.analysis.ja.JapaneseBaseFormFilter}.
  *
- * <pre class="prettyprint">
+ * <pre><code class="language-xml">
  * &lt;fieldType name="text_ja" class="solr.TextField"&gt;
  *   &lt;analyzer&gt;
  *     &lt;tokenizer class="solr.JapaneseTokenizerFactory"/&gt;
  *     &lt;filter class="solr.JapaneseBaseFormFilterFactory"/&gt;
  *   &lt;/analyzer&gt;
  * &lt;/fieldType&gt;
- * </pre>
+ * </code></pre>
  *
  * @since 3.6.0
  * @lucene.spi {@value #NAME}

lucene/CHANGES.txt

@@ -88,7 +88,7 @@
  * example, assuming that documents have a {@link FeatureField} called 'features' with values for
  * the 'pagerank' feature.
  *
- * <pre class="prettyprint">
+ * <pre><code class="language-java">
  * Query query = new BooleanQuery.Builder()
  *     .add(new TermQuery(new Term("body", "apache")), Occur.SHOULD)
  *     .add(new TermQuery(new Term("body", "lucene")), Occur.SHOULD)
@@ -99,7 +99,7 @@
  *     .add(boost, Occur.SHOULD)
  *     .build();
  * TopDocs topDocs = searcher.search(boostedQuery, 10);
- * </pre>
+ * </code></pre>
  *
  * @lucene.experimental
  */

lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseBaseFormFilterFactory.java

@@ -30,37 +30,37 @@
 code comes from org.apache.lucene.TestDemo.
 See LUCENE-8481 for reasons why it's out of sync with the code.
 -->
-<pre class="prettyprint">
-    Analyzer analyzer = new StandardAnalyzer();
-
-    Path indexPath = Files.createTempDirectory("tempIndex");
-    try (Directory directory = FSDirectory.open(indexPath)) {
-      IndexWriterConfig config = new IndexWriterConfig(analyzer);
-      try (IndexWriter iwriter = new IndexWriter(directory, config)) {
-        Document doc = new Document();
-        String text = "This is the text to be indexed.";
-        doc.add(new Field("fieldname", text, TextField.TYPE_STORED));
-        iwriter.addDocument(doc);
-      }
-
-      // Now search the index:
-      try (DirectoryReader ireader = DirectoryReader.open(directory)) {
-        IndexSearcher isearcher = new IndexSearcher(ireader);
-        // Parse a simple query that searches for "text":
-        QueryParser parser = new QueryParser("fieldname", analyzer);
-        Query query = parser.parse("text");
-        ScoreDoc[] hits = isearcher.search(query, 10).scoreDocs;
-        assertEquals(1, hits.length);
-        // Iterate through the results:
-        StoredFields storedFields = isearcher.storedFields();
-        for (int i = 0; i &lt; hits.length; i++) {
-          Document hitDoc = storedFields.document(hits[i].doc);
-          assertEquals("This is the text to be indexed.", hitDoc.get("fieldname"));
-        }
-      }
-    } finally {
-      IOUtils.rm(indexPath);
-    }</pre>
+<pre><code class="language-java">
+Analyzer analyzer = new StandardAnalyzer();
+
+Path indexPath = Files.createTempDirectory("tempIndex");
+try (Directory directory = FSDirectory.open(indexPath)) {
+  IndexWriterConfig config = new IndexWriterConfig(analyzer);
+  try (IndexWriter iwriter = new IndexWriter(directory, config)) {
+    Document doc = new Document();
+    String text = "This is the text to be indexed.";
+    doc.add(new Field("fieldname", text, TextField.TYPE_STORED));
+    iwriter.addDocument(doc);
+  }
+
+  // Now search the index:
+  try (DirectoryReader ireader = DirectoryReader.open(directory)) {
+    IndexSearcher isearcher = new IndexSearcher(ireader);
+    // Parse a simple query that searches for "text":
+    QueryParser parser = new QueryParser("fieldname", analyzer);
+    Query query = parser.parse("text");
+    ScoreDoc[] hits = isearcher.search(query, 10).scoreDocs;
+    assertEquals(1, hits.length);
+    // Iterate through the results:
+    StoredFields storedFields = isearcher.storedFields();
+    for (int i = 0; i &lt; hits.length; i++) {
+      Document hitDoc = storedFields.document(hits[i].doc);
+      assertEquals("This is the text to be indexed.", hitDoc.get("fieldname"));
+    }
+  }
+} finally {
+  IOUtils.rm(indexPath);
+}</code></pre>
 <!-- ======================================================== -->
 
 

lucene/core/src/java/org/apache/lucene/document/FeatureField.java

@@ -210,49 +210,53 @@
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
 import org.junit.Rule;
-import org.junit.Test;
 import org.junit.internal.AssumptionViolatedException;
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
 import org.junit.runner.RunWith;
 
-/**
- * Base class for all Lucene unit tests, Junit3 or Junit4 variant.
- *
- * <h2>Class and instance setup.</h2>
- *
- * <p>The preferred way to specify class (suite-level) setup/cleanup is to use static methods
- * annotated with {@link BeforeClass} and {@link AfterClass}. Any code in these methods is executed
- * within the test framework's control and ensure proper setup has been made. <b>Try not to use
- * static initializers (including complex final field initializers).</b> Static initializers are
- * executed before any setup rules are fired and may cause you (or somebody else) headaches.
- *
- * <p>For instance-level setup, use {@link Before} and {@link After} annotated methods. If you
- * override either {@link #setUp()} or {@link #tearDown()} in your subclass, make sure you call
- * <code>super.setUp()</code> and <code>super.tearDown()</code>. This is detected and enforced.
- *
- * <h2>Specifying test cases</h2>
- *
- * <p>Any test method with a <code>testXXX</code> prefix is considered a test case. Any test method
- * annotated with {@link Test} is considered a test case.
- *
- * <h2>Randomized execution and test facilities</h2>
- *
- * <p>{@link LuceneTestCase} uses {@link RandomizedRunner} to execute test cases. {@link
- * RandomizedRunner} has built-in support for tests randomization including access to a repeatable
- * {@link Random} instance. See {@link #random()} method. Any test using {@link Random} acquired
- * from {@link #random()} should be fully reproducible (assuming no race conditions between threads
- * etc.). The initial seed for a test case is reported in many ways:
- *
- * <ul>
- *   <li>as part of any exception thrown from its body (inserted as a dummy stack trace entry),
- *   <li>as part of the main thread executing the test case (if your test hangs, just dump the stack
- *       trace of all threads and you'll see the seed),
- *   <li>the master seed can also be accessed manually by getting the current context ({@link
- *       RandomizedContext#current()}) and then calling {@link
- *       RandomizedContext#getRunnerSeedAsString()}.
- * </ul>
- */
+/// Base class for all Lucene unit tests (JUnit4 variant).
+///
+/// ## Class and instance setup
+///
+/// The preferred way to specify class (suite-level) setup/cleanup is to use static methods
+/// annotated with [BeforeClass] and [AfterClass]. Any code in these methods is executed
+/// within the test framework's control and ensure proper setup has been made. **Try not to use
+/// static initializers (including complex final field initializers).** Static initializers are
+/// executed before any setup rules are fired and may cause you (or somebody else) headaches.
+///
+/// For instance-level setup, use [Before] and [After] annotated methods. If you
+/// override either [#setUp()] or [#tearDown()] in your subclass, make sure you call
+/// `super.setUp()` and `super.tearDown()`. This is detected and enforced.
+///
+/// ## Specifying test cases
+///
+/// Any test method with a `testXXX` prefix is considered a test case. Any test method
+/// annotated with [org.junit.Test] is considered a test case. For example, these are equivalent
+/// declarations:
+///
+/// ```java
+/// public void testPrefixIsSufficient() {}
+///
+/// @Test
+/// public void annotationIsRequiredHere() {}
+/// ```
+///
+/// ## Randomized execution and test facilities
+///
+/// [LuceneTestCase] uses [RandomizedRunner] to execute test cases.
+/// [RandomizedRunner] has built-in support for tests randomization including access to a repeatable
+/// [Random] instance. See [#random()] method. Any test using [Random] acquired
+/// from [#random()] should be fully reproducible (assuming no race conditions between threads
+/// etc.). The initial seed for a test case is reported in many ways:
+///
+///   - as part of any exception thrown from its body (inserted as a dummy stack trace entry),
+///   - as part of the main thread executing the test case (if your test hangs, just dump the stack
+///     trace of all threads, and you'll see the seed),
+///   - the master seed can also be accessed manually by getting the current context (
+///     [RandomizedContext#current()]) and then calling
+///     [RandomizedContext#getRunnerSeedAsString()].
+///
 @RunWith(RandomizedRunner.class)
 @TestMethodProviders({LuceneJUnit3MethodProvider.class, JUnit4MethodProvider.class})
 @Listeners({RunListenerPrintReproduceInfo.class, FailureMarker.class})