TEZ-4387: use new doclet API in Java 9+

pom.xml

@@ -1385,6 +1385,7 @@
     </profile>
   </profiles>
 
+  <!-- TODO: TEZ-4597: IncludePublicAnnotationsStandardDoclet is not JDK9+ compatible -->
   <reporting>
     <plugins>
       <plugin>

tez-tools/tez-javadoc-tools/findbugs-exclude.xml

@@ -13,6 +13,18 @@
 -->
 <FindBugsFilter>
 
+  <Match>
+    <Class name="org.apache.tez.tools.javadoc.model.ConfigProperty"/>
+    <Field name="description"/>
+    <Bug pattern="UWF_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD"/>
+  </Match>
+
+  <Match>
+    <Class name="org.apache.tez.tools.javadoc.model.ConfigProperty"/>
+    <Field name="validValues"/>
+    <Bug pattern="UWF_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD"/>
+  </Match>
+
   <Match>
     <Class name="org.apache.tez.tools.javadoc.model.ConfigProperty"/>
     <Field name="validValues"/>

tez-tools/tez-javadoc-tools/pom.xml

@@ -78,6 +78,62 @@
         </dependency>
       </dependencies>
     </profile>
+
+    <profile>
+      <id>java8-16</id>
+       <activation>
+             <jdk>[1.8,16]</jdk>
+       </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>build-helper-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <phase>generate-sources</phase>
+                <goals>
+                  <goal>add-source</goal>
+                </goals>
+                <configuration>
+                  <sources>
+                    <source>src/main/java-8-16</source>
+                  </sources>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
+    <profile>
+      <id>java17</id>
+      <activation>
+        <jdk>[17,)</jdk>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>build-helper-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <phase>generate-sources</phase>
+                <goals>
+                  <goal>add-source</goal>
+                </goals>
+                <configuration>
+                  <sources>
+                    <source>src/main/java-17</source>
+                  </sources>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
   </profiles>
 
 

tez-tools/tez-javadoc-tools/src/main/java-17/org/apache/tez/tools/javadoc/doclet/ConfigStandardDoclet.java

@@ -0,0 +1,291 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.tools.javadoc.doclet;
+
+import com.sun.source.util.DocTrees;
+import jdk.javadoc.doclet.Doclet;
+import jdk.javadoc.doclet.DocletEnvironment;
+import jdk.javadoc.doclet.Reporter;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.tez.common.annotation.ConfigurationClass;
+import org.apache.tez.common.annotation.ConfigurationProperty;
+import org.apache.tez.tools.javadoc.model.Config;
+import org.apache.tez.tools.javadoc.model.ConfigProperty;
+import org.apache.tez.tools.javadoc.util.HtmlWriter;
+import org.apache.tez.tools.javadoc.util.XmlWriter;
+
+import javax.lang.model.SourceVersion;
+import javax.lang.model.element.AnnotationMirror;
+import javax.lang.model.element.AnnotationValue;
+import javax.lang.model.element.Element;
+import javax.lang.model.element.ElementKind;
+import javax.lang.model.element.ExecutableElement;
+import javax.lang.model.element.Modifier;
+import javax.lang.model.element.TypeElement;
+import javax.lang.model.element.VariableElement;
+import javax.tools.Diagnostic;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+public class ConfigStandardDoclet implements Doclet {
+
+  private static boolean debugMode = false;
+
+  private Reporter reporter;
+
+  private static String stripQuotes(String s) {
+    if (s.charAt(0) == '"' && s.charAt(s.length() - 1) == '"') {
+      return s.substring(1, s.length() - 1);
+    }
+    return s;
+  }
+
+  @Override
+  public void init(Locale locale, Reporter reporter) {
+    this.reporter = reporter;
+  }
+
+  @Override
+  public String getName() {
+    return "Tez";
+  }
+
+  @Override
+  public SourceVersion getSupportedSourceVersion() {
+    return SourceVersion.RELEASE_9;
+  }
+
+  private void logMessage(String message) {
+    if (!debugMode) {
+      return;
+    }
+    reporter.print(Diagnostic.Kind.NOTE, message);
+  }
+
+  @Override
+  public boolean run(DocletEnvironment docEnv) {
+    logMessage("Running doclet " + ConfigStandardDoclet.class.getSimpleName());
+    DocTrees docTrees = docEnv.getDocTrees();
+    for (Element element : docEnv.getIncludedElements()) {
+      if (element.getKind().equals(ElementKind.CLASS) && element instanceof TypeElement) {
+        processDoc(docTrees, (TypeElement) element);
+      }
+    }
+
+    return true;
+  }
+
+  private void processDoc(DocTrees docTrees, TypeElement doc) {
+    logMessage("Parsing : " + doc);
+    if (!doc.getKind().equals(ElementKind.CLASS)) {
+      logMessage("Ignoring non-class: " + doc);
+      return;
+    }
+
+    List<? extends AnnotationMirror> annotations = doc.getAnnotationMirrors();
+    boolean isConfigClass = false;
+    String templateName = null;
+    for (AnnotationMirror annotation : annotations) {
+      logMessage("Checking annotation: " + annotation.getAnnotationType());
+      if (annotation.getAnnotationType().asElement().toString().equals(
+          ConfigurationClass.class.getName())) {
+        isConfigClass = true;
+        Map<? extends ExecutableElement, ? extends AnnotationValue> elementValues = annotation.getElementValues();
+        for (Map.Entry<? extends ExecutableElement, ? extends AnnotationValue> element : elementValues.entrySet()) {
+          if (element.getKey().getSimpleName().toString().equals("templateFileName")) {
+            templateName = stripQuotes(element.getValue().getValue().toString());
+          }
+        }
+        break;
+      }
+    }
+
+    if (!isConfigClass) {
+      logMessage("Ignoring non-config class: " + doc);
+      return;
+    }
+
+    logMessage("Processing config class: " + doc);
+    Config config = new Config(doc.getSimpleName().toString(), templateName);
+    Map<String, ConfigProperty> configProperties = config.getConfigProperties();
+
+    processElements(docTrees, doc, configProperties);
+
+    HtmlWriter writer = new HtmlWriter();
+    try {
+      writer.write(config);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+
+    XmlWriter xmlWriter = new XmlWriter();
+    try {
+      xmlWriter.write(config);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private void processElements(DocTrees docTrees, TypeElement doc, Map<String, ConfigProperty> configProperties) {
+    List<? extends Element> elements = doc.getEnclosedElements();
+    for (Element f : elements) {
+      if (!(f instanceof VariableElement)) {
+        continue;
+      }
+
+      if (!f.getKind().equals(ElementKind.FIELD)) {
+        continue;
+      }
+
+      VariableElement field = (VariableElement) f;
+
+      if (field.getModifiers().contains(Modifier.PRIVATE)) {
+        logMessage("Skipping private field: " + field);
+        continue;
+      }
+      if (field.getModifiers().contains(Modifier.STATIC)) {
+        logMessage("Skipping non-static field: " + field);
+        continue;
+      }
+
+      String fieldName = field.getSimpleName().toString();
+      if (fieldName.endsWith("_PREFIX")) {
+        logMessage("Skipping non-config prefix constant field: " + field);
+        continue;
+      }
+      if (fieldName.equals("TEZ_SITE_XML")) {
+        logMessage("Skipping constant field: " + field);
+        continue;
+      }
+
+      if (fieldName.endsWith("_DEFAULT")) {
+
+        String name = fieldName.substring(0, fieldName.lastIndexOf("_DEFAULT"));
+        if (!configProperties.containsKey(name)) {
+          configProperties.put(name, new ConfigProperty());
+        }
+        ConfigProperty configProperty = configProperties.get(name);
+        if (field.getConstantValue() == null) {
+          String val = field.getConstantValue().toString();
+          logMessage("Got null constant value"
+              + ", name=" + name
+              + ", field=" + fieldName
+              + ", val=" + val);
+          configProperty.setDefaultValue(val);
+        } else {
+          configProperty.setDefaultValue(field.getConstantValue().toString());
+        }
+        configProperty.setInferredType(field.getSimpleName().toString());
+
+        if (name.equals("TEZ_AM_STAGING_DIR") && configProperty.getDefaultValue() != null) {
+          String defaultValue = configProperty.getDefaultValue();
+          defaultValue = defaultValue.replace(System.getProperty("user.name"), "${user.name}");
+          configProperty.setDefaultValue(defaultValue);
+        }
+
+        continue;
+      }
+
+      if (!configProperties.containsKey(fieldName)) {
+        configProperties.put(fieldName, new ConfigProperty());
+      }
+      ConfigProperty configProperty = configProperties.get(fieldName);
+      configProperty.setPropertyName(field.getConstantValue().toString());
+
+      List<? extends AnnotationMirror> annotationDescs = field.getAnnotationMirrors();
+
+      for (AnnotationMirror annotationDesc : annotationDescs) {
+        String elementFqName = annotationDesc.getAnnotationType().asElement().toString();
+        if (elementFqName.equals(Private.class.getCanonicalName())) {
+          configProperty.setPrivate(true);
+        }
+        if (elementFqName.equals(Unstable.class.getCanonicalName())) {
+          configProperty.setUnstable(true);
+        }
+        if (elementFqName.equals(Evolving.class.getCanonicalName())) {
+          configProperty.setEvolving(true);
+        }
+        if (elementFqName.equals(ConfigurationProperty.class.getCanonicalName())) {
+          configProperty.setValidConfigProp(true);
+
+          for (Map.Entry<? extends ExecutableElement, ? extends AnnotationValue> element
+              : annotationDesc.getElementValues().entrySet()) {
+            if (element.getKey().getSimpleName().toString().equals("type")) {
+              configProperty.setType(stripQuotes(element.getValue().getValue().toString()));
+            } else {
+              logMessage("Unhandled annotation property: " + element.getKey().getSimpleName());
+            }
+          }
+        }
+      }
+      configProperty.setDescription(docTrees.getDocCommentTree(field).getFullBody().toString());
+    }
+  }
+
+  @Override
+  public Set<? extends Option> getSupportedOptions() {
+    Option[] options = {
+        new Option() {
+          private final List<String> someOption = Arrays.asList(
+              "-debug",
+              "--debug"
+          );
+
+          @Override
+          public int getArgumentCount() {
+            return 0;
+          }
+
+          @Override
+          public String getDescription() {
+            return "Debug mode";
+          }
+
+          @Override
+          public Option.Kind getKind() {
+            return Kind.STANDARD;
+          }
+
+          @Override
+          public List<String> getNames() {
+            return someOption;
+          }
+
+          @Override
+          public String getParameters() {
+            return "";
+          }
+
+          @Override
+          public boolean process(String opt, List<String> arguments) {
+            debugMode = true;
+            return true;
+          }
+        }
+    };
+    return new HashSet<>(Arrays.asList(options));
+  }
+}