Draft: Extract more parameters

config.txt

@@ -2,41 +2,14 @@
 
 -- parameter classes that occur under scm
 -- (all these classes have same documentation present at 10 different places)
-$class: 'CCUCMScm'
-$class: 'CVSSCM'
-$class: 'CvsProjectset'
-scmGit
-$class: 'GitSCM'
-$class: 'MercurialSCM'
-$class: 'RTCScm'
-$class: 'SubversionSCM'
-$class: 'hudson.plugins.repo.RepoScm'
-$class: 'MultiSCM'
+hudson.scm.SCM
 
 -- only present in workflow-multibranch.adoc
-$class: 'JobRestrictionProperty'
-pipelineTriggers
-$class: 'YouTrackProjectProperty'
-$class: 'it.dockins.dockerslaves.spec.ContainerSetDefinition'
+hudson.model.JobProperty
 
 -- common for workflow-multibranch.adoc, hubot-steps.adoc, pipeline-input-step.adoc
-$class: 'ExtensibleChoiceParameterDefinition'
-$class: 'RunFilterParameter'
-$class: 'RunSelectorParameter'
+hudson.model.ParameterDefinition
+
 
 -- common for workflow-multibranch.adoc, pipeline-groovy-lib.adoc
-$class: 'BacklogPullRequestSCMSource'
-multiBranch
-dagshubScmSource
-git
-$class: 'GiteaSCMSource'
-multiGraph
-$class: 'MercurialSCMSource'
-scmManager
-scmManagerSvn
-fromScm
-multiStreams
-$class: 'SubversionSCMSource'
-multiSwarm
-Tuleap
-bitbucket
+jenkins.scm.api.SCMSource

src/main/java/org/jenkinsci/pipeline_steps_doc_generator/DeclarativeSteps.java

@@ -44,7 +44,7 @@ public void generateDeclarativeAscii(String declarativeDest, HyperLocalPluginMan
         }
         declDest.mkdirs();
         String declPath = declDest.getAbsolutePath();
-
+        ToAsciiDoc toAsciiDoc = new ToAsciiDoc(pluginManager);
         Map<Class<? extends Descriptor>, Predicate<Descriptor>> filters = getDeclarativeFilters();
 
         for (Map.Entry<String, List<Class<? extends Descriptor>>> entry :
@@ -58,7 +58,7 @@ public void generateDeclarativeAscii(String declarativeDest, HyperLocalPluginMan
                 pluginDescMap = processDescriptors(d, pluginDescMap, filter, pluginManager);
             }
 
-            String whole9yards = ToAsciiDoc.generateDirectiveHelp(entry.getKey(), pluginDescMap, true);
+            String whole9yards = toAsciiDoc.generateDirectiveHelp(entry.getKey(), pluginDescMap, true);
 
             try {
                 Files.writeString(

src/main/java/org/jenkinsci/pipeline_steps_doc_generator/PipelineStepExtractor.java

@@ -4,7 +4,6 @@
 import static org.mockito.Mockito.when;
 
 import hudson.MockJenkins;
-import hudson.PluginManager;
 import hudson.PluginWrapper;
 import hudson.init.InitMilestone;
 import hudson.init.InitStrategy;
@@ -18,6 +17,7 @@
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
@@ -236,7 +236,8 @@ public int compare(StepDescriptor o1, StepDescriptor o2) {
         private static final long serialVersionUID = 1L;
     }
 
-    public void generateAscii(Map<String, Map<String, List<QuasiDescriptor>>> allSteps, PluginManager pluginManager) {
+    public void generateAscii(
+            Map<String, Map<String, List<QuasiDescriptor>>> allSteps, HyperLocalPluginManager pluginManager) {
         File allAscii;
         if (asciiDest != null) {
             allAscii = new File(asciiDest);
@@ -259,25 +260,38 @@ public void generateAscii(Map<String, Map<String, List<QuasiDescriptor>>> allSte
 
         allAscii.mkdirs();
         String allAsciiPath = allAscii.getAbsolutePath();
+
+        ToAsciiDoc toAsciiDoc = new ToAsciiDoc(pluginManager);
         for (String plugin : allSteps.keySet()) {
             LOG.info("processing " + plugin);
             Map<String, List<QuasiDescriptor>> byPlugin = allSteps.get(plugin);
             PluginWrapper thePlugin = pluginManager.getPlugin(plugin);
             String displayName = thePlugin == null ? "Jenkins Core" : thePlugin.getDisplayName();
             boolean isDeprecated = deprecatedPlugins.has(plugin);
-            String whole9yards = ToAsciiDoc.generatePluginHelp(plugin, displayName, byPlugin, isDeprecated, true);
+            String whole9yards = toAsciiDoc.generatePluginHelp(plugin, displayName, byPlugin, isDeprecated, true);
 
             try {
+                Paths.get(allAsciiPath, plugin).toFile().mkdirs();
                 Files.writeString(
-                        new File(allAsciiPath, plugin + ".adoc").toPath(), whole9yards, StandardCharsets.UTF_8);
+                        new File(allAsciiPath, plugin + "/index.adoc").toPath(), whole9yards, StandardCharsets.UTF_8);
             } catch (Exception ex) {
                 LOG.log(Level.SEVERE, "Error generating plugin file for " + plugin + ".  Skip.", ex);
                 // continue to next plugin
             }
         }
-        ProcessAsciiDoc pad = new ProcessAsciiDoc();
-        new File(allAsciiPath + "/params").mkdirs();
-        pad.processDocs(allAsciiPath, 100);
+        for (Map.Entry<String, StringBuilder> entry :
+                toAsciiDoc.getExtractedParams().entrySet()) {
+            String plugin = entry.getKey();
+            try {
+                Paths.get(allAsciiPath, plugin).toFile().mkdirs();
+                Files.writeString(
+                        new File(allAsciiPath, entry.getKey() + "/params.adoc").toPath(),
+                        entry.getValue().toString(),
+                        StandardCharsets.UTF_8);
+            } catch (IOException ex) {
+                LOG.log(Level.SEVERE, "Error generating plugin params file for " + entry.getKey() + ".  Skip.", ex);
+            }
+        }
     }
 
     public void generateDeclarativeSteps() {

src/main/java/org/jenkinsci/pipeline_steps_doc_generator/ToAsciiDoc.java

@@ -1,12 +1,16 @@
 package org.jenkinsci.pipeline_steps_doc_generator;
 
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import hudson.ExtensionList;
 import hudson.Main;
 import hudson.model.Descriptor;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -18,6 +22,7 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
+import org.jenkinsci.infra.tools.HyperLocalPluginManager;
 import org.jenkinsci.plugins.structs.SymbolLookup;
 import org.jenkinsci.plugins.structs.describable.ArrayType;
 import org.jenkinsci.plugins.structs.describable.AtomicType;
@@ -39,6 +44,9 @@ public class ToAsciiDoc {
     private static final Map<String, String> typeDescriptions = new HashMap<>();
     public static final String ARRAY_LIST_OF = "Array / List of ";
     public static final String BUILD_STEP_DESCRIPTION = "Build Step (<code>hudson.tasks.BuildStep</code>)";
+    private final HyperLocalPluginManager pluginManager;
+
+    private static final List<Class<?>> extractable = new ArrayList<>();
 
     static {
         typeDescriptions.put("java.lang.Object", "<code>Object</code>");
@@ -49,7 +57,28 @@ public class ToAsciiDoc {
     /**
      * Keeps track of nested {@link DescribableModel#getType()} to avoid recursion.
      */
-    private static Stack<Class<?>> nesting = new Stack<>();
+    private Stack<Class<?>> nesting = new Stack<>();
+
+    public Map<String, StringBuilder> getExtractedParams() {
+        return extractedParams;
+    }
+
+    private Map<String, StringBuilder> extractedParams = new HashMap<>();
+    private Map<String, String> map;
+
+    public ToAsciiDoc(HyperLocalPluginManager pluginManager) {
+        this.pluginManager = pluginManager;
+        this.map = new HashMap<>();
+        for (Map.Entry<String, String> entry :
+                pluginManager.uberPlusClassLoader.getByPlugin().entrySet()) {
+            ExtensionList<Descriptor> descriptors = ExtensionList.lookup(Descriptor.class);
+            for (Descriptor d : descriptors) {
+                if (entry.getKey().equals(d.getClass().getName())) {
+                    map.put(d.clazz.getName(), entry.getValue());
+                }
+            }
+        }
+    }
 
     /** Asciidoc conversion functions. **/
     private static String header(int depth) {
@@ -61,7 +90,7 @@ private static String helpify(String help) {
                 + "</div>\n";
     }
 
-    static String describeType(ParameterType type, String prefix) throws Exception {
+    String describeType(ParameterType type, String prefix) throws Exception {
         StringBuilder typeInfo = new StringBuilder();
         if (type instanceof EnumType) {
             typeInfo.append("<li><b>")
@@ -84,19 +113,39 @@ static String describeType(ParameterType type, String prefix) throws Exception {
                     .append(generateHelp(((HomogeneousObjectType) type).getSchemaType(), false));
         } else if (type instanceof HeterogeneousObjectType) {
             typeInfo.append("<b>").append(prefix).append("Nested Choice of Objects</b>\n");
-            if (((HeterogeneousObjectType) type).getType() != Object.class) {
+            HeterogeneousObjectType heterogeneousObjectType = (HeterogeneousObjectType) type;
+            if (heterogeneousObjectType.getType() != Object.class) {
                 for (Map.Entry<String, DescribableModel<?>> entry :
-                        ((HeterogeneousObjectType) type).getTypes().entrySet()) {
+                        heterogeneousObjectType.getTypes().entrySet()) {
                     Set<String> symbols =
                             SymbolLookup.getSymbolValue(entry.getValue().getType());
                     String symbol = symbols.isEmpty()
                             ? DescribableModel.CLAZZ + ": '" + entry.getKey() + "'"
                             : symbols.iterator().next();
-                    typeInfo.append("<li><code>")
-                            .append(symbol)
-                            .append("</code><div>\n")
-                            .append(generateHelp(entry.getValue(), true))
-                            .append("</div></li>\n");
+                    String help = generateHelp(entry.getValue(), true);
+                    if (shouldExtract(heterogeneousObjectType.getType())) {
+                        String pluginName = map.get(entry.getValue().getType().getName());
+                        extractedParams
+                                .computeIfAbsent(pluginName, (foo) -> new StringBuilder())
+                                .append(header(3))
+                                .append(symbol)
+                                .append("\n\n")
+                                .append(help)
+                                .append("\n");
+                        typeInfo.append("<li><a href=\"../")
+                                .append(pluginName)
+                                .append("/params#")
+                                .append(symbol)
+                                .append("\">")
+                                .append(symbol)
+                                .append("</a></li>\n");
+                    } else {
+                        typeInfo.append("<li><code>")
+                                .append(symbol)
+                                .append("</code><div>\n")
+                                .append(help)
+                                .append("</div></li>\n");
+                    }
                 }
             }
         } else if (type instanceof ErrorType) { // Shouldn't hit this; open a ticket
@@ -115,6 +164,28 @@ static String describeType(ParameterType type, String prefix) throws Exception {
         return typeInfo.toString();
     }
 
+    private boolean shouldExtract(Class<?> type) {
+        if (extractable.isEmpty()) {
+            try {
+                List<String> config = Files.readAllLines(Paths.get("config.txt"));
+                for (String className : config) {
+                    className = className.trim();
+                    if (className.startsWith("--") || className.isBlank()) {
+                        continue;
+                    }
+                    try {
+                        extractable.add(pluginManager.uberPlusClassLoader.loadClass(className));
+                    } catch (Exception e) {
+                        LOG.log(Level.WARNING, "Cannot load " + className);
+                    }
+                }
+            } catch (IOException ex) {
+                LOG.warning("Cannot load config");
+            }
+        }
+        return extractable.stream().anyMatch(c -> c.isAssignableFrom(type));
+    }
+
     private static String describeErrorType(ParameterType type) {
         return type.getActualType()
                 .toString()
@@ -125,7 +196,7 @@ private static String describeErrorType(ParameterType type) {
                 .replace("'", "&#39;");
     }
 
-    private static String generateAttrHelp(DescribableParameter param) throws Exception {
+    private String generateAttrHelp(DescribableParameter param) throws Exception {
         StringBuilder attrHelp = new StringBuilder();
         String help = param.getHelp();
         if (help != null && !help.equals("")) {
@@ -170,7 +241,7 @@ private static String getTypeDescription(DescribableParameter param) throws Exce
         return typeDesc;
     }
 
-    private static String generateHelp(DescribableModel<?> model, boolean indent) throws Exception {
+    private String generateHelp(DescribableModel<?> model, boolean indent) throws Exception {
         if (nesting.contains(model.getType())) return ""; // if we are recursing, cut the search
         nesting.push(model.getType());
 
@@ -221,7 +292,7 @@ private static String generateHelp(DescribableModel<?> model, boolean indent) th
      * Generate documentation for a plugin step.
      * For delegate steps adds example without Symbol.
      */
-    public static String generateStepHelp(QuasiDescriptor d) {
+    public String generateStepHelp(QuasiDescriptor d) {
         StringBuilder mkDesc =
                 new StringBuilder(header(3)).append(" `").append(d.getSymbol()).append("`: ");
         mkDesc.append(getDisplayName(d.real)).append("\n++++\n");
@@ -253,7 +324,7 @@ private static String getDisplayName(Descriptor<?> d) {
         return "(no description)";
     }
 
-    private static void appendSimpleStepDescription(StringBuilder mkDesc, Class<?> clazz) throws IOException {
+    private void appendSimpleStepDescription(StringBuilder mkDesc, Class<?> clazz) throws IOException {
         try {
             mkDesc.append(generateHelp(new DescribableModel<>(clazz), true));
         } catch (Exception ex) {
@@ -285,7 +356,7 @@ static String getHelp(String name, Class<?> type) throws IOException {
     /**
      * Generate documentation for a {@link Descriptor}
      */
-    private static String generateDescribableHelp(Descriptor<?> d) {
+    private String generateDescribableHelp(Descriptor<?> d) {
         if (d instanceof StepDescriptor) {
             return generateStepHelp(new QuasiDescriptor(d, null));
         } else {
@@ -330,13 +401,13 @@ private static String generateHeader(String pluginName) {
      *
      * @return String  total documentation for the page
      */
-    public static String generatePluginHelp(
+    public String generatePluginHelp(
             String pluginName,
             String displayName,
             Map<String, List<QuasiDescriptor>> byPlugin,
             boolean isDeprecated,
             boolean genHeader) {
-        Main.isUnitTest = true;
+        setUnitTest();
 
         // TODO: if condition
         StringBuilder whole9yards = new StringBuilder();
@@ -360,9 +431,13 @@ public static String generatePluginHelp(
         return whole9yards.toString();
     }
 
-    public static String generateDirectiveHelp(
-            String directiveName, Map<String, List<Descriptor>> descsByPlugin, boolean genHeader) {
+    private static void setUnitTest() {
         Main.isUnitTest = true;
+    }
+
+    public String generateDirectiveHelp(
+            String directiveName, Map<String, List<Descriptor>> descsByPlugin, boolean genHeader) {
+        setUnitTest();
         StringBuilder whole9yards = new StringBuilder();
         if (genHeader) {
             whole9yards.append(generateHeader(directiveName));