variables directory processing

src/main/java/io/openliberty/tools/common/plugins/config/ServerConfigDocument.java

@@ -24,6 +24,7 @@
 import java.net.URLConnection;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -143,6 +144,24 @@ public ServerConfigDocument(CommonLoggerI log) {
         initializeFields(log, null, null, null);
     }
 
+    public void initializeFields(CommonLoggerI log, File serverXML, File configDir, Map<String, File> libertyDirPropertyFiles) {
+        this.log = log;
+        serverXMLFile = serverXML;
+        configDirectory = configDir;
+        if (libertyDirPropertyFiles != null) {
+            libertyDirectoryPropertyToFile = new HashMap<String, File>(libertyDirPropertyFiles);
+        } else {
+            log.warn("The properties for directories are null and could lead to application locations not being resolved correctly.");
+            libertyDirectoryPropertyToFile = new HashMap<String,File>();
+        }
+        locations = new HashSet<String>();
+        names = new HashSet<String>();
+        namelessLocations = new HashSet<String>();
+        locationsAndNames = new HashMap<String, String>();
+        props = new Properties();
+        defaultProps = new Properties();
+    }
+
     private DocumentBuilder getDocumentBuilder() {
         DocumentBuilder docBuilder;
 
@@ -187,11 +206,25 @@ private void initializeAppsLocation(CommonLoggerI log, File serverXML, File conf
             processServerEnv();
 
             // 3. get variables from jvm.options
-            processJvmOptions();
+            // processJvmOptions();
 
             // 3. get variables from bootstrap.properties
             processBootstrapProperties(bootstrapProp, bootstrapFile);
 
+            // 4. Java system properties
+            // configured in Maven/Gradle
+
+            // 5. Variables loaded from 'variables' directory
+            processVariablesDirectory();
+
+            // 6. variable values declared in server.xml
+            // processVariablesForValues(doc);
+
+            // 7. variables delcared on the command line
+            // configured in Maven/Gradle
+
+            // TODO: cleanup rest
+
             // 4. parse variables from include files (both default and non-default values - which we store separately)
             parseIncludeVariables(doc);
 
@@ -216,23 +249,10 @@ private void initializeAppsLocation(CommonLoggerI log, File serverXML, File conf
         }
     }
 
-
-    /**
-     * jvm.options file read order
-     *   1. {wlp.user.dir}/
-     *   2. {server.config.dir}/configDropins/defaults/
-     *   3. {server.config.dir}/
-     *   4. {server.config.dir}/configDropins/overrides/
-     * TODO: potential processing for system property tags? (-D prefixes?)
-     * @throws FileNotFoundException
-     * @throws Exception
-     */
-    public void processJvmOptions() throws FileNotFoundException, Exception {
-        final String jvmOptionsString = "jvm.options";
-        parsePropertiesFromFile(new File(libertyDirectoryPropertyToFile.get(ServerFeatureUtil.WLP_USER_DIR), jvmOptionsString));
-        parsePropertiesFromFile(getFileFromConfigDirectory(CONFIGDROPINS_DEFAULT + File.separator + jvmOptionsString));
-        parsePropertiesFromFile(getFileFromConfigDirectory(jvmOptionsString));
-        parsePropertiesFromFile(getFileFromConfigDirectory(CONFIGDROPINS_OVERRIDES + File.separator + jvmOptionsString));
+    public void parsePropertiesFromFile(File propertiesFile) throws Exception, FileNotFoundException {
+        if (propertiesFile != null && propertiesFile.exists()) {
+            parseProperties(new FileInputStream(propertiesFile));
+        }
     }
 
     /**
@@ -251,28 +271,17 @@ public void processServerEnv() throws Exception, FileNotFoundException {
         parsePropertiesFromFile(new File(libertyDirectoryPropertyToFile.get(ServerFeatureUtil.SERVER_CONFIG_DIR), serverEnvString));
     }
 
-    public void parsePropertiesFromFile(File propertiesFile) throws Exception, FileNotFoundException {
-        if (propertiesFile != null && propertiesFile.exists()) {
-            parseProperties(new FileInputStream(propertiesFile));
-        }
-    }
-
-    public void initializeFields(CommonLoggerI log, File serverXML, File configDir, Map<String, File> libertyDirPropertyFiles) {
-        this.log = log;
-        serverXMLFile = serverXML;
-        configDirectory = configDir;
-        if (libertyDirPropertyFiles != null) {
-            libertyDirectoryPropertyToFile = new HashMap<String, File>(libertyDirPropertyFiles);
-        } else {
-            log.warn("The properties for directories are null and could lead to application locations not being resolved correctly.");
-            libertyDirectoryPropertyToFile = new HashMap<String,File>();
-        }
-        locations = new HashSet<String>();
-        names = new HashSet<String>();
-        namelessLocations = new HashSet<String>();
-        locationsAndNames = new HashMap<String, String>();
-        props = new Properties();
-        defaultProps = new Properties();
+    /**
+     * Likely not needed to be processed by the LMP/LGP tools. These properties benefit the JVM.
+     * @throws FileNotFoundException
+     * @throws Exception
+     */
+    public void processJvmOptions() throws FileNotFoundException, Exception {
+        final String jvmOptionsString = "jvm.options";
+        parsePropertiesFromFile(new File(libertyDirectoryPropertyToFile.get(ServerFeatureUtil.WLP_USER_DIR), jvmOptionsString));
+        parsePropertiesFromFile(getFileFromConfigDirectory(CONFIGDROPINS_DEFAULT + File.separator + jvmOptionsString));
+        parsePropertiesFromFile(getFileFromConfigDirectory(jvmOptionsString));
+        parsePropertiesFromFile(getFileFromConfigDirectory(CONFIGDROPINS_OVERRIDES + File.separator + jvmOptionsString));
     }
 
     /**
@@ -326,7 +335,7 @@ private void processBootstrapInclude(String bootstrapIncludeLocation, Set<String
         if (processedBootstrapIncludes.contains(bootstrapIncludeFile.getAbsolutePath())) {
             return;
         }
-        
+
         if (bootstrapIncludeFile.exists()) {
             parseProperties(new FileInputStream(bootstrapIncludeFile));
             processedBootstrapIncludes.add(bootstrapIncludeFile.getAbsolutePath());
@@ -338,6 +347,77 @@ private String getBootstrapIncludeProperty() {
         return props.getProperty("bootstrap.include");
     }
 
+    /**
+     * By default, ${server.config.directory}/variables is processed.
+     * If VARIABLE_SOURCE_DIRS is defined, those directories are processed instead.
+     * The path delimiter for the property is ';' on Windows, and is ':' on Unix
+     * @throws Exception 
+     * @throws FileNotFoundException 
+     */
+    public void processVariablesDirectory() throws FileNotFoundException, Exception {
+        final String variableDirectoryProperty = "VARIABLE_SOURCE_DIRS";
+
+        ArrayList<File> toProcess = new ArrayList<File>();
+        if (!props.containsKey(variableDirectoryProperty)) {
+            toProcess.add(getFileFromConfigDirectory("variables"));
+        } else {
+            String delimiter = (File.separator.equals("/")) ? ":" : ";";    // OS heuristic
+            String[] splitDirectories = props.get(variableDirectoryProperty).toString().split(delimiter);
+            for (String directory : splitDirectories) {
+                Path directoryPath = Paths.get(directory);
+                File directoryFile = directoryPath.toFile();
+                if (directoryFile.exists()) {
+                    toProcess.add(directoryFile);
+                }
+            }
+        }
+
+        processVariableSourceDirs(toProcess);
+    }
+
+    /**
+     * The file name is the variable and the contents are the values.
+     * If a directory is within the directory, it is recurisvely processed. 
+     * The parent dir gets prepended to the file name for the property name ("{parent directory}/{file name}")
+     * If the file name ends with *.properties, then it's processed as a properties file.
+     * @param directories - The root directories to process
+     * @throws Exception 
+     * @throws FileNotFoundException 
+     */
+    public void processVariableSourceDirs(ArrayList<File> directories) throws FileNotFoundException, Exception {
+        for (File directory : directories) {
+            if (!directory.isDirectory()) {
+                continue;
+            }
+            processNestedVariableSourceDirs(directory, "");
+        }  
+    }
+
+    /**
+     * The nested operation
+     * @param directory      - The directory being processed
+     * @param propertyPrefix - Tracks the nested directories to prepend
+     * @throws FileNotFoundException
+     * @throws Exception
+     */
+    public void processNestedVariableSourceDirs(File directory, String propertyPrefix) throws FileNotFoundException, Exception {
+        for (File child : directory.listFiles()) {
+            if (child.isDirectory()) {
+                processNestedVariableSourceDirs(child, child.getName() + File.separator);
+                continue;
+            }
+
+            if (child.getName().endsWith(".properties")) {
+                parsePropertiesFromFile(child);
+                continue;
+            }
+
+            String propertyName = propertyPrefix + child.getName();
+            String propertyValue = Files.readString(child.toPath());
+            props.setProperty(propertyName, propertyValue);
+        }
+    }
+
     //Checks for application names in the document. Will add locations without names to a Set
     private void parseNames(Document doc, String expression) throws XPathExpressionException, IOException, SAXException {
         // parse input document

src/main/java/io/openliberty/tools/common/plugins/util/VariableUtility.java

@@ -28,6 +28,10 @@ public class VariableUtility {
     private static final String VARIABLE_NAME_PATTERN = "\\$\\{(.*?)\\}";
     private static final Pattern varNamePattern = Pattern.compile(VARIABLE_NAME_PATTERN);
 
+    // If a property is not immediately found, replace non-alphanumeric values with '_'. If still not found, search with toUpper
+    // Integer value properties can be evaluated if 'simple' arithemetic
+    // A list of ports can be defined using keyword 'list'
+
     /**
      * Attempts to resolve all variables in the passed in nodeValue. Variable value/defaultValue can reference other variables.
      * This method is called recursively to resolve the variables. The variableChain collection keeps track of the variable references

src/test/java/io/openliberty/tools/common/config/ServerConfigDocumentTest.java

@@ -12,6 +12,7 @@
 import java.util.Map;
 import java.util.Properties;
 
+import javax.xml.bind.annotation.XmlElement.DEFAULT;
 import javax.xml.xpath.XPathExpressionException;
 
 import org.junit.Test;
@@ -37,14 +38,14 @@ public class ServerConfigDocumentTest {
     private final static Path RESOURCES_DIR = Paths.get("src/test/resources/");
     private final static Path WLP_DIR = RESOURCES_DIR.resolve("serverConfig/liberty/wlp/");
     private final static Path WLP_USER_DIR = RESOURCES_DIR.resolve("serverConfig/liberty/wlp/usr/");
-    private final static Path DEFAULTSERVER_CONFIG_DIR = WLP_USER_DIR.resolve("servers/defaultServer");
-    private final static Path MOCK_SERVER_DIR = RESOURCES_DIR.resolve("servers/");
+    private final static Path SERVER_CONFIG_DIR = WLP_USER_DIR.resolve("servers/defaultServer");
+    private final static Path SERVERS_RESOURCES_DIR = RESOURCES_DIR.resolve("servers/");
 
     // 1. variable default values in server.xml file
     // 6. variable values declared in the server.xml file
     @Test
     public void processServerXml() throws FileNotFoundException, IOException, XPathExpressionException, SAXException {
-        File serversDir = MOCK_SERVER_DIR.toFile();
+        File serversDir = SERVERS_RESOURCES_DIR.toFile();
         Document doc;
 
         // no variables defined
@@ -88,9 +89,9 @@ public void processServerXml() throws FileNotFoundException, IOException, XPathE
     //   2. change all characters to uppercase
     @Test
     public void serverXmlEnvVarVariationLookup() throws FileNotFoundException, Exception {
-        File serverXml = DEFAULTSERVER_CONFIG_DIR.resolve("server.xml").toFile();
+        File serverXml = SERVER_CONFIG_DIR.resolve("server.xml").toFile();
         ServerConfigDocument configDocument = new ServerConfigDocument(new TestLogger());
-        configDocument.initializeFields(new TestLogger(), serverXml, DEFAULTSERVER_CONFIG_DIR.toFile(), new HashMap<>());
+        configDocument.initializeFields(new TestLogger(), serverXml, SERVER_CONFIG_DIR.toFile(), new HashMap<>());
         Document serverXmlDoc = configDocument.parseDocument(serverXml);
         configDocument.parseVariablesForBothValues(serverXmlDoc);
         assertEquals("${this.value}", configDocument.getDefaultProperties().getProperty("server.env.defined"));
@@ -113,7 +114,7 @@ public void serverXmlEnvVarVariationLookup() throws FileNotFoundException, Excep
     public void processServerEnv() throws FileNotFoundException, Exception {
         File wlpInstallDir = WLP_DIR.toFile();
         File wlpUserDir = WLP_USER_DIR.toFile();
-        File serverDir = DEFAULTSERVER_CONFIG_DIR.toFile();
+        File serverDir = SERVER_CONFIG_DIR.toFile();
         ServerConfigDocument configDocument = new ServerConfigDocument(new TestLogger());
         Map<String, File> libertyDirectoryPropertyToFileMap = new HashMap<String, File>();
         libertyDirectoryPropertyToFileMap.put("wlp.install.dir", wlpInstallDir);
@@ -145,7 +146,7 @@ public void environmentVariables() throws FileNotFoundException, Exception {
     // 3. bootstrap.properties
     @Test
     public void processBootstrapProperties() throws FileNotFoundException, Exception {
-        File serversDir = MOCK_SERVER_DIR.toFile();
+        File serversDir = SERVERS_RESOURCES_DIR.toFile();
         ServerConfigDocument configDocument;
 
         // bootstrap.properties in config dir
@@ -157,7 +158,7 @@ public void processBootstrapProperties() throws FileNotFoundException, Exception
         // use bootstrapFile, kept for flexibility
         configDocument = new ServerConfigDocument(new TestLogger());
         configDocument.initializeFields(new TestLogger(), null, serversDir, null);
-        configDocument.processBootstrapProperties(new HashMap<>(), DEFAULTSERVER_CONFIG_DIR.resolve("bootstrap.properties").toFile());
+        configDocument.processBootstrapProperties(new HashMap<>(), SERVER_CONFIG_DIR.resolve("bootstrap.properties").toFile());
         assertEquals(2, configDocument.getProperties().size());
         assertEquals("DEFINED", configDocument.getProperties().getProperty("THAT_VALUE"));
 
@@ -175,13 +176,13 @@ public void processBootstrapProperties() throws FileNotFoundException, Exception
         // bootstrap.include
         configDocument = new ServerConfigDocument(new TestLogger());
         configDocument.initializeFields(new TestLogger(), null, serversDir, null);
-        configDocument.processBootstrapProperties(new HashMap<>(), MOCK_SERVER_DIR.resolve("bootstrapInclude.properties").toFile());
+        configDocument.processBootstrapProperties(new HashMap<>(), SERVERS_RESOURCES_DIR.resolve("bootstrapInclude.properties").toFile());
         assertEquals("extraFeatures.xml", configDocument.getProperties().getProperty("extras.filename"));
 
         // bootstrap.include infinite termination check
         configDocument = new ServerConfigDocument(new TestLogger());
         configDocument.initializeFields(new TestLogger(), null, serversDir, null);
-        configDocument.processBootstrapProperties(new HashMap<>(), MOCK_SERVER_DIR.resolve("bootstrapOuroboros.properties").toFile());
+        configDocument.processBootstrapProperties(new HashMap<>(), SERVERS_RESOURCES_DIR.resolve("bootstrapOuroboros.properties").toFile());
     }
 
     // 4. Java system properties
@@ -193,12 +194,36 @@ public void jvmOptions() {
     // 5. Variables loaded from files in the ${server.config.dir}/variables directory or other 
     //    directories as specified by the VARIABLE_SOURCE_DIRS environment variable
     @Test
-    public void variablesDir() {
-        // TODO: not yet implemented. read copied dir instead of src for now
-        // see https://github.com/OpenLiberty/ci.common/issues/126
+    public void variablesDir() throws FileNotFoundException, Exception {
+        File serversDir = SERVER_CONFIG_DIR.toFile();
+        ServerConfigDocument configDocument = new ServerConfigDocument(new TestLogger());
+        configDocument.initializeFields(new TestLogger(), null, serversDir, null);
+        configDocument.processVariablesDirectory();
+
+        Properties props = configDocument.getProperties(); 
+        assertEquals("9080", props.getProperty("httpPort"));
+        assertEquals("1000", props.getProperty("nested/httpPort"));
+        assertEquals("1", props.getProperty("VALUE_1"));
+        assertEquals("2", props.getProperty("VALUE_2"));
+
+        // process VARIABLE_SOURCE_DIRS
+        configDocument = new ServerConfigDocument(new TestLogger());
+        configDocument.initializeFields(new TestLogger(), null, serversDir, null);
+        Map<String, String> bootstrapProp = new HashMap<String, String>();
+        String delimiter = (File.separator.equals("/")) ? ":" : ";";
+        String variableSourceDirsTestValue = String.join(delimiter, 
+                SERVERS_RESOURCES_DIR.resolve("variables").toString(), 
+                SERVER_CONFIG_DIR.toString(), 
+                "DOES_NOT_EXIST");
+        bootstrapProp.put("VARIABLE_SOURCE_DIRS", variableSourceDirsTestValue);
+        configDocument.processBootstrapProperties(bootstrapProp, null);
+        configDocument.processVariablesDirectory();
+
+        props = configDocument.getProperties();
+        assertEquals("outer_space", props.getProperty("outer.source"));
+        assertEquals("1", props.getProperty("VALUE_1"));
     }
 
-
     // 7. variables declared on the command line
     @Test
     public void CLI() {
@@ -211,7 +236,7 @@ public void CLI() {
     // server.xml override bootstrap.properties, jvm.options, and server.env
     @Test
     public void overrides() {
-        File serversDir = DEFAULTSERVER_CONFIG_DIR.toFile();
+        File serversDir = SERVER_CONFIG_DIR.toFile();
         // server.xml overrides server.env
         ServerConfigDocument configDocument = new ServerConfigDocument(new TestLogger(), 
                 new File(serversDir, "server.xml"), serversDir, new File(serversDir, "bootstrap.properties"), 

src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/httpPort

@@ -0,0 +1 @@
+9080

src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/nested/httpPort

@@ -0,0 +1 @@
+1000

src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/nested/nested.properties

@@ -0,0 +1,2 @@
+VALUE_1=1
+VALUE_2=2

src/test/resources/servers/variables/outer.source

@@ -0,0 +1 @@
+outer_space