Improve metdata table code.

Summary:
Rename InputTable to MetadataTable
Add startup option for Roddy, MDTs are now passed in via
--usemetadatatable, the Roddy class is used to get the data.

A preview diff for you to see what changed in the Core

Merge branch 'develop' into metadataTable

Test Plan: Partly unit tested

Reviewers: kensche, warsow

Differential Revision: https://tbi-cb14.inet.dkfz-heidelberg.de/D190

Author: Michael Heinold

.gitignore

@@ -1,4 +1,5 @@
 .idea/workspace.xml
+out
 RoddyCore/build
 dist/bin/2*
 dist/build

RoddyCore/src/de/dkfz/roddy/config/AnalysisConfigurationProxy.java

@@ -9,11 +9,11 @@
 import java.util.Map;
 
 /**
-*  The AC Proxy is used to enable lazy loading of analyis configuration objects.
+ * The AC Proxy is used to enable lazy loading of analyis configuration objects.
  * It encapsulates a real AC object.
- *
+ * <p>
  * Created by michael on 20.03.15.
-*/
+ */
 public class AnalysisConfigurationProxy extends AnalysisConfiguration {
 
     private AnalysisConfiguration analysisConfiguration;
@@ -36,7 +36,7 @@ public AnalysisConfigurationProxy(AnalysisConfiguration parentConfiguration, Str
     }
 
     private synchronized AnalysisConfiguration checkAnalysisConfig() {
-        if(analysisConfiguration == null) {
+        if (analysisConfiguration == null) {
             analysisConfiguration = ConfigurationFactory.getInstance().lazyLoadAnalysisConfiguration(this);
         }
         return analysisConfiguration;
@@ -229,11 +229,15 @@ public RecursiveOverridableMapContainer getContainer(String id) {
     @Override
     public void setParent(Configuration c) {
         super.setParent(c);
+        if (c != null)
+            checkAnalysisConfig().setParent(c);
     }
 
     @Override
     public void addParent(Configuration p) {
         super.addParent(p);
+        if (p != null)
+            checkAnalysisConfig().addParent(p);
     }
 
     @Override

RoddyCore/src/de/dkfz/roddy/config/ConfigurationFactory.groovy

@@ -684,10 +684,9 @@ public class ConfigurationFactory {
         for (NodeChild analysis in nAnalyses.analysis) {
             String analysisID = extractAttributeText((NodeChild) analysis, "id");
             String analysisCfg = extractAttributeText((NodeChild) analysis, "configuration");
-            AnalysisConfiguration ac = new AnalysisConfigurationProxy(parentConfiguration, analysisID, analysisCfg, analysis); //
+            AnalysisConfiguration ac = new AnalysisConfigurationProxy(parentConfiguration, analysisID, analysisCfg, analysis);
             availableAnalyses[analysisID] = ac;
 
-//        availableAnalyses[analysisID] = ac;
             _loadAnalyses(analysis.subanalyses, ac).each {
                 String k, AnalysisConfiguration subConfig ->
                     availableAnalyses[analysisID + "-" + k] = subConfig;
@@ -742,14 +741,15 @@ public class ConfigurationFactory {
         String key = cvalueNode.@name.text();
         String value = cvalueNode.@value.text();
         String type = extractAttributeText(cvalueNode, "type", "string");
+        List<String> tags = extractAttributeText(cvalueNode, "tags", null)?.split(StringConstants.COMMA);
 
         //OK, here comes some sort of valuable hack. In the past it was so, that sometimes people forgot to set
         //any directory to "path". In case of the output directories, this was a bad thing! So we know about
         //this specific type of variable and just set it to path at any time.
         if (key.endsWith("OutputDirectory"))
             type = "path";
         String description = extractAttributeText(cvalueNode, "description");
-        return new ConfigurationValue(config, key, value, type, description);
+        return new ConfigurationValue(config, key, value, type, description, tags);
     }
 
 

RoddyCore/src/de/dkfz/roddy/config/ConfigurationValue.java

@@ -34,6 +34,8 @@ public class ConfigurationValue implements RecursiveOverridableMapContainer.Iden
      * A description or comment for a configuration value.
      */
     private final String description;
+    private final List<String> tags = new LinkedList<>();
+
     private boolean invalid = false;
 
     public ConfigurationValue(String id, String value) {
@@ -49,15 +51,17 @@ public ConfigurationValue(String id, String value, String type) {
     }
 
     public ConfigurationValue(Configuration config, String id, String value, String type) {
-        this(config, id, value, type, "");
+        this(config, id, value, type, "", null);
     }
 
-    public ConfigurationValue(Configuration config, String id, String value, String type, String description) {
+    public ConfigurationValue(Configuration config, String id, String value, String type, String description, List<String> tags) {
         this.id = id;
         this.value = value;
         this.configuration = config;
         this.type = type;
         this.description = description;
+        if (tags != null)
+            this.tags.addAll(tags);
     }
 
     public Configuration getConfiguration() {
@@ -115,7 +119,7 @@ public File toFile(Analysis analysis) {
 
             if (userID != null)
                 temp = replaceString(temp, "$USERNAME", userID);
-            if( groupID != null)
+            if (groupID != null)
                 temp = replaceString(temp, "$USERGROUP", groupID);
 
             String ud = FileSystemAccessProvider.getInstance().getUserDirectory().getAbsolutePath();
@@ -145,7 +149,7 @@ public File toFile(ExecutionContext context) {
         try {
             String temp = toFile(context.getAnalysis(), context.getDataSet()).getAbsolutePath();
             temp = checkAndCorrectPath(temp);
-            if(value.startsWith("${DIR_BUNDLED_FILES}") || value.startsWith("${DIR_RODDY}"))
+            if (value.startsWith("${DIR_BUNDLED_FILES}") || value.startsWith("${DIR_RODDY}"))
                 temp = Roddy.getApplicationDirectory().getAbsolutePath() + FileSystemAccessProvider.getInstance().getPathSeparator() + temp;
 
             if (temp.contains(ConfigurationConstants.CVALUE_PLACEHOLDER_EXECUTION_DIRECTORY)) {
@@ -228,7 +232,7 @@ public String toString() {
         String temp = value;
         if (configuration != null) {
             List<String> valueIDs = getIDsForParrentValues();
-            for(String vName : valueIDs) {
+            for (String vName : valueIDs) {
                 if (configuration.getConfigurationValues().hasValue(vName))
                     temp = temp.replace("${" + vName + '}', configuration.getConfigurationValues().get(vName).toString());
             }
@@ -253,6 +257,14 @@ public String getType() {
         return type;
     }
 
+    public List<String> getListOfTags() {
+        return tags;
+    }
+
+    public boolean hasTag(String tag) {
+        return tags.contains(tag);
+    }
+
     public EnumerationValue getEnumerationValueType() {
         Enumeration enumeration = null;
         if (getConfiguration().getEnumerations().hasValue("cvalueType"))
@@ -339,7 +351,9 @@ public String getValue() {
         return value;
     }
 
-    public String getDescription() { return description; }
+    public String getDescription() {
+        return description;
+    }
 
     public boolean isQuoteOnConversionSet() {
         return quoteOnConversion;

RoddyCore/src/de/dkfz/roddy/core/DataSet.java

@@ -75,15 +75,6 @@ public Project getProject() {
         return project;
     }
 
-    /**
-     * Get the metadata table for this dataset.
-     * @return
-     */
-    public BaseMetadataTable getMetadataTable() {
-        if(metadataTable == null)
-            metadataTable = MetadataTableFactory.getTable().subsetByDataset(this.id);
-        return metadataTable;
-    }
     /**
      * Returns a folder like i.e. results_per_pid/ICGC_PCA0xxx
      *

RoddyCore/src/de/dkfz/roddy/core/ProjectFactory.groovy

@@ -6,7 +6,8 @@ import de.dkfz.roddy.Roddy
 import de.dkfz.roddy.StringConstants
 import de.dkfz.roddy.client.cliclient.RoddyCLIClient;
 import de.dkfz.roddy.config.*
-import de.dkfz.roddy.config.validation.XSDValidator;
+import de.dkfz.roddy.config.validation.XSDValidator
+import de.dkfz.roddy.execution.io.MetadataTableFactory;
 import de.dkfz.roddy.plugins.LibrariesFactory;
 import de.dkfz.roddy.tools.RoddyConversionHelperMethods;
 import de.dkfz.roddy.tools.Tuple2
@@ -250,6 +251,9 @@ public class ProjectFactory {
         } else if (analysis.getRuntimeService() == null) {
             throw new RuntimeException("There is no runtime service class set for the selected analysis. This has to be set in either the project configuration or the analysis configuration.");
         } else {
+
+            // Try to build up the metadata table from here on. Project and analysis are ready.
+            MetadataTableFactory.getTable(analysis);
             return analysis;
         }
     }

RoddyCore/src/de/dkfz/roddy/core/RuntimeService.groovy

@@ -64,7 +64,7 @@ public abstract class RuntimeService extends CacheProvider {
 
         if(Roddy.isMetadataCLOptionSet()) {
 
-            BaseMetadataTable table = MetadataTableFactory.getTable()
+            BaseMetadataTable table = MetadataTableFactory.getTable(analysis)
             List<String> _datasets = table.listDatasets();
             String pOut = analysis.getOutputBaseDirectory().getAbsolutePath() + File.separator;
             return _datasets.collect { new DataSet(analysis, it, new File(pOut + it)) }