Merge branch 'master' of github.com:gchq/stroom

stroom-proxy/stroom-proxy-app/proxy-prod.yml.jinja2

@@ -178,9 +178,8 @@ proxyConfig:
 
   # If you want multiple forward destinations then you will need to edit this file directly
   # instead of using env var substitution
-  forwardDestinations:
-    - type: "post"
-      enabled: {{ true | envVar('FORWARDING_ENABLED') }}
+  forwardHttpDestinations:
+    - enabled: {{ true | envVar('FORWARDING_ENABLED') }}
       name: "downstream"
       forwardUrl: "{{ 'https://nginx/stroom/datafeed' | envVar('FORWARD_URL') }}"
       sslConfig:

stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/ProxyConfig.java

@@ -27,7 +27,6 @@
 import io.dropwizard.validation.ValidationMethod;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.stream.Stream;
 import javax.validation.constraints.AssertTrue;
@@ -93,10 +92,10 @@ public ProxyConfig() {
         receiveDataConfig = new ReceiveDataConfig();
         proxyRepoConfig = new ProxyRepoConfig();
         eventStoreConfig = new EventStoreConfig();
-        fileScanners = Collections.emptyList();
+        fileScanners = new ArrayList<>();
         aggregatorConfig = new AggregatorConfig();
-        forwardFileDestinations = Collections.emptyList();
-        forwardHttpDestinations = Collections.emptyList();
+        forwardFileDestinations = new ArrayList<>();
+        forwardHttpDestinations = new ArrayList<>();
         logStreamConfig = new LogStreamConfig();
         contentSyncConfig = new ContentSyncConfig();
         feedStatusConfig = new FeedStatusConfig();

stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/forwarder/ForwardFileConfig.java

@@ -21,6 +21,12 @@ public class ForwardFileConfig extends AbstractConfig implements ForwardConfig,
     private final String name;
     private final String path;
 
+    public ForwardFileConfig() {
+        enabled = true;
+        name = null;
+        path = null;
+    }
+
     @SuppressWarnings("unused")
     @JsonCreator
     public ForwardFileConfig(@JsonProperty("enabled") final boolean enabled,

stroom-proxy/stroom-proxy-app/src/test/java/stroom/proxy/dist/GenerateProxyExpectedYaml.java

@@ -23,13 +23,14 @@ public class GenerateProxyExpectedYaml {
 
     private static final String PROXY_CONFIG = "proxyConfig";
 
-    private static final String HEADER = "" +
-            "# This file is generated based on all the default configuration values that are built into stroom.\n" +
-            "# It serves as an example of the structure of the full configuration tree.\n" +
-            "# If any configuration item is not explicitly set then these defaults will be used instead.\n" +
-            "# Some configuration items are expected to set,\n" +
-            "# e.g. appConfig.commonDbDetails.connection.jdbcDriverUrl,\n" +
-            "# but most can be left with their default values.";
+    private static final String HEADER = """
+            # This file is generated based on all the default configuration values that are built into stroom-proxy.
+            # It serves as an example of the structure of the full configuration tree.
+            # If any configuration item is not explicitly set then these defaults will be used instead.
+            # An exception to this is some list based properties like 'fileScanners', 'forwardFileDestinations',
+            # 'forwardHttpDestinations' and 'sqsConnectors' where the default is an empty list but a single
+            # item has been included in this file to show the defaults for each list item.
+            # Some configuration items are expected to set but most can be left with their default values.""";
 
     /**
      * Builds a fresh config object tree with all the hard coded default values
@@ -61,7 +62,6 @@ public static void main(String[] args) throws IOException {
 
         final String generatedYaml = TestProxyYamlUtil.getYamlFromJavaModel();
 
-
         List<String> outputLines;
         if (args.length > 0) {
             // called for a specific output location so add a header

stroom-proxy/stroom-proxy-app/src/test/java/stroom/proxy/dist/TestProxyYamlUtil.java

@@ -2,6 +2,10 @@
 
 import stroom.proxy.app.ProxyConfig;
 import stroom.proxy.app.ProxyYamlUtil;
+import stroom.proxy.app.SqsConnectorConfig;
+import stroom.proxy.app.forwarder.ForwardFileConfig;
+import stroom.proxy.app.forwarder.ForwardHttpPostConfig;
+import stroom.proxy.repo.FileScannerConfig;
 import stroom.util.NullSafe;
 import stroom.util.io.DiffUtil;
 
@@ -114,8 +118,19 @@ static Path getBasePath() {
 
     static String getYamlFromJavaModel() throws IOException {
         final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
-        final ProxyConfig appConfig = new ProxyConfig();
-        ProxyYamlUtil.writeConfig(appConfig, byteArrayOutputStream);
+        final ProxyConfig proxyConfig = new ProxyConfig();
+
+        // These list based props are empty by default, so that make it hard to see what the structure
+        // of the list items is when looking at the generated yaml. Thus, we seed each one with a vanilla item.
+        // This is not ideal as it means the generated yaml is not a true default, but hey ho?
+        // This is also a bit of a hack adding items to lists. The lists ought to be immutable but
+        // walking the whole tree to build it as you go is non-trivial.
+        proxyConfig.getFileScanners().add(new FileScannerConfig());
+        proxyConfig.getForwardHttpDestinations().add(new ForwardHttpPostConfig());
+        proxyConfig.getForwardFileDestinations().add(new ForwardFileConfig());
+        proxyConfig.getSqsConnectors().add(new SqsConnectorConfig());
+
+        ProxyYamlUtil.writeConfig(proxyConfig, byteArrayOutputStream);
         return byteArrayOutputStream.toString();
     }
 

stroom-proxy/stroom-proxy-app/src/test/resources/stroom/dist/proxy-expected.yaml

@@ -44,9 +44,22 @@ proxyConfig:
       expireAfterWrite: null
       maximumSize: 1000
     url: null
-  fileScanners: []
-  forwardFileDestinations: []
-  forwardHttpDestinations: []
+  fileScanners:
+  - path: null
+    scanFrequency: "PT1M"
+  forwardFileDestinations:
+  - enabled: true
+    name: null
+    path: null
+  forwardHttpDestinations:
+  - enabled: true
+    forwardChunkSize: "1M"
+    forwardDelay: "PT0S"
+    forwardTimeout: "PT30S"
+    forwardUrl: null
+    name: null
+    sslConfig: null
+    userAgent: null
   forwardRetry:
     failedForwardDir: "failures"
     maxRetryDelay: "PT1H"
@@ -87,7 +100,12 @@ proxyConfig:
   repository:
     repoDir: "repo"
     storingEnabled: true
-  sqsConnectors: []
+  sqsConnectors:
+  - awsProfileName: null
+    awsRegionName: null
+    pollFrequency: "PT10S"
+    queueName: null
+    queueUrl: null
   threads:
     examineSourceThreadCount: 3
     forwardRetryThreadCount: 2

stroom-util/src/main/java/stroom/util/config/PropertyUtil.java

@@ -570,8 +570,10 @@ public String toString() {
         }
     }
 
+
     // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+
     /**
      * Class to define a config property in the config object tree
      */

unreleased_changes/20230801_092358_225__3638.md

@@ -0,0 +1,24 @@
+* Issue **#3638** : Change the proxy defaults yaml file to include a single default items for list based properties. Also fix the HTTP forward config prop name in the distribution/docker config file.
+
+
+```sh
+# ********************************************************************************
+# Issue title: Update vanilla proxy config for forward destinations
+# Issue link:  https://github.com/gchq/stroom/issues/3638
+# ********************************************************************************
+
+# ONLY the top line will be included as a change entry in the CHANGELOG.
+# The entry should be in GitHub flavour markdown and should be written on a SINGLE
+# line with no hard breaks. You can have multiple change files for a single GitHub issue.
+# The  entry should be written in the imperative mood, i.e. 'Fix nasty bug' rather than
+# 'Fixed nasty bug'.
+#
+# Examples of acceptable entries are:
+#
+#
+# * Issue **123** : Fix bug with an associated GitHub issue in this repository
+#
+# * Issue **namespace/other-repo#456** : Fix bug with an associated GitHub issue in another repository
+#
+# * Fix bug with no associated GitHub issue.
+```