Remove HttpSource.proxy.http.enabled and rename HttpSource.proxy.http…

….server to host (#502)

CHANGES.md

@@ -1,23 +1,36 @@
 # Change Log
 
-# 6.0
+## 6.0
+
+### Endpoints
 
 * Image and information responses include a `Last-Modified` header when
   possible.
 * The health endpoint is enabled via `endpoint.health.enabled` rather than
   `endpoint.api.enabled`.
+* Added an HTTP API method to purge all infos from the derivative cache.
+* Added a configuration option to automatically purge source-cached images
+  whose format cannot be inferred.
+
+### Sources
+
+* HttpSource supports a client HTTP proxy. (Thanks to @mightymax and
+  @mlindeman)
 * HttpSource can be configured to send a ranged GET request instead of a HEAD
   request, enabling it to work with pre-signed URLs that do not allow HEAD
   requests.
 * S3Source supports multiple endpoints when using ScriptLookupStrategy.
-* Added a configuration option to automatically purge source-cached images
-  whose format cannot be inferred.
-* Added an HTTP API method to purge all infos from the derivative cache.
+
+### Caches
+
+* S3Cache uses multipart uploads, which reduces memory usage when caching
+  derivatives larger than 5 MB.
+
+### Delegate Script
+
 * The delegate script pathname can be set using the
   `-Dcantaloupe.delegate_script` VM argument, which takes precedence over the
   `delegate_script.pathname` configuration key.
-* S3Cache uses multipart uploads, which reduces memory usage when caching
-  derivatives larger than 5 MB.
 * The delegate script's `metadata` context key contains a new field,
   `xmp_elements`, that provides a high-level key-value view of the XMP data.
 

UPGRADING.md

@@ -7,6 +7,8 @@ current version.
 
 1. Add the following keys from the sample configuration:
     * `endpoint.health.enabled`
+    * `HttpSource.proxy.http.host`
+    * `HttpSource.proxy.http.port`
     * `HttpSource.BasicLookupStrategy.send_head_requests`
     * `processor.purge_incompatible_from_source_cache`
 2. Add the following methods from the sample delegate script:

cantaloupe.properties.sample

@@ -175,6 +175,10 @@ HttpSource.allow_insecure = false
 # Request timeout in seconds.
 HttpSource.request_timeout =
 
+# !! Client HTTP proxy.
+HttpSource.proxy.http.host =
+HttpSource.proxy.http.port =
+
 # Tells HttpSource how to look up resources. Allowed values are
 # `BasicLookupStrategy` and `ScriptLookupStrategy`. ScriptLookupStrategy
 # uses a delegate method for dynamic lookups; see the user manual.
@@ -212,11 +216,6 @@ HttpSource.chunking.cache.enabled = true
 # Max per-request chunk cache size.
 HttpSource.chunking.cache.max_size = 5M
 
-# Enable HTTP Proxy for HttpSource
-HttpSource.proxy.http.enabled = false
-HttpSource.proxy.http.server = proxy.example.com
-HttpSource.proxy.http.port = 8080
-
 #----------------------------------------
 # S3Source
 #----------------------------------------

src/main/java/edu/illinois/library/cantaloupe/config/Key.java

@@ -108,6 +108,8 @@ public enum Key {
     HTTPSOURCE_CHUNK_SIZE("HttpSource.chunking.chunk_size"),
     HTTPSOURCE_CHUNK_CACHE_ENABLED("HttpSource.chunking.cache.enabled"),
     HTTPSOURCE_CHUNK_CACHE_MAX_SIZE("HttpSource.chunking.cache.max_size"),
+    HTTPSOURCE_HTTP_PROXY_HOST("HttpSource.proxy.http.host"),
+    HTTPSOURCE_HTTP_PROXY_PORT("HttpSource.proxy.http.port"),
     HTTPSOURCE_LOOKUP_STRATEGY("HttpSource.lookup_strategy"),
     HTTPSOURCE_REQUEST_TIMEOUT("HttpSource.request_timeout"),
     HTTPSOURCE_SEND_HEAD_REQUESTS("HttpSource.BasicLookupStrategy.send_head_requests"),
@@ -120,9 +122,6 @@ public enum Key {
     HTTPS_KEY_STORE_PATH("https.key_store_path"),
     HTTPS_KEY_STORE_TYPE("https.key_store_type"),
     HTTPS_PORT("https.port"),
-    HTTPSOURCE_HTTP_PROXY_ENABLED("HttpSource.proxy.http.enabled"),
-    HTTPSOURCE_HTTP_PROXY_SERVER("HttpSource.proxy.http.server"),
-    HTTPSOURCE_HTTP_PROXY_PORT("HttpSource.proxy.http.port"),
     IIIF_1_ENDPOINT_ENABLED("endpoint.iiif.1.enabled"),
     IIIF_2_ENDPOINT_ENABLED("endpoint.iiif.2.enabled"),
     IIIF_3_ENDPOINT_ENABLED("endpoint.iiif.3.enabled"),

src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java

@@ -19,10 +19,10 @@
 import javax.net.ssl.X509TrustManager;
 import javax.script.ScriptException;
 import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Proxy;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.net.Proxy;
-import java.net.InetSocketAddress;
 import java.nio.file.AccessDeniedException;
 import java.nio.file.NoSuchFileException;
 import java.security.KeyManagementException;
@@ -410,27 +410,24 @@ static synchronized OkHttpClient getHTTPClient() {
                     .connectTimeout(getRequestTimeout().getSeconds(), TimeUnit.SECONDS)
                     .readTimeout(getRequestTimeout().getSeconds(), TimeUnit.SECONDS)
                     .writeTimeout(getRequestTimeout().getSeconds(), TimeUnit.SECONDS);
-
             final Configuration config = Configuration.getInstance();
-
-            final boolean httpProxyEnabled = config.getBoolean(
-                Key.HTTPSOURCE_HTTP_PROXY_ENABLED, false);
-            if (httpProxyEnabled) {
-              final String httpProxyServer = config.getString(Key.HTTPSOURCE_HTTP_PROXY_SERVER, "");
-              if (httpProxyServer == "") {
-                throw new RuntimeException("proxy server setting HttpSource.proxy.http.server should not be empty");
-              }
-              final int httpProxyPort = config.getInt(Key.HTTPSOURCE_HTTP_PROXY_PORT, 8080);
-
-              LOGGER.trace("Using HTTP Proxy at server {} on port {}", httpProxyServer, httpProxyPort);
-              Proxy httpProxy = new Proxy(Proxy.Type.HTTP,new InetSocketAddress(httpProxyServer, httpProxyPort));
-              builder.proxy(httpProxy);
-            }
 
-            final boolean allowInsecure = config.getBoolean(
-                    Key.HTTPSOURCE_ALLOW_INSECURE, false);
+            final String proxyHost =
+                    config.getString(Key.HTTPSOURCE_HTTP_PROXY_HOST, "");
+            if (!proxyHost.isBlank()) {
+                final int proxyPort =
+                        config.getInt(Key.HTTPSOURCE_HTTP_PROXY_PORT);
+                if (proxyPort == 0) {
+                    throw new RuntimeException("Proxy port setting " +
+                            Key.HTTPSOURCE_HTTP_PROXY_PORT + " must be set");
+                }
+                LOGGER.debug("Using HTTP proxy: {}:{}", proxyHost, proxyPort);
+                Proxy httpProxy = new Proxy(Proxy.Type.HTTP,
+                        new InetSocketAddress(proxyHost, proxyPort));
+                builder.proxy(httpProxy);
+            }
 
-            if (allowInsecure) {
+            if (config.getBoolean(Key.HTTPSOURCE_ALLOW_INSECURE, false)) {
                 try {
                     X509TrustManager[] tm = new X509TrustManager[]{
                             new X509TrustManager() {
@@ -470,17 +467,6 @@ private static Duration getRequestTimeout() {
         return Duration.ofSeconds(timeout);
     }
 
-    static String getUserAgent() {
-        return String.format("%s/%s (%s/%s; java/%s; %s/%s)",
-                HttpSource.class.getSimpleName(),
-                Application.getVersion(),
-                Application.getName(),
-                Application.getVersion(),
-                System.getProperty("java.version"),
-                System.getProperty("os.name"),
-                System.getProperty("os.version"));
-    }
-
     /**
      * @see #request(HTTPRequestInfo, String, Map)
      */
@@ -505,7 +491,7 @@ static Response request(HTTPRequestInfo requestInfo,
         Request.Builder builder = new Request.Builder()
                 .method(method, null)
                 .url(requestInfo.getURI())
-                .addHeader("User-Agent", getUserAgent());
+                .addHeader("User-Agent", USER_AGENT);
         // Add credentials.
         if (requestInfo.getUsername() != null &&
                 requestInfo.getSecret() != null) {

src/main/resources/admin.vm

@@ -1721,6 +1721,34 @@
                                                        data-requires-restart="false">
                                             </td>
                                         </tr>
+                                        <tr>
+                                            <td>
+                                                Proxy Host
+                                                <a tabindex="0" class="btn btn-sm cl-help"
+                                                   role="button" data-toggle="popover"
+                                                   data-trigger="focus"
+                                                   data-content="Host of a client proxy server.">?</a>
+                                            </td>
+                                            <td>
+                                                <input class="form-control" type="text"
+                                                       name="HttpSource.proxy.http.host"
+                                                       data-requires-restart="true">
+                                            </td>
+                                        </tr>
+                                        <tr>
+                                            <td>
+                                                Proxy Port
+                                                <a tabindex="0" class="btn btn-sm cl-help"
+                                                   role="button" data-toggle="popover"
+                                                   data-trigger="focus"
+                                                   data-content="Port of a client proxy server.">?</a>
+                                            </td>
+                                            <td>
+                                                <input class="form-control" type="number" min="0"
+                                                       name="HttpSource.proxy.http.port"
+                                                       data-requires-restart="true">
+                                            </td>
+                                        </tr>
                                         <tr>
                                             <td>Lookup Strategy
                                                 <a tabindex="0" class="btn btn-sm cl-help"

src/test/java/edu/illinois/library/cantaloupe/resource/admin/AdminResourceUITest.java

@@ -371,6 +371,8 @@ void testSourceSection() throws Exception {
         inputNamed(Key.HTTPSOURCE_CHUNK_SIZE).sendKeys("412");
         inputNamed(Key.HTTPSOURCE_CHUNK_CACHE_ENABLED).click();
         inputNamed(Key.HTTPSOURCE_CHUNK_CACHE_MAX_SIZE).sendKeys("333");
+        inputNamed(Key.HTTPSOURCE_HTTP_PROXY_HOST).sendKeys("example.org");
+        inputNamed(Key.HTTPSOURCE_HTTP_PROXY_PORT).sendKeys("12345");
         inputNamed(Key.HTTPSOURCE_REQUEST_TIMEOUT).sendKeys("13");
         selectNamed(Key.HTTPSOURCE_LOOKUP_STRATEGY).
                 selectByValue("BasicLookupStrategy");
@@ -456,6 +458,10 @@ void testSourceSection() throws Exception {
                 config.getBoolean(Key.HTTPSOURCE_CHUNK_CACHE_ENABLED));
         assertEquals("333",
                 config.getString(Key.HTTPSOURCE_CHUNK_CACHE_MAX_SIZE));
+        assertEquals("example.org",
+                config.getString(Key.HTTPSOURCE_HTTP_PROXY_HOST));
+        assertEquals("12345",
+                config.getString(Key.HTTPSOURCE_HTTP_PROXY_PORT));
         assertEquals("13",
                 config.getString(Key.HTTPSOURCE_REQUEST_TIMEOUT));
         assertEquals("BasicLookupStrategy",

src/test/java/edu/illinois/library/cantaloupe/source/HTTPStreamFactoryTest.java

@@ -7,10 +7,12 @@
 import edu.illinois.library.cantaloupe.source.stream.HTTPImageInputStream;
 import edu.illinois.library.cantaloupe.test.BaseTest;
 import edu.illinois.library.cantaloupe.test.WebServer;
+import edu.illinois.library.cantaloupe.util.SocketUtils;
 import org.eclipse.jetty.server.Request;
 import org.eclipse.jetty.server.handler.DefaultHandler;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 import jakarta.servlet.http.HttpServletRequest;
@@ -71,6 +73,28 @@ void isSeekingDirect() {
         assertTrue(instance.isSeekingDirect());
     }
 
+    @Disabled
+    @Test
+    void newInputStreamWithProxy() throws Exception {
+        final int proxyPort = SocketUtils.getOpenPort();
+
+        // Set up the proxy
+        // TODO: write this
+
+        // Set up HttpSource
+        final var config = Configuration.getInstance();
+        config.setProperty(Key.HTTPSOURCE_HTTP_PROXY_HOST, "127.0.0.1");
+        config.setProperty(Key.HTTPSOURCE_HTTP_PROXY_PORT, proxyPort);
+
+        int length = 0;
+        try (InputStream is = newInstance(true).newInputStream()) {
+            while (is.read() != -1) {
+                length++;
+            }
+        }
+        assertTrue(length > 1000);
+    }
+
     @Test
     void newInputStreamSendsCustomHeaders() throws Exception {
         server.setHandler(new DefaultHandler() {
@@ -138,6 +162,28 @@ void newSeekableStreamWhenChunkingIsDisabled() throws Exception {
         }
     }
 
+    @Disabled
+    @Test
+    void newSeekableStreamWithProxy() throws Exception {
+        final int proxyPort = SocketUtils.getOpenPort();
+
+        // Set up the proxy
+        // TODO: write this
+
+        // Set up HttpSource
+        final var config = Configuration.getInstance();
+        config.setProperty(Key.HTTPSOURCE_HTTP_PROXY_HOST, "127.0.0.1");
+        config.setProperty(Key.HTTPSOURCE_HTTP_PROXY_PORT, proxyPort);
+
+        int length = 0;
+        try (ImageInputStream is = newInstance(true).newSeekableStream()) {
+            while (is.read() != -1) {
+                length++;
+            }
+        }
+        assertTrue(length > 1000);
+    }
+
     @Test
     void newSeekableStreamSendsCustomHeaders() throws Exception {
         server.setHandler(new DefaultHandler() {