2296 upgrade to apache httpclient 5

karate-core/pom.xml

@@ -50,22 +50,10 @@
             </exclusions>            
         </dependency>              
         <dependency>
-            <groupId>org.apache.httpcomponents</groupId>
-            <artifactId>httpclient</artifactId>
-            <version>4.5.14</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>commons-logging</groupId>
-                    <artifactId>commons-logging</artifactId>
-                </exclusion>
-            </exclusions>
+            <groupId>org.apache.httpcomponents.client5</groupId>
+            <artifactId>httpclient5</artifactId>
+            <version>5.3</version>            
         </dependency>
-        <!-- can be removed once httpclient is upgraded -->
-        <dependency>
-            <groupId>commons-codec</groupId>
-            <artifactId>commons-codec</artifactId>
-            <version>1.16.0</version>
-        </dependency>        
         <dependency>
             <groupId>ch.qos.logback</groupId>
             <artifactId>logback-classic</artifactId>
@@ -135,6 +123,13 @@
             <version>${junit5.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <version>5.5.0</version>
+            <scope>test</scope>
+        </dependency>
+
     </dependencies>
 
     <build>       

karate-core/src/main/java/com/intuit/karate/http/ArmeriaHttpClient.java

@@ -134,4 +134,9 @@ public HttpResponse execute(com.linecorp.armeria.client.HttpClient delegate, Cli
         return delegate.execute(ctx, req);
     }
 
+    @Override
+    public void close() {
+        // No op. close() was introduced mainly for ApacheHttpClient, see https://github.com/karatelabs/karate/pull/2471
+    }
+
 }

karate-core/src/main/java/com/intuit/karate/http/CustomHttpRequestRetryHandler.java

@@ -2,9 +2,12 @@
 
 import java.io.IOException;
 
-import org.apache.http.NoHttpResponseException;
-import org.apache.http.client.HttpRequestRetryHandler;
-import org.apache.http.protocol.HttpContext;
+import org.apache.hc.client5.http.HttpRequestRetryStrategy;
+import org.apache.hc.core5.http.HttpRequest;
+import org.apache.hc.core5.http.HttpResponse;
+import org.apache.hc.core5.http.NoHttpResponseException;
+import org.apache.hc.core5.http.protocol.HttpContext;
+import org.apache.hc.core5.util.TimeValue;
 
 import com.intuit.karate.Logger;
 
@@ -13,7 +16,7 @@
  * This is usually the case when there is steal connection. The retry cause that
  * the connection is renewed and the second call will succeed.
  */
-public class CustomHttpRequestRetryHandler implements HttpRequestRetryHandler
+public class CustomHttpRequestRetryHandler implements HttpRequestRetryStrategy
 {
     private final Logger logger;
 
@@ -22,9 +25,7 @@ public CustomHttpRequestRetryHandler(Logger logger)
         this.logger = logger;
     }
 
-    @Override
-    public boolean retryRequest(IOException exception, int executionCount, HttpContext context)
-    {
+    private boolean shouldRetry(IOException exception, int executionCount) {
         if (exception instanceof NoHttpResponseException && executionCount < 1)
         {
             logger.error("Thrown an NoHttpResponseException retry...");
@@ -36,4 +37,19 @@ public boolean retryRequest(IOException exception, int executionCount, HttpConte
             return false;
         }
     }
+
+    @Override
+    public boolean retryRequest(HttpRequest request, IOException exception, int executionCount, HttpContext context) {
+        return shouldRetry(exception, executionCount);                
+    }
+
+    @Override
+    public boolean retryRequest(HttpResponse response, int execCount, HttpContext context) {
+        return false;
+     }
+
+    @Override
+    public TimeValue getRetryInterval(HttpResponse response, int execCount, HttpContext context) {
+        return TimeValue.ofSeconds(1); // NOt sure what the interval was in httpclient4 ... Sticking with the default value of the default http5 implementation.         
+    }
 }

karate-core/src/main/java/com/intuit/karate/http/HttpClient.java

@@ -37,4 +37,5 @@ public interface HttpClient {
 
     Response invoke(HttpRequest request);
 
+    void close();
 }

karate-core/src/main/java/com/intuit/karate/http/HttpRequestBuilder.java

@@ -51,7 +51,7 @@
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
-import org.apache.http.client.utils.URIBuilder;
+import org.apache.hc.core5.net.URIBuilder;
 import org.graalvm.polyglot.Value;
 import org.graalvm.polyglot.proxy.ProxyObject;
 import org.slf4j.Logger;

karate-core/src/main/java/org/apache/http/conn/ssl/LenientSslConnectionSocketFactory.java → karate-core/src/main/java/org/apache/hc/client5/http/ssl/LenientSslConnectionSocketFactory.java

@@ -21,13 +21,14 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-package org.apache.http.conn.ssl;
+package org.apache.hc.client5.http.ssl;
 
 import java.io.IOException;
 import java.net.Socket;
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.SSLContext;
-import org.apache.http.protocol.HttpContext;
+
+import org.apache.hc.core5.http.protocol.HttpContext;
 
 /**
  * in a separate package just for log level config consistency

karate-core/src/test/java/com/intuit/karate/core/DummyClient.java

@@ -24,6 +24,11 @@ public Config getConfig() {
     @Override
     public Response invoke(HttpRequest request) {
         throw new UnsupportedOperationException("not implemented");
+    }
+
+    @Override
+    public void close() {
+        // No op. close() was introduced mainly for ApacheHttpClient, see https://github.com/karatelabs/karate/pull/2471
     }   
 
 }

karate-core/src/test/java/com/intuit/karate/core/FeatureRuntimeTest.java

@@ -401,9 +401,10 @@ void testTypeConv() {
         run("type-conv.feature");
     }
 
-    @Test
-    void testConfigureNtlmAuthentication() {
-        run("ntlm-authentication.feature");
-    }
+    // NTLM not supported in apache client 5.3
+    // @Test
+    // void testConfigureNtlmAuthentication() {
+    //     run("ntlm-authentication.feature");
+    // }
 
 }

karate-core/src/test/java/com/intuit/karate/core/MockClient.java

@@ -33,4 +33,9 @@ public Response invoke(HttpRequest request) {
         return handler.handle(request.toRequest());
     }
 
+    @Override
+    public void close() {
+        // No op. close() was introduced mainly for ApacheHttpClient, see https://github.com/karatelabs/karate/pull/2471
+    }
+
 }

karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerSslTest.java

@@ -10,17 +10,20 @@
 import java.nio.charset.StandardCharsets;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.TrustManager;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
+
+import org.apache.hc.client5.http.classic.methods.HttpGet;
+import org.apache.hc.client5.http.classic.methods.HttpPost;
+import org.apache.hc.client5.http.classic.methods.HttpUriRequest;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.impl.classic.HttpClients;
+import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
+import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
+import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactoryBuilder;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.HttpResponse;
+import org.apache.hc.core5.http.io.entity.StringEntity;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
@@ -81,15 +84,18 @@ int http(HttpUriRequest request) throws Exception {
         SSLContext sc = SSLContext.getInstance("SSL");
         sc.init(null, new TrustManager[]{LenientTrustManager.INSTANCE}, null);
         CloseableHttpClient client = HttpClients.custom()
-                .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE)
-                .setSSLContext(sc)
+                .setConnectionManager(PoolingHttpClientConnectionManagerBuilder.create()
+                        .setSSLSocketFactory(SSLConnectionSocketFactoryBuilder.create()
+                                .setSslContext(sc)
+                                .setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
+                                .build())
+                        .build())
                 .setProxy(new HttpHost("localhost", proxy.getPort()))
                 .build();
         HttpResponse response = client.execute(request);
-        InputStream is = response.getEntity().getContent();
-        String responseString = FileUtils.toString(is);
+        String responseString = response.getReasonPhrase();
         logger.debug("response: {}", responseString);
-        return response.getStatusLine().getStatusCode();
+        return response.getCode();
     }
 
 }

karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerTest.java

@@ -7,16 +7,17 @@
 import com.intuit.karate.core.MockServer;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
+
+import org.apache.hc.client5.http.classic.methods.HttpGet;
+import org.apache.hc.client5.http.classic.methods.HttpPost;
+import org.apache.hc.client5.http.classic.methods.HttpUriRequest;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.impl.classic.HttpClients;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.HttpEntity;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.HttpResponse;
+import org.apache.hc.core5.http.io.entity.StringEntity;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
@@ -81,10 +82,9 @@ static int http(HttpUriRequest request) throws Exception {
                 .setProxy(new HttpHost("localhost", proxy.getPort()))
                 .build();
         HttpResponse response = client.execute(request);
-        InputStream is = response.getEntity().getContent();
-        String responseString = FileUtils.toString(is);
+        String responseString = response.getReasonPhrase();
         logger.debug("response: {}", responseString);
-        return response.getStatusLine().getStatusCode();
+        return response.getCode();
     }
 
 }

karate-core/src/test/java/com/intuit/karate/http/ApacheHttpServerTest.java

@@ -0,0 +1,91 @@
+package com.intuit.karate.http;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.mockito.Mockito.mock;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hc.client5.http.HttpRoute;
+import org.apache.hc.core5.http.HttpException;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.protocol.HttpContext;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import com.intuit.karate.core.Config;
+import com.intuit.karate.core.ScenarioEngine;
+import com.intuit.karate.core.Variable;
+
+class ApacheHttpServerTest {
+
+    private ScenarioEngine engine;
+    private Config config;
+    private HttpHost host;
+    private HttpContext context;
+
+    private ApacheHttpClient client;
+
+    @BeforeEach
+    void configure() {
+        engine = mock(ScenarioEngine.class);
+        config = new Config();
+        Mockito.when(engine.getConfig()).thenReturn(config);
+        host = new HttpHost("foo.com");
+        context = mock(HttpContext.class);
+
+        client = new ApacheHttpClient(engine);        
+    }
+
+    @Test
+    void noProxy() {
+        HttpRoute route = determineRoute(host);
+        Assertions.assertNull(route.getProxyHost());
+        assertNull(route.getLocalAddress());
+    }
+
+    @Test
+    void proxy() {
+        config.configure("proxy", new Variable("http://proxy:80"));
+        HttpRoute route =  determineRoute(host);
+        assertEquals("http://proxy:80", route.getProxyHost().toURI());
+    }
+
+    @Test
+    void nonProxyHosts() {
+        Map<String, Object> proxyConfiguration = new HashMap<>();
+        proxyConfiguration.put("uri", "http://proxy:80");
+        proxyConfiguration.put("nonProxyHosts", Collections.singletonList("foo.com"));
+        config.configure("proxy", new Variable(proxyConfiguration));
+
+        HttpRoute nonProxiedRoute = determineRoute(host);
+        assertNull(nonProxiedRoute.getProxyHost());
+
+        HttpRoute proxiedRoute = determineRoute(new HttpHost("bar.com"));
+        assertEquals("http://proxy:80", proxiedRoute.getProxyHost().toURI());
+    }
+
+    // From a Karate perspective, localAddress is primarily designed to be used with Gatling and is not related to proxy.
+    // However, in apache client 5, it is handled by the RoutePlanner. 
+    @Test
+    void localAddress() {
+        config.configure("localAddress", new Variable("localhost"));
+
+        HttpRoute route = determineRoute(host);
+
+        assertNull(route.getProxyHost());
+        assertEquals("localhost", route.getLocalAddress().getHostName());
+    }
+
+    private HttpRoute determineRoute(HttpHost host) {
+        try {
+            return client.buildRoutePlanner(config).determineRoute(host, context);
+        } catch (HttpException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

karate-demo/pom.xml

@@ -20,6 +20,12 @@
     <!-- for a skeleton project, use the archetype: https://github.com/karatelabs/karate#quickstart !-->
     <dependencyManagement>
         <dependencies>
+            <dependency>
+                <groupId>org.apache.httpcomponents.client5</groupId>
+                <artifactId>httpclient5</artifactId>
+				<!-- Consistent with version from karate-core -->
+                <version>5.3</version>            
+            </dependency>  		
             <dependency>
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-dependencies</artifactId>