refactor: migrate Druid HTTP client from Netty 3 to 4

embedded-tests/pom.xml

@@ -120,12 +120,12 @@
     </dependency>
     <dependency>
       <groupId>io.netty</groupId>
-      <artifactId>netty</artifactId>
+      <artifactId>netty-common</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>io.netty</groupId>
-      <artifactId>netty-common</artifactId>
+      <artifactId>netty-codec-http</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>
@@ -899,6 +899,14 @@
         <artifactId>maven-surefire-plugin</artifactId>
         <configuration>
           <excludedGroups>docker-test,perf</excludedGroups>
+          <!-- Enable Netty 4 PARANOID leak detection in the embedded tests because this is where
+               real cross-node HTTP traffic exercises the migrated HttpClient code paths (chunked
+               responses, long-poll syncs, TLS, ByteBuf-bearing handlers). The Stage 1 CI workers
+               that happen to run embedded-tests classes pay a small overhead per class; in exchange
+               we catch ref-count regressions immediately rather than as a flaky off-heap leak. -->
+          <systemPropertyVariables>
+            <io.netty.leakDetection.level>PARANOID</io.netty.leakDetection.level>
+          </systemPropertyVariables>
         </configuration>
       </plugin>
       <plugin>
@@ -945,6 +953,7 @@
                 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
                 -Daws.region=us-east-1
                 -Ddruid.test.stupidPool.poison=true
+                -Dio.netty.leakDetection.level=PARANOID
               </argLine>
             </configuration>
             <executions>

embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AbstractAuthConfigurationTest.java

@@ -25,6 +25,8 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpResponseStatus;
 import org.apache.calcite.avatica.AvaticaSqlException;
 import org.apache.druid.common.utils.IdUtils;
 import org.apache.druid.indexing.common.task.Task;
@@ -55,8 +57,6 @@
 import org.apache.druid.testing.embedded.indexing.MoreResources;
 import org.apache.druid.testing.embedded.junit5.EmbeddedClusterTestBase;
 import org.apache.druid.timeline.DataSegment;
-import org.jboss.netty.handler.codec.http.HttpMethod;
-import org.jboss.netty.handler.codec.http.HttpResponseStatus;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;

embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/BasicAuthConfigurationTest.java

@@ -19,6 +19,8 @@
 
 package org.apache.druid.testing.embedded.auth;
 
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpResponseStatus;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.http.client.CredentialedHttpClient;
 import org.apache.druid.java.util.http.client.HttpClient;
@@ -27,8 +29,6 @@
 import org.apache.druid.server.security.Access;
 import org.apache.druid.server.security.ResourceAction;
 import org.apache.druid.testing.embedded.EmbeddedResource;
-import org.jboss.netty.handler.codec.http.HttpMethod;
-import org.jboss.netty.handler.codec.http.HttpResponseStatus;
 import org.junit.jupiter.api.Test;
 
 import java.util.List;

embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/BasicAuthIndexingTest.java

@@ -20,11 +20,11 @@
 package org.apache.druid.testing.embedded.auth;
 
 import com.fasterxml.jackson.core.type.TypeReference;
+import io.netty.handler.codec.http.HttpMethod;
 import org.apache.druid.rpc.RequestBuilder;
 import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
 import org.apache.druid.testing.embedded.EmbeddedRouter;
 import org.apache.druid.testing.embedded.indexing.IndexTaskTest;
-import org.jboss.netty.handler.codec.http.HttpMethod;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 

embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/BasicAuthLdapConfigurationTest.java

@@ -20,6 +20,8 @@
 package org.apache.druid.testing.embedded.auth;
 
 import com.google.common.collect.ImmutableMap;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpResponseStatus;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.http.client.CredentialedHttpClient;
 import org.apache.druid.java.util.http.client.HttpClient;
@@ -28,8 +30,6 @@
 import org.apache.druid.server.security.Access;
 import org.apache.druid.server.security.ResourceAction;
 import org.apache.druid.testing.embedded.EmbeddedResource;
-import org.jboss.netty.handler.codec.http.HttpMethod;
-import org.jboss.netty.handler.codec.http.HttpResponseStatus;
 import org.junit.jupiter.api.Test;
 
 import java.util.List;

embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/EmbeddedBasicAuthResource.java

@@ -19,6 +19,8 @@
 
 package org.apache.druid.testing.embedded.auth;
 
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpResponseStatus;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.http.client.HttpClient;
 import org.apache.druid.security.basic.BasicSecurityDruidModule;
@@ -28,8 +30,6 @@
 import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
 import org.apache.druid.testing.embedded.EmbeddedDruidServer;
 import org.apache.druid.testing.embedded.EmbeddedResource;
-import org.jboss.netty.handler.codec.http.HttpMethod;
-import org.jboss.netty.handler.codec.http.HttpResponseStatus;
 
 import java.util.List;
 

embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/HttpUtil.java

@@ -20,15 +20,15 @@
 package org.apache.druid.testing.embedded.auth;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpResponseStatus;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.http.client.HttpClient;
 import org.apache.druid.java.util.http.client.Request;
 import org.apache.druid.java.util.http.client.response.StatusResponseHandler;
 import org.apache.druid.java.util.http.client.response.StatusResponseHolder;
 import org.apache.druid.segment.TestHelper;
-import org.jboss.netty.handler.codec.http.HttpMethod;
-import org.jboss.netty.handler.codec.http.HttpResponseStatus;
 
 import javax.annotation.Nullable;
 import javax.ws.rs.core.MediaType;

embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/SecurityClient.java

@@ -20,11 +20,11 @@
 package org.apache.druid.testing.embedded.auth;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import io.netty.handler.codec.http.HttpMethod;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.rpc.RequestBuilder;
 import org.apache.druid.server.security.ResourceAction;
 import org.apache.druid.testing.embedded.EmbeddedServiceClient;
-import org.jboss.netty.handler.codec.http.HttpMethod;
 
 import java.util.List;
 import java.util.Map;

embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/TLSTest.java

@@ -20,6 +20,8 @@
 package org.apache.druid.testing.embedded.auth;
 
 import com.google.common.base.Throwables;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpResponseStatus;
 import org.apache.druid.guice.http.DruidHttpClientConfig;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.RetryUtils;
@@ -44,8 +46,6 @@
 import org.apache.druid.testing.embedded.EmbeddedOverlord;
 import org.apache.druid.testing.embedded.EmbeddedRouter;
 import org.apache.druid.testing.embedded.junit5.EmbeddedClusterTestBase;
-import org.jboss.netty.handler.codec.http.HttpMethod;
-import org.jboss.netty.handler.codec.http.HttpResponseStatus;
 import org.joda.time.Duration;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;

embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/TestCatalogClient.java

@@ -22,6 +22,7 @@
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.core.type.TypeReference;
+import io.netty.handler.codec.http.HttpMethod;
 import org.apache.druid.catalog.http.CatalogResource;
 import org.apache.druid.catalog.http.TableEditRequest;
 import org.apache.druid.catalog.model.TableId;
@@ -31,7 +32,6 @@
 import org.apache.druid.rpc.RequestBuilder;
 import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
 import org.apache.druid.testing.embedded.EmbeddedServiceClient;
-import org.jboss.netty.handler.codec.http.HttpMethod;
 
 import java.util.List;
 

embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionResourceTestClient.java

@@ -20,6 +20,7 @@
 package org.apache.druid.testing.embedded.compact;
 
 import com.fasterxml.jackson.core.type.TypeReference;
+import io.netty.handler.codec.http.HttpMethod;
 import org.apache.druid.indexing.overlord.http.CompactionConfigsResponse;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.logger.Logger;
@@ -33,7 +34,6 @@
 import org.apache.druid.server.http.ServletResourceUtils;
 import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
 import org.apache.druid.testing.embedded.EmbeddedServiceClient;
-import org.jboss.netty.handler.codec.http.HttpMethod;
 import org.junit.jupiter.api.Assertions;
 
 import java.util.List;

embedded-tests/src/test/java/org/apache/druid/testing/embedded/docker/IngestionBackwardCompatibilityDockerTest.java

@@ -20,6 +20,7 @@
 package org.apache.druid.testing.embedded.docker;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import io.netty.handler.codec.http.HttpMethod;
 import org.apache.druid.client.coordinator.Coordinator;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.http.client.response.BytesFullResponseHandler;
@@ -33,7 +34,6 @@
 import org.apache.druid.testing.embedded.EmbeddedHistorical;
 import org.apache.druid.testing.embedded.EmbeddedRouter;
 import org.apache.druid.testing.embedded.indexing.IngestionSmokeTest;
-import org.jboss.netty.handler.codec.http.HttpMethod;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 

embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/AbstractITBatchIndexTest.java

@@ -22,6 +22,7 @@
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.ImmutableList;
+import io.netty.handler.codec.http.HttpMethod;
 import org.apache.commons.io.IOUtils;
 import org.apache.druid.indexer.TaskStatusPlus;
 import org.apache.druid.indexer.report.IngestionStatsAndErrors;
@@ -48,7 +49,6 @@
 import org.apache.druid.timeline.DataSegment;
 import org.apache.druid.timeline.SegmentTimeline;
 import org.apache.druid.timeline.TimelineObjectHolder;
-import org.jboss.netty.handler.codec.http.HttpMethod;
 import org.junit.Assert;
 
 import java.io.IOException;

embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/StreamIndexFaultToleranceTest.java

@@ -19,13 +19,13 @@
 
 package org.apache.druid.testing.embedded.indexing;
 
+import io.netty.handler.codec.http.HttpMethod;
 import org.apache.druid.indexer.TaskStatusPlus;
 import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.query.DruidMetrics;
 import org.apache.druid.rpc.RequestBuilder;
 import org.apache.druid.testing.embedded.StreamIngestResource;
-import org.jboss.netty.handler.codec.http.HttpMethod;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;

embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/autoscaler/CostBasedAutoScalerIntegrationTest.java

@@ -20,6 +20,7 @@
 package org.apache.druid.testing.embedded.indexing.autoscaler;
 
 import com.fasterxml.jackson.core.type.TypeReference;
+import io.netty.handler.codec.http.HttpMethod;
 import org.apache.druid.indexer.TaskState;
 import org.apache.druid.indexer.TaskStatusPlus;
 import org.apache.druid.indexing.kafka.simulate.KafkaResource;
@@ -35,7 +36,6 @@
 import org.apache.druid.testing.embedded.StreamIngestResource;
 import org.apache.druid.testing.embedded.indexing.StreamIndexTestBase;
 import org.hamcrest.Matchers;
-import org.jboss.netty.handler.codec.http.HttpMethod;
 import org.joda.time.Duration;
 import org.joda.time.Period;
 import org.junit.jupiter.api.Assertions;

embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedDartReportApiTest.java

@@ -23,6 +23,8 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.util.concurrent.ListenableFuture;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpResponseStatus;
 import org.apache.druid.common.utils.IdUtils;
 import org.apache.druid.data.input.impl.CsvInputFormat;
 import org.apache.druid.error.DruidException;
@@ -64,8 +66,6 @@
 import org.apache.druid.testing.embedded.junit5.EmbeddedClusterTestBase;
 import org.hamcrest.CoreMatchers;
 import org.hamcrest.MatcherAssert;
-import org.jboss.netty.handler.codec.http.HttpMethod;
-import org.jboss.netty.handler.codec.http.HttpResponseStatus;
 import org.junit.internal.matchers.ThrowableMessageMatcher;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;

embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedMSQApis.java

@@ -22,6 +22,7 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.util.concurrent.ListenableFuture;
+import io.netty.handler.codec.http.HttpResponseStatus;
 import org.apache.druid.error.DruidException;
 import org.apache.druid.indexer.TaskState;
 import org.apache.druid.indexer.report.TaskReport;
@@ -42,7 +43,6 @@
 import org.apache.druid.testing.embedded.EmbeddedClusterApis;
 import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
 import org.apache.druid.testing.embedded.EmbeddedOverlord;
-import org.jboss.netty.handler.codec.http.HttpResponseStatus;
 
 import javax.annotation.Nullable;
 import java.util.ArrayList;

embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/MsqTaskSubmitTest.java

@@ -20,12 +20,12 @@
 package org.apache.druid.testing.embedded.query;
 
 import com.google.common.util.concurrent.ListenableFuture;
+import io.netty.handler.codec.http.HttpResponseStatus;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
 import org.apache.druid.java.util.http.client.response.StatusResponseHolder;
 import org.apache.druid.query.http.ClientSqlQuery;
 import org.apache.druid.testing.embedded.EmbeddedClusterApis;
-import org.jboss.netty.handler.codec.http.HttpResponseStatus;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
@@ -75,8 +75,8 @@ public void test_msqTaskSubmissionThroughRouter_reportedAsSuccess() throws Excep
     StatusResponseHolder response = future.get();
 
     Assertions.assertEquals(
-        HttpResponseStatus.ACCEPTED.getCode(),
-        response.getStatus().getCode(),
+        HttpResponseStatus.ACCEPTED.code(),
+        response.getStatus().code(),
         "MSQ task submission should return 202 Accepted"
     );
 

embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryTestBase.java

@@ -22,6 +22,7 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.util.concurrent.ListenableFuture;
+import io.netty.handler.codec.http.HttpMethod;
 import org.apache.druid.common.utils.IdUtils;
 import org.apache.druid.guice.SleepModule;
 import org.apache.druid.indexing.common.task.IndexTask;
@@ -41,7 +42,6 @@
 import org.apache.druid.testing.embedded.EmbeddedRouter;
 import org.apache.druid.testing.embedded.indexing.MoreResources;
 import org.apache.druid.testing.embedded.junit5.EmbeddedClusterTestBase;
-import org.jboss.netty.handler.codec.http.HttpMethod;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 
@@ -204,7 +204,7 @@ protected void executeQueryWithContentType(
     Assertions.assertNotNull(response);
 
     onResponse.accept(
-        response.getStatus().getCode(),
+        response.getStatus().code(),
         response.getContent().trim()
     );
   }

embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/SqlQueryCancelTest.java

@@ -21,11 +21,11 @@
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.util.concurrent.ListenableFuture;
+import io.netty.handler.codec.http.HttpResponseStatus;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.http.client.response.StatusResponseHolder;
 import org.apache.druid.query.BaseQuery;
 import org.apache.druid.query.http.ClientSqlQuery;
-import org.jboss.netty.handler.codec.http.HttpResponseStatus;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
@@ -66,8 +66,8 @@ public void testCancelValidQuery() throws Exception
 
     StatusResponseHolder queryResponce = f.get();
 
-    Assertions.assertEquals(HttpResponseStatus.ACCEPTED.getCode(), queryCancellationResponse.getStatus().getCode());
-    Assertions.assertEquals(HttpResponseStatus.INTERNAL_SERVER_ERROR.getCode(), queryResponce.getStatus().getCode());
+    Assertions.assertEquals(HttpResponseStatus.ACCEPTED.code(), queryCancellationResponse.getStatus().code());
+    Assertions.assertEquals(HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), queryResponce.getStatus().code());
   }
 
   @Test
@@ -91,7 +91,7 @@ public void test_cancelInvalidQuery_returnsNotFound() throws Exception
 
     StatusResponseHolder queryResponse = f.get();
 
-    Assertions.assertEquals(HttpResponseStatus.NOT_FOUND.getCode(), queryCancellationResponse.getStatus().getCode());
-    Assertions.assertEquals(HttpResponseStatus.OK.getCode(), queryResponse.getStatus().getCode());
+    Assertions.assertEquals(HttpResponseStatus.NOT_FOUND.code(), queryCancellationResponse.getStatus().code());
+    Assertions.assertEquals(HttpResponseStatus.OK.code(), queryResponse.getStatus().code());
   }
 }