Support enablePathStyleAccess, disableChunkedEncoding, and forceGloba…

…lBucketAccessEnabled for aws client (apache#5702)

* Support enablePathStyleAccess and disableChunkedEncoding for aws client

* add an option for forceGlobalBucketAccessEnabled

* add missing doc

aws-common/src/main/java/io/druid/common/aws/AWSClientConfig.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package io.druid.common.aws;
+
+import com.amazonaws.services.s3.S3ClientOptions;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class AWSClientConfig
+{
+  @JsonProperty
+  private boolean disableChunkedEncoding = S3ClientOptions.DEFAULT_CHUNKED_ENCODING_DISABLED;
+
+  @JsonProperty
+  private boolean enablePathStyleAccess = S3ClientOptions.DEFAULT_PATH_STYLE_ACCESS;
+
+  @JsonProperty
+  protected boolean forceGlobalBucketAccessEnabled = S3ClientOptions.DEFAULT_FORCE_GLOBAL_BUCKET_ACCESS_ENABLED;
+
+  public boolean isDisableChunkedEncoding()
+  {
+    return disableChunkedEncoding;
+  }
+
+  public boolean isEnablePathStyleAccess()
+  {
+    return enablePathStyleAccess;
+  }
+
+  public boolean isForceGlobalBucketAccessEnabled()
+  {
+    return forceGlobalBucketAccessEnabled;
+  }
+}

aws-common/src/main/java/io/druid/common/aws/AWSEndpointConfig.java

@@ -21,30 +21,25 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 
+import javax.annotation.Nullable;
+
 public class AWSEndpointConfig
 {
+  @Nullable
   @JsonProperty
   private String url;
 
-  @JsonProperty
-  private String serviceName;
-
+  @Nullable
   @JsonProperty
   private String signingRegion;
 
-  @JsonProperty
+  @Nullable
   public String getUrl()
   {
     return url;
   }
 
-  @JsonProperty
-  public String getServiceName()
-  {
-    return serviceName;
-  }
-
-  @JsonProperty
+  @Nullable
   public String getSigningRegion()
   {
     return signingRegion;

aws-common/src/main/java/io/druid/common/aws/AWSProxyConfig.java

@@ -35,25 +35,21 @@ public class AWSProxyConfig
   @JsonProperty
   private String password;
 
-  @JsonProperty
   public String getHost()
   {
     return host;
   }
 
-  @JsonProperty
   public int getPort()
   {
     return port;
   }
 
-  @JsonProperty
   public String getUsername()
   {
     return username;
   }
 
-  @JsonProperty
   public String getPassword()
   {
     return password;

docs/content/development/extensions-core/s3.md

@@ -18,6 +18,9 @@ S3-compatible deep storage is basically either S3 or something like Google Stora
 |`druid.s3.secretKey`|S3 secret key.|Must be set.|
 |`druid.storage.bucket`|Bucket to store in.|Must be set.|
 |`druid.storage.baseKey`|Base key prefix to use, i.e. what directory.|Must be set.|
+|`druid.s3.disableChunkedEncoding`|Disables chunked encoding. See [AWS document](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#disableChunkedEncoding--) for details.|false|
+|`druid.s3.enablePathStyleAccess`|Enables path style access. See [AWS document](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#enablePathStyleAccess--) for details.|false|
+|`druid.s3.forceGlobalBucketAccessEnabled`|Enables global bucket access. See [AWS document](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#setForceGlobalBucketAccessEnabled-java.lang.Boolean-) for details.|false|
 |`druid.s3.endpoint.url`|Service endpoint either with or without the protocol.|None|
 |`druid.s3.endpoint.signingRegion`|Region to use for SigV4 signing of requests (e.g. us-west-1).|None|
 |`druid.s3.proxy.host`|Proxy host to connect through.|None|

extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java

@@ -22,15 +22,17 @@
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.ClientConfigurationFactory;
 import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.S3ClientOptions;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.fasterxml.jackson.core.Version;
 import com.fasterxml.jackson.databind.Module;
 import com.google.common.collect.ImmutableList;
 import com.google.inject.Binder;
 import com.google.inject.Provides;
 import com.google.inject.multibindings.MapBinder;
+import io.druid.common.aws.AWSClientConfig;
 import io.druid.common.aws.AWSCredentialsConfig;
 import io.druid.common.aws.AWSEndpointConfig;
 import io.druid.common.aws.AWSProxyConfig;
@@ -80,6 +82,7 @@ public void setupModule(SetupContext context)
   public void configure(Binder binder)
   {
     JsonConfigProvider.bind(binder, "druid.s3", AWSCredentialsConfig.class);
+    JsonConfigProvider.bind(binder, "druid.s3", AWSClientConfig.class);
     JsonConfigProvider.bind(binder, "druid.s3.proxy", AWSProxyConfig.class);
     JsonConfigProvider.bind(binder, "druid.s3.endpoint", AWSEndpointConfig.class);
     MapBinder.newMapBinder(binder, String.class, SearchableVersionedDataFinder.class)
@@ -111,25 +114,26 @@ public void configure(Binder binder)
   public AmazonS3 getAmazonS3Client(
       AWSCredentialsProvider provider,
       AWSProxyConfig proxyConfig,
-      AWSEndpointConfig endpointConfig
+      AWSEndpointConfig endpointConfig,
+      AWSClientConfig clientConfig
   )
   {
-    // AmazonS3ClientBuilder can't be used because it makes integration tests failed
     final ClientConfiguration configuration = new ClientConfigurationFactory().getConfig();
-    final AmazonS3Client client = new AmazonS3Client(provider, setProxyConfig(configuration, proxyConfig));
+    final AmazonS3ClientBuilder builder = AmazonS3Client
+        .builder()
+        .withCredentials(provider)
+        .withClientConfiguration(setProxyConfig(configuration, proxyConfig))
+        .withChunkedEncodingDisabled(clientConfig.isDisableChunkedEncoding())
+        .withPathStyleAccessEnabled(clientConfig.isEnablePathStyleAccess())
+        .withForceGlobalBucketAccessEnabled(clientConfig.isForceGlobalBucketAccessEnabled());
 
     if (StringUtils.isNotEmpty(endpointConfig.getUrl())) {
-      if (StringUtils.isNotEmpty(endpointConfig.getServiceName()) &&
-          StringUtils.isNotEmpty(endpointConfig.getSigningRegion())) {
-        client.setEndpoint(endpointConfig.getUrl(), endpointConfig.getServiceName(), endpointConfig.getSigningRegion());
-      } else {
-        client.setEndpoint(endpointConfig.getUrl());
-      }
+      builder.setEndpointConfiguration(
+          new EndpointConfiguration(endpointConfig.getUrl(), endpointConfig.getSigningRegion())
+      );
     }
 
-    client.setS3ClientOptions(S3ClientOptions.builder().enableForceGlobalBucketAccess().build());
-
-    return client;
+    return builder.build();
   }
 
   private static ClientConfiguration setProxyConfig(ClientConfiguration conf, AWSProxyConfig proxyConfig)

extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java

@@ -23,6 +23,7 @@
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.AWSSessionCredentials;
 import com.google.common.io.Files;
+import io.druid.common.aws.AWSClientConfig;
 import io.druid.common.aws.AWSCredentialsConfig;
 import io.druid.common.aws.AWSEndpointConfig;
 import io.druid.common.aws.AWSProxyConfig;
@@ -43,6 +44,9 @@
 
 public class TestAWSCredentialsProvider
 {
+  @Rule
+  public TemporaryFolder folder = new TemporaryFolder();
+
   private final AWSModule awsModule = new AWSModule();
   private final S3StorageDruidModule s3Module = new S3StorageDruidModule();
 
@@ -60,12 +64,9 @@ public void testWithFixedAWSKeys()
     assertEquals(credentials.getAWSSecretKey(), "secretKeySample");
 
     // try to create
-    s3Module.getAmazonS3Client(provider, new AWSProxyConfig(), new AWSEndpointConfig());
+    s3Module.getAmazonS3Client(provider, new AWSProxyConfig(), new AWSEndpointConfig(), new AWSClientConfig());
   }
 
-  @Rule
-  public TemporaryFolder folder = new TemporaryFolder();
-
   @Test
   public void testWithFileSessionCredentials() throws IOException
   {
@@ -88,6 +89,6 @@ public void testWithFileSessionCredentials() throws IOException
     assertEquals(sessionCredentials.getSessionToken(), "sessionTokenSample");
 
     // try to create
-    s3Module.getAmazonS3Client(provider, new AWSProxyConfig(), new AWSEndpointConfig());
+    s3Module.getAmazonS3Client(provider, new AWSProxyConfig(), new AWSEndpointConfig(), new AWSClientConfig());
   }
 }

integration-tests/docker/historical.conf

@@ -38,3 +38,4 @@ redirect_stderr=true
 priority=100
 autorestart=false
 stdout_logfile=/shared/logs/historical.log
+environment=AWS_REGION=us-east-1

integration-tests/docker/middlemanager.conf

@@ -41,3 +41,4 @@ redirect_stderr=true
 priority=100
 autorestart=false
 stdout_logfile=/shared/logs/middlemanager.log
+environment=AWS_REGION=us-east-1

server/src/main/java/io/druid/guice/AWSModule.java

@@ -25,6 +25,7 @@
 import com.google.inject.Binder;
 import com.google.inject.Module;
 import com.google.inject.Provides;
+import io.druid.common.aws.AWSClientConfig;
 import io.druid.common.aws.AWSCredentialsConfig;
 import io.druid.common.aws.AWSCredentialsUtils;
 import io.druid.common.aws.AWSEndpointConfig;
@@ -38,6 +39,7 @@ public class AWSModule implements Module
   public void configure(Binder binder)
   {
     JsonConfigProvider.bind(binder, "druid.s3", AWSCredentialsConfig.class);
+    JsonConfigProvider.bind(binder, "druid.s3", AWSClientConfig.class);
     JsonConfigProvider.bind(binder, "druid.s3.proxy", AWSProxyConfig.class);
     JsonConfigProvider.bind(binder, "druid.s3.endpoint", AWSEndpointConfig.class);
   }