feat(s3Endpoint): enable configurable s3 endpoint as global or region…

…al (#1525)

pom.xml

@@ -38,7 +38,7 @@
             <dependency>
                 <groupId>software.amazon.awssdk</groupId>
                 <artifactId>bom</artifactId>
-                <version>2.17.295</version>
+                <version>2.20.138</version>
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
@@ -139,7 +139,7 @@
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>greengrassv2-data</artifactId>
-            <version>2.17.x-SNAPSHOT</version>
+            <version>2.20.x-SNAPSHOT</version>
             <exclusions>
                 <exclusion>
                     <groupId>software.amazon.awssdk</groupId>

src/main/java/com/aws/greengrass/componentmanager/builtins/ArtifactDownloaderFactory.java

@@ -14,6 +14,7 @@
 import com.aws.greengrass.componentmanager.plugins.docker.DockerImageDownloader;
 import com.aws.greengrass.componentmanager.plugins.docker.Image;
 import com.aws.greengrass.dependency.Context;
+import com.aws.greengrass.deployment.DeviceConfiguration;
 import com.aws.greengrass.deployment.errorcode.DeploymentErrorCode;
 import com.aws.greengrass.util.GreengrassServiceClientFactory;
 import com.aws.greengrass.util.S3SdkClientFactory;
@@ -50,23 +51,28 @@ public class ArtifactDownloaderFactory {
 
     private final Context context;
 
+    private final DeviceConfiguration deviceConfiguration;
+
     /**
      * ArtifactDownloaderFactory constructor.
      *
      * @param s3SdkClientFactory                      s3SdkClientFactory
      * @param greengrassServiceClientFactory greengrassComponentServiceClientFactory
      * @param componentStore                          componentStore
      * @param context                                 context
+     * @param deviceConfiguration                     deviceConfiguration
      */
     @Inject
     public ArtifactDownloaderFactory(S3SdkClientFactory s3SdkClientFactory,
                                      GreengrassServiceClientFactory greengrassServiceClientFactory,
                                      ComponentStore componentStore,
-                                     Context context) {
+                                     Context context,
+                                     DeviceConfiguration deviceConfiguration) {
         this.s3ClientFactory = s3SdkClientFactory;
         this.clientFactory = greengrassServiceClientFactory;
         this.componentStore = componentStore;
         this.context = context;
+        this.deviceConfiguration = deviceConfiguration;
     }
 
     /**
@@ -84,7 +90,8 @@ public ArtifactDownloader getArtifactDownloader(ComponentIdentifier identifier,
         URI artifactUri = artifact.getArtifactUri();
         String scheme = artifactUri.getScheme() == null ? null : artifactUri.getScheme().toUpperCase();
         if (GREENGRASS_SCHEME.equals(scheme)) {
-            return new GreengrassRepositoryDownloader(clientFactory, identifier, artifact, artifactDir, componentStore);
+            return new GreengrassRepositoryDownloader(clientFactory, identifier, artifact, artifactDir, componentStore,
+                    deviceConfiguration);
         }
         if (S3_SCHEME.equals(scheme)) {
             return new S3Downloader(s3ClientFactory, identifier, artifact, artifactDir, componentStore);

src/main/java/com/aws/greengrass/componentmanager/builtins/GreengrassRepositoryDownloader.java

@@ -10,9 +10,11 @@
 import com.aws.greengrass.componentmanager.exceptions.PackageLoadingException;
 import com.aws.greengrass.componentmanager.models.ComponentArtifact;
 import com.aws.greengrass.componentmanager.models.ComponentIdentifier;
+import com.aws.greengrass.deployment.DeviceConfiguration;
 import com.aws.greengrass.deployment.errorcode.DeploymentErrorCode;
 import com.aws.greengrass.deployment.exceptions.DeviceConfigurationException;
 import com.aws.greengrass.deployment.exceptions.RetryableServerErrorException;
+import com.aws.greengrass.util.Coerce;
 import com.aws.greengrass.util.GreengrassServiceClientFactory;
 import com.aws.greengrass.util.ProxyUtils;
 import com.aws.greengrass.util.RetryUtils;
@@ -53,6 +55,8 @@ public class GreengrassRepositoryDownloader extends ArtifactDownloader {
     private final ComponentStore componentStore;
     private final GreengrassServiceClientFactory clientFactory;
     private Long artifactSize = null;
+    private final DeviceConfiguration deviceConfiguration;
+
     // Setter for unit test
     @Setter(AccessLevel.PACKAGE)
     @Getter(AccessLevel.PACKAGE)
@@ -64,10 +68,12 @@ public class GreengrassRepositoryDownloader extends ArtifactDownloader {
 
     protected GreengrassRepositoryDownloader(GreengrassServiceClientFactory clientFactory,
                                              ComponentIdentifier identifier, ComponentArtifact artifact,
-                                             Path artifactDir, ComponentStore componentStore) {
+                                             Path artifactDir, ComponentStore componentStore,
+                                             DeviceConfiguration deviceConfiguration) {
         super(identifier, artifact, artifactDir, componentStore);
         this.clientFactory = clientFactory;
         this.componentStore = componentStore;
+        this.deviceConfiguration = deviceConfiguration;
     }
 
     protected static String getArtifactFilename(ComponentArtifact artifact) {
@@ -229,7 +235,9 @@ private String getArtifactDownloadURL(ComponentIdentifier componentIdentifier, S
             return RetryUtils.runWithRetry(clientExceptionRetryConfig, () -> {
                 try {
                     GetComponentVersionArtifactRequest getComponentArtifactRequest =
-                            GetComponentVersionArtifactRequest.builder().artifactName(artifactName).arn(arn).build();
+                            GetComponentVersionArtifactRequest.builder().artifactName(artifactName)
+                                    .s3EndpointType(Coerce.toString(deviceConfiguration.gets3EndpointType()))
+                                    .arn(arn).build();
                     GetComponentVersionArtifactResponse getComponentArtifactResult =
                             clientFactory.fetchGreengrassV2DataClient()
                                     .getComponentVersionArtifact(getComponentArtifactRequest);

src/main/java/com/aws/greengrass/deployment/DeploymentDocumentDownloader.java

@@ -172,7 +172,7 @@ private GetDeploymentConfigurationResponse getDeploymentConfiguration(String dep
         String thingName = Coerce.toString(deviceConfiguration.getThingName());
         GetDeploymentConfigurationRequest getDeploymentConfigurationRequest =
                 GetDeploymentConfigurationRequest.builder().deploymentId(deploymentId).coreDeviceThingName(thingName)
-                        .build();
+                        .s3EndpointType(Coerce.toString(deviceConfiguration.gets3EndpointType())).build();
 
         GetDeploymentConfigurationResponse deploymentConfiguration;
 

src/main/java/com/aws/greengrass/deployment/DeviceConfiguration.java

@@ -24,6 +24,7 @@
 import com.aws.greengrass.deployment.errorcode.DeploymentErrorCode;
 import com.aws.greengrass.deployment.exceptions.ComponentConfigurationValidationException;
 import com.aws.greengrass.deployment.exceptions.DeviceConfigurationException;
+import com.aws.greengrass.deployment.model.S3EndpointType;
 import com.aws.greengrass.lifecyclemanager.GreengrassService;
 import com.aws.greengrass.lifecyclemanager.Kernel;
 import com.aws.greengrass.lifecyclemanager.KernelAlternatives;
@@ -43,6 +44,7 @@
 import com.vdurmont.semver4j.Semver;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import org.slf4j.event.Level;
+import software.amazon.awssdk.core.SdkSystemSetting;
 import software.amazon.awssdk.core.exception.SdkClientException;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain;
@@ -122,6 +124,8 @@ public class DeviceConfiguration {
     public static final String NUCLEUS_CONFIG_LOGGING_TOPICS = "logging";
     public static final String TELEMETRY_CONFIG_LOGGING_TOPICS = "telemetry";
 
+    public static final String S3_ENDPOINT_TYPE = "s3EndpointType";
+    public static final String S3_ENDPOINT_PROP_NAME = SdkSystemSetting.AWS_S3_US_EAST_1_REGIONAL_ENDPOINT.property();
     public static final String DEVICE_NETWORK_PROXY_NAMESPACE = "networkProxy";
     public static final String DEVICE_PROXY_NAMESPACE = "proxy";
     public static final String DEVICE_PARAM_NO_PROXY_ADDRESSES = "noProxyAddresses";
@@ -198,6 +202,10 @@ public DeviceConfiguration(Kernel kernel, String thingName, String iotDataEndpoi
         getAWSRegion().withValue(awsRegion);
         getIotRoleAlias().withValue(tesRoleAliasName);
 
+        if (System.getProperty(S3_ENDPOINT_PROP_NAME) != null
+                && System.getProperty(S3_ENDPOINT_PROP_NAME).equalsIgnoreCase(S3EndpointType.REGIONAL.name())) {
+            gets3EndpointType().withValue(S3EndpointType.REGIONAL.name());
+        }
         validate();
     }
 
@@ -679,6 +687,15 @@ public Topic getDeploymentPollingFrequencySeconds() {
         return getTopic(DEPLOYMENT_POLLING_FREQUENCY_SECONDS);
     }
 
+    /**
+     * Get s3 endpoint topic.
+     *
+     * @return s3 endpoint topic
+     */
+    public Topic gets3EndpointType() {
+        return getTopic(S3_ENDPOINT_TYPE).dflt(S3EndpointType.GLOBAL.name());
+    }
+
     /**
      * Subscribe to all device configuration change.
      *

src/main/java/com/aws/greengrass/deployment/model/S3EndpointType.java

@@ -0,0 +1,10 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.aws.greengrass.deployment.model;
+
+public enum S3EndpointType {
+    GLOBAL,REGIONAL
+}

src/main/java/com/aws/greengrass/util/S3SdkClientFactory.java

@@ -7,7 +7,11 @@
 
 import com.aws.greengrass.deployment.DeviceConfiguration;
 import com.aws.greengrass.deployment.exceptions.DeviceConfigurationException;
+import com.aws.greengrass.deployment.model.S3EndpointType;
+import com.aws.greengrass.logging.api.Logger;
+import com.aws.greengrass.logging.impl.LogManager;
 import com.aws.greengrass.tes.LazyCredentialProvider;
+import software.amazon.awssdk.core.SdkSystemSetting;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.S3Configuration;
@@ -23,7 +27,9 @@ public class S3SdkClientFactory {
     static final Map<Region, S3Client> clientCache = new ConcurrentHashMap<>();
     private final LazyCredentialProvider credentialsProvider;
     private final DeviceConfiguration deviceConfiguration;
-
+    private static final Logger logger = LogManager.getLogger(S3SdkClientFactory.class);
+    private static final String S3_ENDPOINT_PROP_NAME = SdkSystemSetting.AWS_S3_US_EAST_1_REGIONAL_ENDPOINT.property();
+    private static final String S3_REGIONAL_ENDPOINT_VALUE = "regional";
     private DeviceConfigurationException configValidationError;
     private Region region;
 
@@ -73,6 +79,7 @@ public S3Client getS3Client() throws DeviceConfigurationException {
         if (configValidationError != null) {
             throw configValidationError;
         }
+        setS3EndpointType(Coerce.toString(deviceConfiguration.gets3EndpointType()));
         return getClientForRegion(region);
     }
 
@@ -88,4 +95,30 @@ public S3Client getClientForRegion(Region r) {
                 .serviceConfiguration(S3Configuration.builder().useArnRegionEnabled(true).build())
                 .credentialsProvider(credentialsProvider).region(r).build());
     }
+
+    /**
+     * Set s3 endpoint type.
+     *
+     * @param type s3EndpointType
+     */
+    private void setS3EndpointType(String type) {
+        //Check if system property and device config are consistent
+        //If not consistent, set system property according to device config value
+        String s3EndpointSystemProp = System.getProperty(S3_ENDPOINT_PROP_NAME);
+        boolean isGlobal = S3EndpointType.GLOBAL.name().equals(type);
+
+        if (isGlobal && S3_REGIONAL_ENDPOINT_VALUE.equals(s3EndpointSystemProp)) {
+            System.clearProperty(S3_ENDPOINT_PROP_NAME);
+            refreshClientCache();
+            logger.atDebug().log("s3 endpoint set to global");
+        } else if (!isGlobal && !S3_REGIONAL_ENDPOINT_VALUE.equals(s3EndpointSystemProp)) {
+            System.setProperty(S3_ENDPOINT_PROP_NAME, S3_REGIONAL_ENDPOINT_VALUE);
+            refreshClientCache();
+            logger.atDebug().log("s3 endpoint set to regional");
+        }
+    }
+
+    private void refreshClientCache() {
+        clientCache.remove(region);
+    }
 }

src/test/java/com/aws/greengrass/componentmanager/builtins/ArtifactDownloaderFactoryTest.java

@@ -10,6 +10,7 @@
 import com.aws.greengrass.componentmanager.exceptions.PackageLoadingException;
 import com.aws.greengrass.componentmanager.models.ComponentArtifact;
 import com.aws.greengrass.componentmanager.models.ComponentIdentifier;
+import com.aws.greengrass.deployment.DeviceConfiguration;
 import com.aws.greengrass.dependency.Context;
 import com.aws.greengrass.testcommons.testutilities.GGExtension;
 import com.aws.greengrass.util.GreengrassServiceClientFactory;
@@ -54,13 +55,16 @@ class ArtifactDownloaderFactoryTest {
     @Mock
     Context context;
 
+    @Mock
+    DeviceConfiguration deviceConfiguration;
+
     ArtifactDownloaderFactory artifactDownloaderFactory;
 
     @BeforeEach
     public void setup() {
         artifactDownloaderFactory =
                 new ArtifactDownloaderFactory(s3SdkClientFactory, greengrassServiceClientFactory,
-                        componentStore, context);
+                        componentStore, context, deviceConfiguration);
     }
 
     @Test