From 87846ad481c4fafebdfa9258d6c481e101c7da0e Mon Sep 17 00:00:00 2001 From: "Bala.FA" Date: Sun, 9 Jul 2023 15:50:17 +0530 Subject: [PATCH] Add generic AWS S3 domain support Signed-off-by: Bala.FA --- minio/api.py | 113 ++- minio/helpers.py | 322 +++++--- tests/unit/helpers.py | 1743 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1988 insertions(+), 190 deletions(-) diff --git a/minio/api.py b/minio/api.py index d0648a5fb..d20bca3cc 100644 --- a/minio/api.py +++ b/minio/api.py @@ -402,7 +402,7 @@ def _execute( no_body_trace=False, ): """Execute HTTP request.""" - region = self._get_region(bucket_name, None) + region = self._get_region(bucket_name) try: return self._url_open( @@ -442,22 +442,12 @@ def _execute( ) raise exc.copy(code, message) - def _get_region(self, bucket_name, region): + def _get_region(self, bucket_name): """ Return region of given bucket either from region cache or set in constructor. """ - if region: - # Error out if region does not match with region passed via - # constructor. - if self._base_url.region and self._base_url.region != region: - raise ValueError( - f"region must be {self._base_url.region}, " - f"but passed {region}" - ) - return region - if self._base_url.region: return self._base_url.region @@ -479,7 +469,7 @@ def _get_region(self, bucket_name, region): element = ET.fromstring(response.data.decode()) if not element.text: region = "us-east-1" - elif element.text == "EU": + elif element.text == "EU" and self._base_url.is_aws_host: region = "eu-west-1" else: region = element.text @@ -566,7 +556,7 @@ def select_object_content(self, bucket_name, object_name, request): print(data.decode()) print(result.stats()) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) check_non_empty_string(object_name) if not isinstance(request, SelectRequest): raise ValueError("request must be SelectRequest type") @@ -600,7 +590,8 @@ def make_bucket(self, bucket_name, location=None, object_lock=False): # Create bucket with object-lock feature on specific region. client.make_bucket("my-bucket", "eu-west-2", object_lock=True) """ - check_bucket_name(bucket_name, True) + check_bucket_name(bucket_name, True, + s3_check=self._base_url.is_aws_host) if self._base_url.region: # Error out if region does not match with region passed via # constructor. @@ -658,7 +649,7 @@ def bucket_exists(self, bucket_name): else: print("my-bucket does not exist") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) try: self._execute("HEAD", bucket_name) return True @@ -676,7 +667,7 @@ def remove_bucket(self, bucket_name): Example:: client.remove_bucket("my-bucket") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) self._execute("DELETE", bucket_name) self._region_map.pop(bucket_name, None) @@ -690,7 +681,7 @@ def get_bucket_policy(self, bucket_name): Example:: policy = client.get_bucket_policy("my-bucket") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) response = self._execute( "GET", bucket_name, query_params={"policy": ""}, ) @@ -705,7 +696,7 @@ def delete_bucket_policy(self, bucket_name): Example:: client.delete_bucket_policy("my-bucket") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) self._execute("DELETE", bucket_name, query_params={"policy": ""}) def set_bucket_policy(self, bucket_name, policy): @@ -718,7 +709,7 @@ def set_bucket_policy(self, bucket_name, policy): Example:: client.set_bucket_policy("my-bucket", policy) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) is_valid_policy_type(policy) self._execute( "PUT", @@ -738,7 +729,7 @@ def get_bucket_notification(self, bucket_name): Example:: config = client.get_bucket_notification("my-bucket") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) response = self._execute( "GET", bucket_name, query_params={"notification": ""}, ) @@ -764,7 +755,7 @@ def set_bucket_notification(self, bucket_name, config): ) client.set_bucket_notification("my-bucket", config) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) if not isinstance(config, NotificationConfig): raise ValueError("config must be NotificationConfig type") body = marshal(config) @@ -800,7 +791,7 @@ def set_bucket_encryption(self, bucket_name, config): "my-bucket", SSEConfig(Rule.new_sse_s3_rule()), ) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) if not isinstance(config, SSEConfig): raise ValueError("config must be SSEConfig type") body = marshal(config) @@ -822,7 +813,7 @@ def get_bucket_encryption(self, bucket_name): Example:: config = client.get_bucket_encryption("my-bucket") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) try: response = self._execute( "GET", @@ -844,7 +835,7 @@ def delete_bucket_encryption(self, bucket_name): Example:: client.delete_bucket_encryption("my-bucket") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) try: self._execute( "DELETE", @@ -878,7 +869,7 @@ def listen_bucket_notification(self, bucket_name, prefix='', suffix='', for event in events: print(event) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) if self._base_url.is_aws_host: raise ValueError( "ListenBucketNotification API is not supported in Amazon S3", @@ -909,7 +900,7 @@ def set_bucket_versioning(self, bucket_name, config): "my-bucket", VersioningConfig(ENABLED), ) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) if not isinstance(config, VersioningConfig): raise ValueError("config must be VersioningConfig type") body = marshal(config) @@ -932,7 +923,7 @@ def get_bucket_versioning(self, bucket_name): config = client.get_bucket_versioning("my-bucket") print(config.status) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) response = self._execute( "GET", bucket_name, @@ -1033,7 +1024,7 @@ def fget_object(self, bucket_name, object_name, file_path, ssec=SseCustomerKey(b"32byteslongsecretkeymustprovided"), ) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) check_non_empty_string(object_name) if progress and not isinstance(progress, Thread): raise TypeError("progress object must be instance of Thread") @@ -1144,7 +1135,7 @@ def get_object(self, bucket_name, object_name, offset=0, length=0, response.close() response.release_conn() """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) check_non_empty_string(object_name) check_ssec(ssec) @@ -1223,7 +1214,7 @@ def copy_object(self, bucket_name, object_name, source, ) print(result.object_name, result.version_id) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) check_non_empty_string(object_name) if not isinstance(source, CopySource): raise ValueError("source must be CopySource type") @@ -1426,7 +1417,7 @@ def compose_object( # pylint: disable=too-many-branches ) print(result.object_name, result.version_id) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) check_non_empty_string(object_name) if not isinstance(sources, (list, tuple)) or not sources: raise ValueError("sources must be non-empty list or tuple type") @@ -1660,7 +1651,7 @@ def put_object(self, bucket_name, object_name, data, length, legal_hold=True, ) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) check_non_empty_string(object_name) check_sse(sse) if tags is not None and not isinstance(tags, Tags): @@ -1861,7 +1852,7 @@ def stat_object(self, bucket_name, object_name, ssec=None, version_id=None, ) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) check_non_empty_string(object_name) check_ssec(ssec) @@ -1909,7 +1900,7 @@ def remove_object(self, bucket_name, object_name, version_id=None): version_id="dfbd25b3-abec-4184-a4e8-5a35a5c1174d", ) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) check_non_empty_string(object_name) self._execute( "DELETE", @@ -1985,7 +1976,7 @@ def remove_objects(self, bucket_name, delete_object_list, for error in errors: print("error occurred when deleting object", error) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) # turn list like objects into an iterator. delete_object_list = itertools.chain(delete_object_list) @@ -2048,12 +2039,12 @@ def get_presigned_url(self, method, bucket_name, object_name, ) print(url) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) check_non_empty_string(object_name) if expires.total_seconds() < 1 or expires.total_seconds() > 604800: raise ValueError("expires must be between 1 second to 7 days") - region = self._get_region(bucket_name, None) + region = self._get_region(bucket_name) query_params = extra_query_params or {} query_params.update({"versionId": version_id} if version_id else {}) query_params.update(response_headers or {}) @@ -2178,9 +2169,11 @@ def presigned_post_policy(self, policy): raise ValueError( "anonymous access does not require presigned post form-data", ) + check_bucket_name( + policy.bucket_name, s3_check=self._base_url.is_aws_host) return policy.form_data( self._provider.retrieve(), - self._get_region(policy.bucket_name, None), + self._get_region(policy.bucket_name), ) def delete_bucket_replication(self, bucket_name): @@ -2192,7 +2185,7 @@ def delete_bucket_replication(self, bucket_name): Example:: client.delete_bucket_replication("my-bucket") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) self._execute("DELETE", bucket_name, query_params={"replication": ""}) def get_bucket_replication(self, bucket_name): @@ -2205,7 +2198,7 @@ def get_bucket_replication(self, bucket_name): Example:: config = client.get_bucket_replication("my-bucket") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) try: response = self._execute( "GET", bucket_name, query_params={"replication": ""}, @@ -2248,7 +2241,7 @@ def set_bucket_replication(self, bucket_name, config): ) client.set_bucket_replication("my-bucket", config) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) if not isinstance(config, ReplicationConfig): raise ValueError("config must be ReplicationConfig type") body = marshal(config) @@ -2269,7 +2262,7 @@ def delete_bucket_lifecycle(self, bucket_name): Example:: client.delete_bucket_lifecycle("my-bucket") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) self._execute("DELETE", bucket_name, query_params={"lifecycle": ""}) def get_bucket_lifecycle(self, bucket_name): @@ -2282,7 +2275,7 @@ def get_bucket_lifecycle(self, bucket_name): Example:: config = client.get_bucket_lifecycle("my-bucket") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) try: response = self._execute( "GET", bucket_name, query_params={"lifecycle": ""}, @@ -2321,7 +2314,7 @@ def set_bucket_lifecycle(self, bucket_name, config): ) client.set_bucket_lifecycle("my-bucket", config) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) if not isinstance(config, LifecycleConfig): raise ValueError("config must be LifecycleConfig type") body = marshal(config) @@ -2342,7 +2335,7 @@ def delete_bucket_tags(self, bucket_name): Example:: client.delete_bucket_tags("my-bucket") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) self._execute("DELETE", bucket_name, query_params={"tagging": ""}) def get_bucket_tags(self, bucket_name): @@ -2355,7 +2348,7 @@ def get_bucket_tags(self, bucket_name): Example:: tags = client.get_bucket_tags("my-bucket") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) try: response = self._execute( "GET", bucket_name, query_params={"tagging": ""}, @@ -2380,7 +2373,7 @@ def set_bucket_tags(self, bucket_name, tags): tags["User"] = "jsmith" client.set_bucket_tags("my-bucket", tags) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) if not isinstance(tags, Tags): raise ValueError("tags must be Tags type") body = marshal(Tagging(tags)) @@ -2403,7 +2396,7 @@ def delete_object_tags(self, bucket_name, object_name, version_id=None): Example:: client.delete_object_tags("my-bucket", "my-object") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) check_non_empty_string(object_name) query_params = {"versionId": version_id} if version_id else {} query_params["tagging"] = "" @@ -2426,7 +2419,7 @@ def get_object_tags(self, bucket_name, object_name, version_id=None): Example:: tags = client.get_object_tags("my-bucket", "my-object") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) check_non_empty_string(object_name) query_params = {"versionId": version_id} if version_id else {} query_params["tagging"] = "" @@ -2459,7 +2452,7 @@ def set_object_tags(self, bucket_name, object_name, tags, version_id=None): tags["User"] = "jsmith" client.set_object_tags("my-bucket", "my-object", tags) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) check_non_empty_string(object_name) if not isinstance(tags, Tags): raise ValueError("tags must be Tags type") @@ -2488,7 +2481,7 @@ def enable_object_legal_hold( Example:: client.enable_object_legal_hold("my-bucket", "my-object") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) check_non_empty_string(object_name) body = marshal(LegalHold(True)) query_params = {"versionId": version_id} if version_id else {} @@ -2515,7 +2508,7 @@ def disable_object_legal_hold( Example:: client.disable_object_legal_hold("my-bucket", "my-object") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) check_non_empty_string(object_name) body = marshal(LegalHold(False)) query_params = {"versionId": version_id} if version_id else {} @@ -2545,7 +2538,7 @@ def is_object_legal_hold_enabled( else: print("legal hold is not enabled on my-object") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) check_non_empty_string(object_name) query_params = {"versionId": version_id} if version_id else {} query_params["legal-hold"] = "" @@ -2586,7 +2579,7 @@ def get_object_lock_config(self, bucket_name): Example:: config = client.get_object_lock_config("my-bucket") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) response = self._execute( "GET", bucket_name, query_params={"object-lock": ""}, ) @@ -2603,7 +2596,7 @@ def set_object_lock_config(self, bucket_name, config): config = ObjectLockConfig(GOVERNANCE, 15, DAYS) client.set_object_lock_config("my-bucket", config) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) if not isinstance(config, ObjectLockConfig): raise ValueError("config must be ObjectLockConfig type") body = marshal(config) @@ -2629,7 +2622,7 @@ def get_object_retention( Example:: config = client.get_object_retention("my-bucket", "my-object") """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) check_non_empty_string(object_name) query_params = {"versionId": version_id} if version_id else {} query_params["retention"] = "" @@ -2663,7 +2656,7 @@ def set_object_retention( ) client.set_object_retention("my-bucket", "my-object", config) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) check_non_empty_string(object_name) if not isinstance(config, Retention): raise ValueError("config must be Retention type") @@ -2718,7 +2711,7 @@ def upload_snowball_objects(self, bucket_name, object_list, metadata=None, ], ) """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) object_name = f"snowball.{random()}.tar" @@ -2778,7 +2771,7 @@ def _list_objects( # pylint: disable=too-many-arguments,too-many-branches policies. """ - check_bucket_name(bucket_name) + check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host) if version_id_marker: include_version = True diff --git a/minio/helpers.py b/minio/helpers.py index b955d8790..727ae5406 100644 --- a/minio/helpers.py +++ b/minio/helpers.py @@ -37,18 +37,36 @@ MAX_PART_SIZE = 5 * 1024 * 1024 * 1024 # 5GiB MIN_PART_SIZE = 5 * 1024 * 1024 # 5MiB -_VALID_BUCKETNAME_REGEX = re.compile( - '^[A-Za-z0-9][A-Za-z0-9\\.\\-\\_\\:]{1,61}[A-Za-z0-9]$') -_VALID_BUCKETNAME_STRICT_REGEX = re.compile( - '^[a-z0-9][a-z0-9\\.\\-]{1,61}[a-z0-9]$') -_VALID_IP_ADDRESS = re.compile( - r'^(\d+\.){3}\d+$') -_ALLOWED_HOSTNAME_REGEX = re.compile( - '^((?!-)(?!_)[A-Z_\\d-]{1,63}(? 63: - raise ValueError('Bucket name cannot be greater than' - ' 63 characters.') + if strict: + if not _BUCKET_NAME_REGEX.match(bucket_name): + raise ValueError('invalid bucket name {bucket_name}') + else: + if not _OLD_BUCKET_NAME_REGEX.match(bucket_name): + raise ValueError('invalid bucket name {bucket_name}') - match = _VALID_IP_ADDRESS.match(bucket_name) - if match: - raise ValueError('Bucket name cannot be an ip address') + if _IPV4_REGEX.match(bucket_name): + raise ValueError('bucket name {bucket_name} must not be formatted ' + 'as an IP address') unallowed_successive_chars = ['..', '.-', '-.'] if any(x in bucket_name for x in unallowed_successive_chars): - raise ValueError('Bucket name contains invalid ' - 'successive chars ' - + str(unallowed_successive_chars) + '.') - - if strict: - match = _VALID_BUCKETNAME_STRICT_REGEX.match(bucket_name) - if (not match) or match.end() != len(bucket_name): - raise ValueError('Bucket name contains invalid ' - 'characters (strictly enforced).') - - match = _VALID_BUCKETNAME_REGEX.match(bucket_name) - if (not match) or match.end() != len(bucket_name): - raise ValueError( - f"Bucket name does not follow S3 standards. Bucket: {bucket_name}", - ) + raise ValueError('bucket name {bucket_name} contains invalid ' + 'successive characters') + + if ( + s3_check and + bucket_name.startswith("xn--") or + bucket_name.endswith("-s3alias") or + bucket_name.endswith("--ol-s3") + ): + raise ValueError("bucket name {bucket_name} must not start with " + "'xn--' and must not end with '--s3alias' or " + "'--ol-s3'") def check_non_empty_string(string): @@ -365,22 +374,57 @@ def genheaders(headers, sse, tags, retention, legal_hold): return headers -def _extract_region(host): - """Extract region from Amazon S3 host.""" +def _get_aws_info(host, https, region): + """Extract AWS domain information. """ - tokens = host.split(".") - token = tokens[1] + if not _HOSTNAME_REGEX.match(host): + return (None, None) - # If token is "dualstack", then region might be in next token. - if token == "dualstack": - token = tokens[2] + if _AWS_ELB_ENDPOINT_REGEX.match(host): + region_in_host = host.split(".elb.amazonaws.com", 1)[0].split(".")[-1] + return (None, region or region_in_host) - # If token is equal to "amazonaws", region is not passed in the host. - if token == "amazonaws": - return None + if not _AWS_ENDPOINT_REGEX.match(host): + return (None, None) + + if not _AWS_S3_ENDPOINT_REGEX.match(host): + raise ValueError(f"invalid Amazon AWS host {host}") - # Return token as region. - return token + end = _AWS_S3_PREFIX_REGEX.match(host).end() + aws_s3_prefix = host[:end] + + if "s3-accesspoint" in aws_s3_prefix and not https: + raise ValueError(f"use HTTPS scheme for host {host}") + + tokens = host[end:].split(".") + dualstack = tokens[0] == "dualstack" + if dualstack: + tokens = tokens[1:] + region_in_host = None + if tokens[0] not in ["vpce", "amazonaws"]: + region_in_host = tokens[0] + tokens = tokens[1:] + aws_domain_suffix = ".".join(tokens) + + if host in "s3-external-1.amazonaws.com": + region_in_host = "us-east-1" + + if host in ["s3-us-gov-west-1.amazonaws.com", + "s3-fips-us-gov-west-1.amazonaws.com"]: + region_in_host = "us-gov-west-1" + + if (aws_domain_suffix.endswith(".cn") and + not aws_s3_prefix.endswith("s3-accelerate.") and + not region_in_host and + not region): + raise ValueError( + f"region missing in Amazon S3 China endpoint {host}", + ) + + return ({"s3_prefix": aws_s3_prefix, + "domain_suffix": aws_domain_suffix, + "region": region or region_in_host, + "dualstack": dualstack}, None) class BaseURL: @@ -423,42 +467,22 @@ def __init__(self, endpoint, region): ): url = url_replace(url, netloc=host) - self._accelerate_host_flag = host.startswith("s3-accelerate.") - self._is_aws_host = ( - ( - host.startswith("s3.") or self._accelerate_host_flag - ) and - ( - host.endswith(".amazonaws.com") or - host.endswith(".amazonaws.com.cn") - ) - ) + if region and not _REGION_REGEX.match(region): + raise ValueError(f"invalid region {region}") + + self._aws_info, region_in_host = _get_aws_info( + host, url.scheme == "https", region) self._virtual_style_flag = ( - self._is_aws_host or host.endswith("aliyuncs.com") + self._aws_info or host.endswith("aliyuncs.com") ) - - region_in_host = None - if self._is_aws_host: - is_aws_china_host = host.endswith(".cn") - url = url_replace( - url, - netloc=( - "amazonaws.com.cn" - if is_aws_china_host else "amazonaws.com" - ), - ) - region_in_host = _extract_region(host) - - if is_aws_china_host and not region_in_host and not region: - raise ValueError( - f"region missing in Amazon S3 China endpoint {endpoint}", - ) - self._dualstack_host_flag = ".dualstack." in host - else: - self._accelerate_host_flag = False - self._url = url self._region = region or region_in_host + self._accelerate_host_flag = False + if self._aws_info: + self._region = self._aws_info["region"] + self._accelerate_host_flag = ( + self._aws_info["s3_prefix"].endswith("s3-accelerate.") + ) @property def region(self): @@ -478,29 +502,41 @@ def host(self): @property def is_aws_host(self): """Check if URL points to AWS host.""" - return self._is_aws_host + return self._aws_info is not None + + @property + def aws_s3_prefix(self): + """Get AWS S3 domain prefix.""" + return self._aws_info["s3_prefix"] if self._aws_info else None + + @aws_s3_prefix.setter + def aws_s3_prefix(self, s3_prefix): + """Set AWS s3 domain prefix.""" + if not _AWS_S3_PREFIX_REGEX.match(s3_prefix): + raise ValueError(f"invalid AWS S3 domain prefix {s3_prefix}") + if self._aws_info: + self._aws_info["s3_prefix"] = s3_prefix @property def accelerate_host_flag(self): - """Check if URL points to AWS accelerate host.""" + """Get AWS accelerate host flag.""" return self._accelerate_host_flag @accelerate_host_flag.setter def accelerate_host_flag(self, flag): - """Check if URL points to AWS accelerate host.""" - if self._is_aws_host: - self._accelerate_host_flag = flag + """Set AWS accelerate host flag.""" + self._accelerate_host_flag = flag @property def dualstack_host_flag(self): """Check if URL points to AWS dualstack host.""" - return self._dualstack_host_flag + return self._aws_info["dualstack"] if self._aws_info else False @dualstack_host_flag.setter def dualstack_host_flag(self, flag): - """Check to use virtual style or not.""" - if self._is_aws_host: - self._dualstack_host_flag = flag + """Set AWS dualstack host.""" + if self._aws_info: + self._aws_info["dualstack"] = flag @property def virtual_style_flag(self): @@ -512,17 +548,67 @@ def virtual_style_flag(self, flag): """Check to use virtual style or not.""" self._virtual_style_flag = flag + def _build_aws_url(self, url, bucket_name, enforce_path_style, region): + """Build URL for given information.""" + s3_prefix = self._aws_info["s3_prefix"] + domain_suffix = self._aws_info["domain_suffix"] + + host = f"{s3_prefix}{domain_suffix}" + if host in ["s3-external-1.amazonaws.com", + "s3-us-gov-west-1.amazonaws.com", + "s3-fips-us-gov-west-1.amazonaws.com"]: + return url_replace(url, netloc=host) + + netloc = s3_prefix + if "s3-accelerate" in s3_prefix: + if "." in bucket_name: + raise ValueError( + f"bucket name '{bucket_name}' with '.' is not allowed " + f"for accelerate endpoint" + ) + if enforce_path_style: + netloc = netloc.replace("-accelerate", "", 1) + + if self._aws_info["dualstack"]: + netloc += "dualstack." + if "s3-accelerate" not in s3_prefix: + netloc += region + "." + netloc += domain_suffix + + return url_replace(url, netloc=netloc) + + def _build_list_buckets_url(self, url, region): + """Build URL for ListBuckets API.""" + if not self._aws_info: + return url + + s3_prefix = self._aws_info["s3_prefix"] + domain_suffix = self._aws_info["domain_suffix"] + + host = f"{s3_prefix}{domain_suffix}" + if host in ["s3-external-1.amazonaws.com", + "s3-us-gov-west-1.amazonaws.com", + "s3-fips-us-gov-west-1.amazonaws.com"]: + return url_replace(url, netloc=host) + + if s3_prefix.startswith("s3.") or s3_prefix.startswith("s3-"): + s3_prefix = "s3." + cn_suffix = ".cn" if domain_suffix.endswith(".cn") else "" + domain_suffix = f"amazonaws.com{cn_suffix}" + return url_replace(url, netloc=f"{s3_prefix}{region}.{domain_suffix}") + def build( self, method, region, bucket_name=None, object_name=None, query_params=None, ): """Build URL for given information.""" - if not bucket_name and object_name: raise ValueError( f"empty bucket name for object name {object_name}", ) + url = url_replace(self._url, path="/") + query = [] for key, values in sorted((query_params or {}).items()): values = values if isinstance(values, (list, tuple)) else [values] @@ -530,15 +616,10 @@ def build( f"{queryencode(key)}={queryencode(value)}" for value in sorted(values) ] - url = url_replace(self._url, query="&".join(query)) - host = self._url.netloc + url = url_replace(url, query="&".join(query)) if not bucket_name: - url = url_replace(url, path="/") - return ( - url_replace(url, netloc="s3." + region + "." + host) - if self._is_aws_host else url - ) + return self._build_list_buckets_url(url, region) enforce_path_style = ( # CreateBucket API requires path style in Amazon AWS S3. @@ -552,40 +633,21 @@ def build( ("." in bucket_name and self._url.scheme == "https") ) - if self._is_aws_host: - s3_domain = "s3." - if self._accelerate_host_flag: - if "." in bucket_name: - raise ValueError( - f"bucket name '{bucket_name}' with '.' is not allowed " - f"for accelerated endpoint" - ) - - if not enforce_path_style: - s3_domain = "s3-accelerate." + if self._aws_info: + url = self._build_aws_url( + url, bucket_name, enforce_path_style, region) - dual_stack = "dualstack." if self._dualstack_host_flag else "" - endpoint = s3_domain + dual_stack - if enforce_path_style or not self._accelerate_host_flag: - endpoint += region + "." - host = endpoint + host + netloc = url.netloc + path = "/" if enforce_path_style or not self._virtual_style_flag: - url = url_replace(url, netloc=host) - url = url_replace(url, path="/" + bucket_name) + path = f"/{bucket_name}" else: - url = url_replace( - url, - netloc=bucket_name + "." + host, - path="/", - ) - + netloc = f"{bucket_name}.{netloc}" if object_name: - path = url.path path += ("" if path.endswith("/") else "/") + quote(object_name) - url = url_replace(url, path=path) - return url + return url_replace(url, netloc=netloc, path=path) class ObjectWriteResult: diff --git a/tests/unit/helpers.py b/tests/unit/helpers.py index c37fc5c53..40db65d78 100644 --- a/tests/unit/helpers.py +++ b/tests/unit/helpers.py @@ -14,6 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from collections import namedtuple +from unittest import TestCase +from urllib.parse import urlunsplit + +from minio.helpers import BaseURL + def generate_error(code, message, request_id, host_id, resource, bucket_name, object_name): @@ -29,3 +35,1740 @@ def generate_error(code, message, request_id, host_id, '''.format(code, message, request_id, host_id, resource, bucket_name, object_name) + + +class BaseURLTests(TestCase): + def test_aws_new_baseurl_error(self): + cases = [ + # invalid Amazon AWS host error + "https://z3.amazonaws.com", + "https://1234567890.s3.amazonaws.com", + "https://1234567890.s3-accelerate.amazonaws.com", + "https://1234567890.abcdefgh.s3-control.amazonaws.com", + "https://s3fips.amazonaws.com", + "https://s3-fips.s3.amazonaws.com", + "https://s3-fips.s3accelerate.amazonaws.com", + "https://s3-fips.s3-accelerate.amazonaws.com", + "https://bucket.bucket.vpce-1a2b3c4d-5e6f.s3.us-east-1." + "vpce.amazonaws.com", + "https://accesspoint.accesspoint.vpce-1a2b3c4d-5e6f.s3.us-east-1." + "vpce.amazonaws.com", + "https://accesspoint.vpce-1123.vpce-xyz.s3.amazonaws.com", + # use HTTPS scheme for host error + "http://s3-accesspoint.amazonaws.com", + # region missing in Amazon S3 China endpoint error + "https://s3.amazonaws.com.cn", + ] + for endpoint in cases: + self.assertRaises(ValueError, BaseURL, endpoint, None) + + def test_aws_new_baseurl(self): + Case = namedtuple("Case", ["args", "result"]) + cases = [ + Case( + ("https://s3.amazonaws.com", None), + { + "s3_prefix": "s3.", + "domain_suffix": "amazonaws.com", + "region": None, + "dualstack": False, + }, + ), + Case( + ("https://s3.amazonaws.com", "ap-south-1a"), + { + "s3_prefix": "s3.", + "domain_suffix": "amazonaws.com", + "region": "ap-south-1a", + "dualstack": False, + }, + ), + Case( + ("https://s3.us-gov-east-1.amazonaws.com", None), + { + "s3_prefix": "s3.", + "domain_suffix": "amazonaws.com", + "region": "us-gov-east-1", + "dualstack": False, + }, + ), + Case( + ("https://s3.me-south-1.amazonaws.com", "cn-northwest-1"), + { + "s3_prefix": "s3.", + "domain_suffix": "amazonaws.com", + "region": "cn-northwest-1", + "dualstack": False, + }, + ), + ### + Case( + ("https://s3.dualstack.amazonaws.com", None), + { + "s3_prefix": "s3.", + "domain_suffix": "amazonaws.com", + "region": None, + "dualstack": True, + }, + ), + Case( + ("https://s3.dualstack.amazonaws.com", "ap-south-1a"), + { + "s3_prefix": "s3.", + "domain_suffix": "amazonaws.com", + "region": "ap-south-1a", + "dualstack": True, + }, + ), + Case( + ("https://s3.dualstack.us-gov-east-1.amazonaws.com", None), + { + "s3_prefix": "s3.", + "domain_suffix": "amazonaws.com", + "region": "us-gov-east-1", + "dualstack": True, + }, + ), + Case( + ("https://s3.dualstack.me-south-1.amazonaws.com", + "cn-northwest-1"), + { + "s3_prefix": "s3.", + "domain_suffix": "amazonaws.com", + "region": "cn-northwest-1", + "dualstack": True, + }, + ), + ### + Case( + ("https://s3-accelerate.amazonaws.com", None), + { + "s3_prefix": "s3-accelerate.", + "domain_suffix": "amazonaws.com", + "region": None, + "dualstack": False, + }, + ), + Case( + ("https://s3-accelerate.amazonaws.com", "ap-south-1a"), + { + "s3_prefix": "s3-accelerate.", + "domain_suffix": "amazonaws.com", + "region": "ap-south-1a", + "dualstack": False, + }, + ), + Case( + ("https://s3-accelerate.us-gov-east-1.amazonaws.com", None), + { + "s3_prefix": "s3-accelerate.", + "domain_suffix": "amazonaws.com", + "region": "us-gov-east-1", + "dualstack": False, + }, + ), + Case( + ("https://s3-accelerate.me-south-1.amazonaws.com", + "cn-northwest-1"), + { + "s3_prefix": "s3-accelerate.", + "domain_suffix": "amazonaws.com", + "region": "cn-northwest-1", + "dualstack": False, + }, + ), + ### + Case( + ("https://s3-accelerate.dualstack.amazonaws.com", None), + { + "s3_prefix": "s3-accelerate.", + "domain_suffix": "amazonaws.com", + "region": None, + "dualstack": True, + }, + ), + Case( + ("https://s3-accelerate.dualstack.amazonaws.com", + "ap-south-1a"), + { + "s3_prefix": "s3-accelerate.", + "domain_suffix": "amazonaws.com", + "region": "ap-south-1a", + "dualstack": True, + }, + ), + Case( + ("https://s3-accelerate.dualstack.us-gov-east-1.amazonaws.com", + None), + { + "s3_prefix": "s3-accelerate.", + "domain_suffix": "amazonaws.com", + "region": "us-gov-east-1", + "dualstack": True, + }, + ), + Case( + ("https://s3-accelerate.dualstack.me-south-1.amazonaws.com", + "cn-northwest-1"), + { + "s3_prefix": "s3-accelerate.", + "domain_suffix": "amazonaws.com", + "region": "cn-northwest-1", + "dualstack": True, + }, + ), + ### + Case( + ("https://s3-fips.amazonaws.com", None), + { + "s3_prefix": "s3-fips.", + "domain_suffix": "amazonaws.com", + "region": None, + "dualstack": False, + }, + ), + Case( + ("https://s3-fips.amazonaws.com", "ap-south-1a"), + { + "s3_prefix": "s3-fips.", + "domain_suffix": "amazonaws.com", + "region": "ap-south-1a", + "dualstack": False, + }, + ), + Case( + ("https://s3-fips.us-gov-east-1.amazonaws.com", None), + { + "s3_prefix": "s3-fips.", + "domain_suffix": "amazonaws.com", + "region": "us-gov-east-1", + "dualstack": False, + }, + ), + Case( + ("https://s3-fips.me-south-1.amazonaws.com", "cn-northwest-1"), + { + "s3_prefix": "s3-fips.", + "domain_suffix": "amazonaws.com", + "region": "cn-northwest-1", + "dualstack": False, + }, + ), + ### + Case( + ("https://s3-fips.dualstack.amazonaws.com", None), + { + "s3_prefix": "s3-fips.", + "domain_suffix": "amazonaws.com", + "region": None, + "dualstack": True, + }, + ), + Case( + ("https://s3-fips.dualstack.amazonaws.com", "ap-south-1a"), + { + "s3_prefix": "s3-fips.", + "domain_suffix": "amazonaws.com", + "region": "ap-south-1a", + "dualstack": True, + }, + ), + Case( + ("https://s3-fips.dualstack.us-gov-east-1.amazonaws.com", None), + { + "s3_prefix": "s3-fips.", + "domain_suffix": "amazonaws.com", + "region": "us-gov-east-1", + "dualstack": True, + }, + ), + Case( + ("https://s3-fips.dualstack.me-south-1.amazonaws.com", + "cn-northwest-1"), + { + "s3_prefix": "s3-fips.", + "domain_suffix": "amazonaws.com", + "region": "cn-northwest-1", + "dualstack": True, + }, + ), + ### + Case( + ("https://s3-external-1.amazonaws.com", None), + { + "s3_prefix": "s3-external-1.", + "domain_suffix": "amazonaws.com", + "region": "us-east-1", + "dualstack": False, + }, + ), + Case( + ("https://s3-us-gov-west-1.amazonaws.com", None), + { + "s3_prefix": "s3-us-gov-west-1.", + "domain_suffix": "amazonaws.com", + "region": "us-gov-west-1", + "dualstack": False, + }, + ), + Case( + ("https://s3-fips-us-gov-west-1.amazonaws.com", None), + { + "s3_prefix": "s3-fips-us-gov-west-1.", + "domain_suffix": "amazonaws.com", + "region": "us-gov-west-1", + "dualstack": False, + }, + ), + ### + Case( + ("https://bucket.vpce-1a2b3c4d-5e6f.s3.us-east-1." + "vpce.amazonaws.com", None), + { + "s3_prefix": "bucket.vpce-1a2b3c4d-5e6f.s3.", + "domain_suffix": "vpce.amazonaws.com", + "region": "us-east-1", + "dualstack": False, + }, + ), + Case( + ("https://accesspoint.vpce-1a2b3c4d-5e6f.s3.us-east-1." + "vpce.amazonaws.com", None), + { + "s3_prefix": "accesspoint.vpce-1a2b3c4d-5e6f.s3.", + "domain_suffix": "vpce.amazonaws.com", + "region": "us-east-1", + "dualstack": False, + }, + ), + ### + Case( + ("https://012345678901.s3-control.amazonaws.com", None), + { + "s3_prefix": "012345678901.s3-control.", + "domain_suffix": "amazonaws.com", + "region": None, + "dualstack": False, + }, + ), + Case( + ("https://012345678901.s3-control.amazonaws.com", + "ap-south-1a"), + { + "s3_prefix": "012345678901.s3-control.", + "domain_suffix": "amazonaws.com", + "region": "ap-south-1a", + "dualstack": False, + }, + ), + Case( + ("https://012345678901.s3-control.us-gov-east-1.amazonaws.com", + None), + { + "s3_prefix": "012345678901.s3-control.", + "domain_suffix": "amazonaws.com", + "region": "us-gov-east-1", + "dualstack": False, + }, + ), + Case( + ("https://012345678901.s3-control.us-gov-east-1.amazonaws.com", + "cn-northwest-1"), + { + "s3_prefix": "012345678901.s3-control.", + "domain_suffix": "amazonaws.com", + "region": "cn-northwest-1", + "dualstack": False, + }, + ), + ### + Case( + ("https://012345678901.s3-control.dualstack.amazonaws.com", + None), + { + "s3_prefix": "012345678901.s3-control.", + "domain_suffix": "amazonaws.com", + "region": None, + "dualstack": True, + }, + ), + Case( + ("https://012345678901.s3-control.dualstack.amazonaws.com", + "ap-south-1a"), + { + "s3_prefix": "012345678901.s3-control.", + "domain_suffix": "amazonaws.com", + "region": "ap-south-1a", + "dualstack": True, + }, + ), + Case( + ("https://012345678901.s3-control.dualstack.us-gov-east-1." + "amazonaws.com", + None), + { + "s3_prefix": "012345678901.s3-control.", + "domain_suffix": "amazonaws.com", + "region": "us-gov-east-1", + "dualstack": True, + }, + ), + Case( + ("https://012345678901.s3-control.dualstack.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + { + "s3_prefix": "012345678901.s3-control.", + "domain_suffix": "amazonaws.com", + "region": "cn-northwest-1", + "dualstack": True, + }, + ), + ### + Case( + ("https://012345678901.s3-control-fips.amazonaws.com", None), + { + "s3_prefix": "012345678901.s3-control-fips.", + "domain_suffix": "amazonaws.com", + "region": None, + "dualstack": False, + }, + ), + Case( + ("https://012345678901.s3-control-fips.amazonaws.com", + "ap-south-1a"), + { + "s3_prefix": "012345678901.s3-control-fips.", + "domain_suffix": "amazonaws.com", + "region": "ap-south-1a", + "dualstack": False, + }, + ), + Case( + ("https://012345678901.s3-control-fips.us-gov-east-1." + "amazonaws.com", + None), + { + "s3_prefix": "012345678901.s3-control-fips.", + "domain_suffix": "amazonaws.com", + "region": "us-gov-east-1", + "dualstack": False, + }, + ), + Case( + ("https://012345678901.s3-control-fips.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + { + "s3_prefix": "012345678901.s3-control-fips.", + "domain_suffix": "amazonaws.com", + "region": "cn-northwest-1", + "dualstack": False, + }, + ), + ### + Case( + ("https://012345678901.s3-control-fips.dualstack.amazonaws.com", + None), + { + "s3_prefix": "012345678901.s3-control-fips.", + "domain_suffix": "amazonaws.com", + "region": None, + "dualstack": True, + }, + ), + Case( + ("https://012345678901.s3-control-fips.dualstack.amazonaws.com", + "ap-south-1a"), + { + "s3_prefix": "012345678901.s3-control-fips.", + "domain_suffix": "amazonaws.com", + "region": "ap-south-1a", + "dualstack": True, + }, + ), + Case( + ("https://012345678901.s3-control-fips.dualstack.us-gov-east-1." + "amazonaws.com", + None), + { + "s3_prefix": "012345678901.s3-control-fips.", + "domain_suffix": "amazonaws.com", + "region": "us-gov-east-1", + "dualstack": True, + }, + ), + Case( + ("https://012345678901.s3-control-fips.dualstack.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + { + "s3_prefix": "012345678901.s3-control-fips.", + "domain_suffix": "amazonaws.com", + "region": "cn-northwest-1", + "dualstack": True, + }, + ), + ### + Case( + ("https://s3-accesspoint.amazonaws.com", None), + { + "s3_prefix": "s3-accesspoint.", + "domain_suffix": "amazonaws.com", + "region": None, + "dualstack": False, + }, + ), + Case( + ("https://s3-accesspoint.amazonaws.com", + "ap-south-1a"), + { + "s3_prefix": "s3-accesspoint.", + "domain_suffix": "amazonaws.com", + "region": "ap-south-1a", + "dualstack": False, + }, + ), + Case( + ("https://s3-accesspoint.us-gov-east-1.amazonaws.com", + None), + { + "s3_prefix": "s3-accesspoint.", + "domain_suffix": "amazonaws.com", + "region": "us-gov-east-1", + "dualstack": False, + }, + ), + Case( + ("https://s3-accesspoint.us-gov-east-1.amazonaws.com", + "cn-northwest-1"), + { + "s3_prefix": "s3-accesspoint.", + "domain_suffix": "amazonaws.com", + "region": "cn-northwest-1", + "dualstack": False, + }, + ), + ### + Case( + ("https://s3-accesspoint.dualstack.amazonaws.com", + None), + { + "s3_prefix": "s3-accesspoint.", + "domain_suffix": "amazonaws.com", + "region": None, + "dualstack": True, + }, + ), + Case( + ("https://s3-accesspoint.dualstack.amazonaws.com", + "ap-south-1a"), + { + "s3_prefix": "s3-accesspoint.", + "domain_suffix": "amazonaws.com", + "region": "ap-south-1a", + "dualstack": True, + }, + ), + Case( + ("https://s3-accesspoint.dualstack.us-gov-east-1." + "amazonaws.com", + None), + { + "s3_prefix": "s3-accesspoint.", + "domain_suffix": "amazonaws.com", + "region": "us-gov-east-1", + "dualstack": True, + }, + ), + Case( + ("https://s3-accesspoint.dualstack.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + { + "s3_prefix": "s3-accesspoint.", + "domain_suffix": "amazonaws.com", + "region": "cn-northwest-1", + "dualstack": True, + }, + ), + ### + Case( + ("https://s3-accesspoint-fips.amazonaws.com", None), + { + "s3_prefix": "s3-accesspoint-fips.", + "domain_suffix": "amazonaws.com", + "region": None, + "dualstack": False, + }, + ), + Case( + ("https://s3-accesspoint-fips.amazonaws.com", + "ap-south-1a"), + { + "s3_prefix": "s3-accesspoint-fips.", + "domain_suffix": "amazonaws.com", + "region": "ap-south-1a", + "dualstack": False, + }, + ), + Case( + ("https://s3-accesspoint-fips.us-gov-east-1." + "amazonaws.com", + None), + { + "s3_prefix": "s3-accesspoint-fips.", + "domain_suffix": "amazonaws.com", + "region": "us-gov-east-1", + "dualstack": False, + }, + ), + Case( + ("https://s3-accesspoint-fips.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + { + "s3_prefix": "s3-accesspoint-fips.", + "domain_suffix": "amazonaws.com", + "region": "cn-northwest-1", + "dualstack": False, + }, + ), + ### + Case( + ("https://s3-accesspoint-fips.dualstack.amazonaws.com", + None), + { + "s3_prefix": "s3-accesspoint-fips.", + "domain_suffix": "amazonaws.com", + "region": None, + "dualstack": True, + }, + ), + Case( + ("https://s3-accesspoint-fips.dualstack.amazonaws.com", + "ap-south-1a"), + { + "s3_prefix": "s3-accesspoint-fips.", + "domain_suffix": "amazonaws.com", + "region": "ap-south-1a", + "dualstack": True, + }, + ), + Case( + ("https://s3-accesspoint-fips.dualstack.us-gov-east-1." + "amazonaws.com", + None), + { + "s3_prefix": "s3-accesspoint-fips.", + "domain_suffix": "amazonaws.com", + "region": "us-gov-east-1", + "dualstack": True, + }, + ), + Case( + ("https://s3-accesspoint-fips.dualstack.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + { + "s3_prefix": "s3-accesspoint-fips.", + "domain_suffix": "amazonaws.com", + "region": "cn-northwest-1", + "dualstack": True, + }, + ), + ### + Case( + ("https://my-load-balancer-1234567890.us-west-2.elb." + "amazonaws.com", "us-west-2"), + None, + ), + ] + + for case in cases: + url = BaseURL(*case.args) + self.assertEqual(url._aws_info, case.result) + + def test_aws_list_buckets_build(self): + Case = namedtuple("Case", ["args", "result"]) + cases = [ + Case( + ("https://s3.amazonaws.com", None), + "https://s3.us-east-1.amazonaws.com/", + ), + Case( + ("https://s3.amazonaws.com", "ap-south-1a"), + "https://s3.ap-south-1a.amazonaws.com/", + ), + Case( + ("https://s3.us-gov-east-1.amazonaws.com", None), + "https://s3.us-gov-east-1.amazonaws.com/", + ), + Case( + ("https://s3.me-south-1.amazonaws.com", "cn-northwest-1"), + "https://s3.cn-northwest-1.amazonaws.com/", + ), + ### + Case( + ("https://s3.dualstack.amazonaws.com", None), + "https://s3.us-east-1.amazonaws.com/", + ), + Case( + ("https://s3.dualstack.amazonaws.com", "ap-south-1a"), + "https://s3.ap-south-1a.amazonaws.com/", + ), + Case( + ("https://s3.dualstack.us-gov-east-1.amazonaws.com", None), + "https://s3.us-gov-east-1.amazonaws.com/", + ), + Case( + ("https://s3.dualstack.me-south-1.amazonaws.com", + "cn-northwest-1"), + "https://s3.cn-northwest-1.amazonaws.com/", + ), + ### + Case( + ("https://s3-accelerate.amazonaws.com", None), + "https://s3.us-east-1.amazonaws.com/", + ), + Case( + ("https://s3-accelerate.amazonaws.com", "ap-south-1a"), + "https://s3.ap-south-1a.amazonaws.com/", + ), + Case( + ("https://s3-accelerate.us-gov-east-1.amazonaws.com", None), + "https://s3.us-gov-east-1.amazonaws.com/", + ), + Case( + ("https://s3-accelerate.me-south-1.amazonaws.com", + "cn-northwest-1"), + "https://s3.cn-northwest-1.amazonaws.com/", + ), + ### + Case( + ("https://s3-accelerate.dualstack.amazonaws.com", None), + "https://s3.us-east-1.amazonaws.com/", + ), + Case( + ("https://s3-accelerate.dualstack.amazonaws.com", + "ap-south-1a"), + "https://s3.ap-south-1a.amazonaws.com/", + ), + Case( + ("https://s3-accelerate.dualstack.us-gov-east-1.amazonaws.com", + None), + "https://s3.us-gov-east-1.amazonaws.com/", + ), + Case( + ("https://s3-accelerate.dualstack.me-south-1.amazonaws.com", + "cn-northwest-1"), + "https://s3.cn-northwest-1.amazonaws.com/", + ), + ### + Case( + ("https://s3-fips.amazonaws.com", None), + "https://s3.us-east-1.amazonaws.com/", + ), + Case( + ("https://s3-fips.amazonaws.com", "ap-south-1a"), + "https://s3.ap-south-1a.amazonaws.com/", + ), + Case( + ("https://s3-fips.us-gov-east-1.amazonaws.com", None), + "https://s3.us-gov-east-1.amazonaws.com/", + ), + Case( + ("https://s3-fips.me-south-1.amazonaws.com", "cn-northwest-1"), + "https://s3.cn-northwest-1.amazonaws.com/", + ), + ### + Case( + ("https://s3-fips.dualstack.amazonaws.com", None), + "https://s3.us-east-1.amazonaws.com/", + ), + Case( + ("https://s3-fips.dualstack.amazonaws.com", "ap-south-1a"), + "https://s3.ap-south-1a.amazonaws.com/", + ), + Case( + ("https://s3-fips.dualstack.us-gov-east-1.amazonaws.com", None), + "https://s3.us-gov-east-1.amazonaws.com/", + ), + Case( + ("https://s3-fips.dualstack.me-south-1.amazonaws.com", + "cn-northwest-1"), + "https://s3.cn-northwest-1.amazonaws.com/", + ), + ### + Case( + ("https://s3-external-1.amazonaws.com", None), + "https://s3-external-1.amazonaws.com/", + ), + Case( + ("https://s3-us-gov-west-1.amazonaws.com", None), + "https://s3-us-gov-west-1.amazonaws.com/", + ), + Case( + ("https://s3-fips-us-gov-west-1.amazonaws.com", None), + "https://s3-fips-us-gov-west-1.amazonaws.com/", + ), + ### + Case( + ("https://bucket.vpce-1a2b3c4d-5e6f.s3.us-east-1." + "vpce.amazonaws.com", None), + "https://bucket.vpce-1a2b3c4d-5e6f.s3.us-east-1." + "vpce.amazonaws.com/", + ), + Case( + ("https://accesspoint.vpce-1a2b3c4d-5e6f.s3.us-east-1." + "vpce.amazonaws.com", None), + "https://accesspoint.vpce-1a2b3c4d-5e6f.s3.us-east-1." + "vpce.amazonaws.com/", + ), + ### + Case( + ("https://012345678901.s3-control.amazonaws.com", None), + "https://012345678901.s3-control.us-east-1.amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control.amazonaws.com", + "ap-south-1a"), + "https://012345678901.s3-control.ap-south-1a.amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control.us-gov-east-1.amazonaws.com", + None), + "https://012345678901.s3-control.us-gov-east-1.amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control.us-gov-east-1.amazonaws.com", + "cn-northwest-1"), + "https://012345678901.s3-control.cn-northwest-1.amazonaws.com/", + ), + ### + Case( + ("https://012345678901.s3-control.dualstack.amazonaws.com", + None), + "https://012345678901.s3-control.us-east-1." + "amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control.dualstack.amazonaws.com", + "ap-south-1a"), + "https://012345678901.s3-control.ap-south-1a." + "amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control.dualstack.us-gov-east-1." + "amazonaws.com", + None), + "https://012345678901.s3-control.us-gov-east-1." + "amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control.dualstack.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + "https://012345678901.s3-control.cn-northwest-1." + "amazonaws.com/", + ), + ### + Case( + ("https://012345678901.s3-control-fips.amazonaws.com", None), + "https://012345678901.s3-control-fips.us-east-1.amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control-fips.amazonaws.com", + "ap-south-1a"), + "https://012345678901.s3-control-fips.ap-south-1a." + "amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control-fips.us-gov-east-1." + "amazonaws.com", + None), + "https://012345678901.s3-control-fips.us-gov-east-1." + "amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control-fips.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + "https://012345678901.s3-control-fips.cn-northwest-1." + "amazonaws.com/", + ), + ### + Case( + ("https://012345678901.s3-control-fips.dualstack.amazonaws.com", + None), + "https://012345678901.s3-control-fips.us-east-1." + "amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control-fips.dualstack.amazonaws.com", + "ap-south-1a"), + "https://012345678901.s3-control-fips.ap-south-1a." + "amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control-fips.dualstack.us-gov-east-1." + "amazonaws.com", + None), + "https://012345678901.s3-control-fips.us-gov-east-1." + "amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control-fips.dualstack.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + "https://012345678901.s3-control-fips.cn-northwest-1." + "amazonaws.com/", + ), + ### + Case( + ("https://s3-accesspoint.amazonaws.com", None), + "https://s3.us-east-1.amazonaws.com/", + ), + Case( + ("https://s3-accesspoint.amazonaws.com", + "ap-south-1a"), + "https://s3.ap-south-1a.amazonaws.com/", + ), + Case( + ("https://s3-accesspoint.us-gov-east-1.amazonaws.com", + None), + "https://s3.us-gov-east-1.amazonaws.com/", + ), + Case( + ("https://s3-accesspoint.us-gov-east-1.amazonaws.com", + "cn-northwest-1"), + "https://s3.cn-northwest-1.amazonaws.com/", + ), + ### + Case( + ("https://s3-accesspoint.dualstack.amazonaws.com", + None), + "https://s3.us-east-1.amazonaws.com/", + ), + Case( + ("https://s3-accesspoint.dualstack.amazonaws.com", + "ap-south-1a"), + "https://s3.ap-south-1a.amazonaws.com/", + ), + Case( + ("https://s3-accesspoint.dualstack.us-gov-east-1." + "amazonaws.com", + None), + "https://s3.us-gov-east-1.amazonaws.com/", + ), + Case( + ("https://s3-accesspoint.dualstack.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + "https://s3.cn-northwest-1.amazonaws.com/", + ), + ### + Case( + ("https://s3-accesspoint-fips.amazonaws.com", None), + "https://s3.us-east-1.amazonaws.com/", + ), + Case( + ("https://s3-accesspoint-fips.amazonaws.com", + "ap-south-1a"), + "https://s3.ap-south-1a.amazonaws.com/", + ), + Case( + ("https://s3-accesspoint-fips.us-gov-east-1." + "amazonaws.com", + None), + "https://s3.us-gov-east-1.amazonaws.com/", + ), + Case( + ("https://s3-accesspoint-fips.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + "https://s3.cn-northwest-1.amazonaws.com/", + ), + ### + Case( + ("https://s3-accesspoint-fips.dualstack.amazonaws.com", + None), + "https://s3.us-east-1.amazonaws.com/", + ), + Case( + ("https://s3-accesspoint-fips.dualstack.amazonaws.com", + "ap-south-1a"), + "https://s3.ap-south-1a.amazonaws.com/", + ), + Case( + ("https://s3-accesspoint-fips.dualstack.us-gov-east-1." + "amazonaws.com", + None), + "https://s3.us-gov-east-1.amazonaws.com/", + ), + Case( + ("https://s3-accesspoint-fips.dualstack.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + "https://s3.cn-northwest-1.amazonaws.com/", + ), + ### + Case( + ("https://my-load-balancer-1234567890.us-west-2.elb." + "amazonaws.com", "us-west-2"), + "https://my-load-balancer-1234567890.us-west-2.elb." + "amazonaws.com/", + ), + ] + + for case in cases: + base_url = BaseURL(*case.args) + url = urlunsplit(base_url.build( + "GET", base_url.region or "us-east-1")) + self.assertEqual(str(url), case.result) + + def test_aws_bucket_build(self): + Case = namedtuple("Case", ["args", "result"]) + cases = [ + Case( + ("https://s3.amazonaws.com", None), + "https://my-bucket.s3.us-east-1.amazonaws.com/", + ), + Case( + ("https://s3.amazonaws.com", "ap-south-1a"), + "https://my-bucket.s3.ap-south-1a.amazonaws.com/", + ), + Case( + ("https://s3.us-gov-east-1.amazonaws.com", None), + "https://my-bucket.s3.us-gov-east-1.amazonaws.com/", + ), + Case( + ("https://s3.me-south-1.amazonaws.com", "cn-northwest-1"), + "https://my-bucket.s3.cn-northwest-1.amazonaws.com/", + ), + ### + Case( + ("https://s3.dualstack.amazonaws.com", None), + "https://my-bucket.s3.dualstack.us-east-1.amazonaws.com/", + ), + Case( + ("https://s3.dualstack.amazonaws.com", "ap-south-1a"), + "https://my-bucket.s3.dualstack.ap-south-1a.amazonaws.com/", + ), + Case( + ("https://s3.dualstack.us-gov-east-1.amazonaws.com", None), + "https://my-bucket.s3.dualstack.us-gov-east-1.amazonaws.com/", + ), + Case( + ("https://s3.dualstack.me-south-1.amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.s3.dualstack.cn-northwest-1.amazonaws.com/", + ), + ### + Case( + ("https://s3-accelerate.amazonaws.com", None), + "https://my-bucket.s3-accelerate.amazonaws.com/", + ), + Case( + ("https://s3-accelerate.amazonaws.com", "ap-south-1a"), + "https://my-bucket.s3-accelerate.amazonaws.com/", + ), + Case( + ("https://s3-accelerate.us-gov-east-1.amazonaws.com", None), + "https://my-bucket.s3-accelerate.amazonaws.com/", + ), + Case( + ("https://s3-accelerate.me-south-1.amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.s3-accelerate.amazonaws.com/", + ), + ### + Case( + ("https://s3-accelerate.dualstack.amazonaws.com", None), + "https://my-bucket.s3-accelerate.dualstack.amazonaws.com/", + ), + Case( + ("https://s3-accelerate.dualstack.amazonaws.com", + "ap-south-1a"), + "https://my-bucket.s3-accelerate.dualstack.amazonaws.com/", + ), + Case( + ("https://s3-accelerate.dualstack.us-gov-east-1.amazonaws.com", + None), + "https://my-bucket.s3-accelerate.dualstack.amazonaws.com/", + ), + Case( + ("https://s3-accelerate.dualstack.me-south-1.amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.s3-accelerate.dualstack.amazonaws.com/", + ), + ### + Case( + ("https://s3-fips.amazonaws.com", None), + "https://my-bucket.s3-fips.us-east-1.amazonaws.com/", + ), + Case( + ("https://s3-fips.amazonaws.com", "ap-south-1a"), + "https://my-bucket.s3-fips.ap-south-1a.amazonaws.com/", + ), + Case( + ("https://s3-fips.us-gov-east-1.amazonaws.com", None), + "https://my-bucket.s3-fips.us-gov-east-1.amazonaws.com/", + ), + Case( + ("https://s3-fips.me-south-1.amazonaws.com", "cn-northwest-1"), + "https://my-bucket.s3-fips.cn-northwest-1.amazonaws.com/", + ), + ### + Case( + ("https://s3-fips.dualstack.amazonaws.com", None), + "https://my-bucket.s3-fips.dualstack.us-east-1.amazonaws.com/", + ), + Case( + ("https://s3-fips.dualstack.amazonaws.com", "ap-south-1a"), + "https://my-bucket.s3-fips.dualstack.ap-south-1a." + "amazonaws.com/", + ), + Case( + ("https://s3-fips.dualstack.us-gov-east-1.amazonaws.com", None), + "https://my-bucket.s3-fips.dualstack.us-gov-east-1." + "amazonaws.com/", + ), + Case( + ("https://s3-fips.dualstack.me-south-1.amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.s3-fips.dualstack.cn-northwest-1." + "amazonaws.com/", + ), + ### + Case( + ("https://s3-external-1.amazonaws.com", None), + "https://my-bucket.s3-external-1.amazonaws.com/", + ), + Case( + ("https://s3-us-gov-west-1.amazonaws.com", None), + "https://my-bucket.s3-us-gov-west-1.amazonaws.com/", + ), + Case( + ("https://s3-fips-us-gov-west-1.amazonaws.com", None), + "https://my-bucket.s3-fips-us-gov-west-1.amazonaws.com/", + ), + ### + Case( + ("https://bucket.vpce-1a2b3c4d-5e6f.s3.us-east-1." + "vpce.amazonaws.com", None), + "https://my-bucket.bucket.vpce-1a2b3c4d-5e6f.s3.us-east-1." + "vpce.amazonaws.com/", + ), + Case( + ("https://accesspoint.vpce-1a2b3c4d-5e6f.s3.us-east-1." + "vpce.amazonaws.com", None), + "https://my-bucket.accesspoint.vpce-1a2b3c4d-5e6f.s3.us-east-1." + "vpce.amazonaws.com/", + ), + ### + Case( + ("https://012345678901.s3-control.amazonaws.com", None), + "https://my-bucket.012345678901.s3-control.us-east-1." + "amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control.amazonaws.com", + "ap-south-1a"), + "https://my-bucket.012345678901.s3-control.ap-south-1a." + "amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control.us-gov-east-1.amazonaws.com", + None), + "https://my-bucket.012345678901.s3-control.us-gov-east-1." + "amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control.us-gov-east-1.amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.012345678901.s3-control.cn-northwest-1." + "amazonaws.com/", + ), + ### + Case( + ("https://012345678901.s3-control.dualstack.amazonaws.com", + None), + "https://my-bucket.012345678901.s3-control.dualstack.us-east-1." + "amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control.dualstack.amazonaws.com", + "ap-south-1a"), + "https://my-bucket.012345678901.s3-control.dualstack." + "ap-south-1a.amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control.dualstack.us-gov-east-1." + "amazonaws.com", + None), + "https://my-bucket.012345678901.s3-control.dualstack." + "us-gov-east-1.amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control.dualstack.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.012345678901.s3-control.dualstack." + "cn-northwest-1.amazonaws.com/", + ), + ### + Case( + ("https://012345678901.s3-control-fips.amazonaws.com", None), + "https://my-bucket.012345678901.s3-control-fips.us-east-1." + "amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control-fips.amazonaws.com", + "ap-south-1a"), + "https://my-bucket.012345678901.s3-control-fips.ap-south-1a." + "amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control-fips.us-gov-east-1." + "amazonaws.com", + None), + "https://my-bucket.012345678901.s3-control-fips.us-gov-east-1." + "amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control-fips.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.012345678901.s3-control-fips.cn-northwest-1." + "amazonaws.com/", + ), + ### + Case( + ("https://012345678901.s3-control-fips.dualstack.amazonaws.com", + None), + "https://my-bucket.012345678901.s3-control-fips.dualstack." + "us-east-1.amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control-fips.dualstack.amazonaws.com", + "ap-south-1a"), + "https://my-bucket.012345678901.s3-control-fips.dualstack." + "ap-south-1a.amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control-fips.dualstack.us-gov-east-1." + "amazonaws.com", + None), + "https://my-bucket.012345678901.s3-control-fips.dualstack." + "us-gov-east-1.amazonaws.com/", + ), + Case( + ("https://012345678901.s3-control-fips.dualstack.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.012345678901.s3-control-fips.dualstack." + "cn-northwest-1.amazonaws.com/", + ), + ### + Case( + ("https://s3-accesspoint.amazonaws.com", None), + "https://my-bucket.s3-accesspoint.us-east-1.amazonaws.com/", + ), + Case( + ("https://s3-accesspoint.amazonaws.com", + "ap-south-1a"), + "https://my-bucket.s3-accesspoint.ap-south-1a.amazonaws.com/", + ), + Case( + ("https://s3-accesspoint.us-gov-east-1.amazonaws.com", + None), + "https://my-bucket.s3-accesspoint.us-gov-east-1.amazonaws.com/", + ), + Case( + ("https://s3-accesspoint.us-gov-east-1.amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.s3-accesspoint.cn-northwest-1." + "amazonaws.com/", + ), + ### + Case( + ("https://s3-accesspoint.dualstack.amazonaws.com", + None), + "https://my-bucket.s3-accesspoint.dualstack.us-east-1." + "amazonaws.com/", + ), + Case( + ("https://s3-accesspoint.dualstack.amazonaws.com", + "ap-south-1a"), + "https://my-bucket.s3-accesspoint.dualstack.ap-south-1a." + "amazonaws.com/", + ), + Case( + ("https://s3-accesspoint.dualstack.us-gov-east-1." + "amazonaws.com", + None), + "https://my-bucket.s3-accesspoint.dualstack.us-gov-east-1." + "amazonaws.com/", + ), + Case( + ("https://s3-accesspoint.dualstack.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.s3-accesspoint.dualstack.cn-northwest-1." + "amazonaws.com/", + ), + ### + Case( + ("https://s3-accesspoint-fips.amazonaws.com", None), + "https://my-bucket.s3-accesspoint-fips.us-east-1." + "amazonaws.com/", + ), + Case( + ("https://s3-accesspoint-fips.amazonaws.com", + "ap-south-1a"), + "https://my-bucket.s3-accesspoint-fips.ap-south-1a." + "amazonaws.com/", + ), + Case( + ("https://s3-accesspoint-fips.us-gov-east-1." + "amazonaws.com", + None), + "https://my-bucket.s3-accesspoint-fips.us-gov-east-1." + "amazonaws.com/", + ), + Case( + ("https://s3-accesspoint-fips.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.s3-accesspoint-fips.cn-northwest-1." + "amazonaws.com/", + ), + ### + Case( + ("https://s3-accesspoint-fips.dualstack.amazonaws.com", + None), + "https://my-bucket.s3-accesspoint-fips.dualstack.us-east-1." + "amazonaws.com/", + ), + Case( + ("https://s3-accesspoint-fips.dualstack.amazonaws.com", + "ap-south-1a"), + "https://my-bucket.s3-accesspoint-fips.dualstack.ap-south-1a." + "amazonaws.com/", + ), + Case( + ("https://s3-accesspoint-fips.dualstack.us-gov-east-1." + "amazonaws.com", + None), + "https://my-bucket.s3-accesspoint-fips.dualstack.us-gov-east-1." + "amazonaws.com/", + ), + Case( + ("https://s3-accesspoint-fips.dualstack.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.s3-accesspoint-fips.dualstack." + "cn-northwest-1.amazonaws.com/", + ), + ### + Case( + ("https://my-load-balancer-1234567890.us-west-2.elb." + "amazonaws.com", "us-west-2"), + "https://my-load-balancer-1234567890.us-west-2.elb." + "amazonaws.com/my-bucket", + ), + ] + + for case in cases: + base_url = BaseURL(*case.args) + url = urlunsplit(base_url.build( + "GET", base_url.region or "us-east-1", bucket_name="my-bucket")) + self.assertEqual(str(url), case.result) + + def test_aws_object_build(self): + Case = namedtuple("Case", ["args", "result"]) + cases = [ + Case( + ("https://s3.amazonaws.com", None), + "https://my-bucket.s3.us-east-1.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3.amazonaws.com", "ap-south-1a"), + "https://my-bucket.s3.ap-south-1a.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3.us-gov-east-1.amazonaws.com", None), + "https://my-bucket.s3.us-gov-east-1.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3.me-south-1.amazonaws.com", "cn-northwest-1"), + "https://my-bucket.s3.cn-northwest-1.amazonaws.com/" + "path/to/my/object", + ), + ### + Case( + ("https://s3.dualstack.amazonaws.com", None), + "https://my-bucket.s3.dualstack.us-east-1.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3.dualstack.amazonaws.com", "ap-south-1a"), + "https://my-bucket.s3.dualstack.ap-south-1a.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3.dualstack.us-gov-east-1.amazonaws.com", None), + "https://my-bucket.s3.dualstack.us-gov-east-1.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3.dualstack.me-south-1.amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.s3.dualstack.cn-northwest-1.amazonaws.com/" + "path/to/my/object", + ), + ### + Case( + ("https://s3-accelerate.amazonaws.com", None), + "https://my-bucket.s3-accelerate.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3-accelerate.amazonaws.com", "ap-south-1a"), + "https://my-bucket.s3-accelerate.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3-accelerate.us-gov-east-1.amazonaws.com", None), + "https://my-bucket.s3-accelerate.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3-accelerate.me-south-1.amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.s3-accelerate.amazonaws.com/" + "path/to/my/object", + ), + ### + Case( + ("https://s3-accelerate.dualstack.amazonaws.com", None), + "https://my-bucket.s3-accelerate.dualstack.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3-accelerate.dualstack.amazonaws.com", + "ap-south-1a"), + "https://my-bucket.s3-accelerate.dualstack.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3-accelerate.dualstack.us-gov-east-1.amazonaws.com", + None), + "https://my-bucket.s3-accelerate.dualstack.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3-accelerate.dualstack.me-south-1.amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.s3-accelerate.dualstack.amazonaws.com/" + "path/to/my/object", + ), + ### + Case( + ("https://s3-fips.amazonaws.com", None), + "https://my-bucket.s3-fips.us-east-1.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3-fips.amazonaws.com", "ap-south-1a"), + "https://my-bucket.s3-fips.ap-south-1a.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3-fips.us-gov-east-1.amazonaws.com", None), + "https://my-bucket.s3-fips.us-gov-east-1.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3-fips.me-south-1.amazonaws.com", "cn-northwest-1"), + "https://my-bucket.s3-fips.cn-northwest-1.amazonaws.com/" + "path/to/my/object", + ), + ### + Case( + ("https://s3-fips.dualstack.amazonaws.com", None), + "https://my-bucket.s3-fips.dualstack.us-east-1.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3-fips.dualstack.amazonaws.com", "ap-south-1a"), + "https://my-bucket.s3-fips.dualstack.ap-south-1a." + "amazonaws.com/path/to/my/object", + ), + Case( + ("https://s3-fips.dualstack.us-gov-east-1.amazonaws.com", None), + "https://my-bucket.s3-fips.dualstack.us-gov-east-1." + "amazonaws.com/path/to/my/object", + ), + Case( + ("https://s3-fips.dualstack.me-south-1.amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.s3-fips.dualstack.cn-northwest-1." + "amazonaws.com/path/to/my/object", + ), + ### + Case( + ("https://s3-external-1.amazonaws.com", None), + "https://my-bucket.s3-external-1.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3-us-gov-west-1.amazonaws.com", None), + "https://my-bucket.s3-us-gov-west-1.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3-fips-us-gov-west-1.amazonaws.com", None), + "https://my-bucket.s3-fips-us-gov-west-1.amazonaws.com/" + "path/to/my/object", + ), + ### + Case( + ("https://bucket.vpce-1a2b3c4d-5e6f.s3.us-east-1." + "vpce.amazonaws.com", None), + "https://my-bucket.bucket.vpce-1a2b3c4d-5e6f.s3.us-east-1." + "vpce.amazonaws.com/path/to/my/object", + ), + Case( + ("https://accesspoint.vpce-1a2b3c4d-5e6f.s3.us-east-1." + "vpce.amazonaws.com", None), + "https://my-bucket.accesspoint.vpce-1a2b3c4d-5e6f.s3.us-east-1." + "vpce.amazonaws.com/path/to/my/object", + ), + ### + Case( + ("https://012345678901.s3-control.amazonaws.com", None), + "https://my-bucket.012345678901.s3-control.us-east-1." + "amazonaws.com/path/to/my/object", + ), + Case( + ("https://012345678901.s3-control.amazonaws.com", + "ap-south-1a"), + "https://my-bucket.012345678901.s3-control.ap-south-1a." + "amazonaws.com/path/to/my/object", + ), + Case( + ("https://012345678901.s3-control.us-gov-east-1.amazonaws.com", + None), + "https://my-bucket.012345678901.s3-control.us-gov-east-1." + "amazonaws.com/path/to/my/object", + ), + Case( + ("https://012345678901.s3-control.us-gov-east-1.amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.012345678901.s3-control.cn-northwest-1." + "amazonaws.com/path/to/my/object", + ), + ### + Case( + ("https://012345678901.s3-control.dualstack.amazonaws.com", + None), + "https://my-bucket.012345678901.s3-control.dualstack.us-east-1." + "amazonaws.com/path/to/my/object", + ), + Case( + ("https://012345678901.s3-control.dualstack.amazonaws.com", + "ap-south-1a"), + "https://my-bucket.012345678901.s3-control.dualstack." + "ap-south-1a.amazonaws.com/path/to/my/object", + ), + Case( + ("https://012345678901.s3-control.dualstack.us-gov-east-1." + "amazonaws.com", + None), + "https://my-bucket.012345678901.s3-control.dualstack." + "us-gov-east-1.amazonaws.com/path/to/my/object", + ), + Case( + ("https://012345678901.s3-control.dualstack.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.012345678901.s3-control.dualstack." + "cn-northwest-1.amazonaws.com/path/to/my/object", + ), + ### + Case( + ("https://012345678901.s3-control-fips.amazonaws.com", None), + "https://my-bucket.012345678901.s3-control-fips.us-east-1." + "amazonaws.com/path/to/my/object", + ), + Case( + ("https://012345678901.s3-control-fips.amazonaws.com", + "ap-south-1a"), + "https://my-bucket.012345678901.s3-control-fips.ap-south-1a." + "amazonaws.com/path/to/my/object", + ), + Case( + ("https://012345678901.s3-control-fips.us-gov-east-1." + "amazonaws.com", + None), + "https://my-bucket.012345678901.s3-control-fips.us-gov-east-1." + "amazonaws.com/path/to/my/object", + ), + Case( + ("https://012345678901.s3-control-fips.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.012345678901.s3-control-fips.cn-northwest-1." + "amazonaws.com/path/to/my/object", + ), + ### + Case( + ("https://012345678901.s3-control-fips.dualstack.amazonaws.com", + None), + "https://my-bucket.012345678901.s3-control-fips.dualstack." + "us-east-1.amazonaws.com/path/to/my/object", + ), + Case( + ("https://012345678901.s3-control-fips.dualstack.amazonaws.com", + "ap-south-1a"), + "https://my-bucket.012345678901.s3-control-fips.dualstack." + "ap-south-1a.amazonaws.com/path/to/my/object", + ), + Case( + ("https://012345678901.s3-control-fips.dualstack.us-gov-east-1." + "amazonaws.com", + None), + "https://my-bucket.012345678901.s3-control-fips.dualstack." + "us-gov-east-1.amazonaws.com/path/to/my/object", + ), + Case( + ("https://012345678901.s3-control-fips.dualstack.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.012345678901.s3-control-fips.dualstack." + "cn-northwest-1.amazonaws.com/path/to/my/object", + ), + ### + Case( + ("https://s3-accesspoint.amazonaws.com", None), + "https://my-bucket.s3-accesspoint.us-east-1.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3-accesspoint.amazonaws.com", + "ap-south-1a"), + "https://my-bucket.s3-accesspoint.ap-south-1a.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3-accesspoint.us-gov-east-1.amazonaws.com", + None), + "https://my-bucket.s3-accesspoint.us-gov-east-1.amazonaws.com/" + "path/to/my/object", + ), + Case( + ("https://s3-accesspoint.us-gov-east-1.amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.s3-accesspoint.cn-northwest-1." + "amazonaws.com/path/to/my/object", + ), + ### + Case( + ("https://s3-accesspoint.dualstack.amazonaws.com", + None), + "https://my-bucket.s3-accesspoint.dualstack.us-east-1." + "amazonaws.com/path/to/my/object", + ), + Case( + ("https://s3-accesspoint.dualstack.amazonaws.com", + "ap-south-1a"), + "https://my-bucket.s3-accesspoint.dualstack.ap-south-1a." + "amazonaws.com/path/to/my/object", + ), + Case( + ("https://s3-accesspoint.dualstack.us-gov-east-1." + "amazonaws.com", + None), + "https://my-bucket.s3-accesspoint.dualstack.us-gov-east-1." + "amazonaws.com/path/to/my/object", + ), + Case( + ("https://s3-accesspoint.dualstack.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.s3-accesspoint.dualstack.cn-northwest-1." + "amazonaws.com/path/to/my/object", + ), + ### + Case( + ("https://s3-accesspoint-fips.amazonaws.com", None), + "https://my-bucket.s3-accesspoint-fips.us-east-1." + "amazonaws.com/path/to/my/object", + ), + Case( + ("https://s3-accesspoint-fips.amazonaws.com", + "ap-south-1a"), + "https://my-bucket.s3-accesspoint-fips.ap-south-1a." + "amazonaws.com/path/to/my/object", + ), + Case( + ("https://s3-accesspoint-fips.us-gov-east-1." + "amazonaws.com", + None), + "https://my-bucket.s3-accesspoint-fips.us-gov-east-1." + "amazonaws.com/path/to/my/object", + ), + Case( + ("https://s3-accesspoint-fips.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.s3-accesspoint-fips.cn-northwest-1." + "amazonaws.com/path/to/my/object", + ), + ### + Case( + ("https://s3-accesspoint-fips.dualstack.amazonaws.com", + None), + "https://my-bucket.s3-accesspoint-fips.dualstack.us-east-1." + "amazonaws.com/path/to/my/object", + ), + Case( + ("https://s3-accesspoint-fips.dualstack.amazonaws.com", + "ap-south-1a"), + "https://my-bucket.s3-accesspoint-fips.dualstack.ap-south-1a." + "amazonaws.com/path/to/my/object", + ), + Case( + ("https://s3-accesspoint-fips.dualstack.us-gov-east-1." + "amazonaws.com", + None), + "https://my-bucket.s3-accesspoint-fips.dualstack.us-gov-east-1." + "amazonaws.com/path/to/my/object", + ), + Case( + ("https://s3-accesspoint-fips.dualstack.us-gov-east-1." + "amazonaws.com", + "cn-northwest-1"), + "https://my-bucket.s3-accesspoint-fips.dualstack." + "cn-northwest-1.amazonaws.com/path/to/my/object", + ), + ### + Case( + ("https://my-load-balancer-1234567890.us-west-2.elb." + "amazonaws.com", "us-west-2"), + "https://my-load-balancer-1234567890.us-west-2.elb." + "amazonaws.com/my-bucket/path/to/my/object", + ), + ] + + for case in cases: + base_url = BaseURL(*case.args) + url = urlunsplit(base_url.build( + "GET", base_url.region or "us-east-1", + bucket_name="my-bucket", object_name="path/to/my/object")) + self.assertEqual(str(url), case.result)