diff --git a/pkg/objstore/providers/s3/bucket_client.go b/pkg/objstore/providers/s3/bucket_client.go index 05630eddb3..5fca2d3eea 100644 --- a/pkg/objstore/providers/s3/bucket_client.go +++ b/pkg/objstore/providers/s3/bucket_client.go @@ -39,8 +39,10 @@ func newS3Config(cfg Config) (s3.Config, error) { } bucketLookupType := s3.AutoLookup - if cfg.ForcePathStyle { + if cfg.ForcePathStyle || cfg.BucketLookupType == PathStyleLookup { bucketLookupType = s3.PathLookup + } else if cfg.BucketLookupType == VirtualHostedStyleLookup { + bucketLookupType = s3.VirtualHostLookup } return s3.Config{ diff --git a/pkg/objstore/providers/s3/config.go b/pkg/objstore/providers/s3/config.go index 64f13f6601..c6307eb0b8 100644 --- a/pkg/objstore/providers/s3/config.go +++ b/pkg/objstore/providers/s3/config.go @@ -31,14 +31,25 @@ const ( // SSES3 config type constant to configure S3 server side encryption with AES-256 // https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html SSES3 = "SSE-S3" + + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access + PathStyleLookup = "path-style" + + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access + VirtualHostedStyleLookup = "virtual-hosted-style" + + AutoLookup = "auto" ) var ( supportedSignatureVersions = []string{SignatureVersionV4, SignatureVersionV2} supportedSSETypes = []string{SSEKMS, SSES3} + supportedBucketLookupTypes = []string{PathStyleLookup, VirtualHostedStyleLookup, AutoLookup} errUnsupportedSignatureVersion = errors.New("unsupported signature version") errUnsupportedSSEType = errors.New("unsupported S3 SSE type") errInvalidSSEContext = errors.New("invalid S3 SSE encryption context") + errBucketLookupConfigConflict = errors.New("cannot use s3.force-path-style = true and s3.bucket-lookup-type = virtual-hosted-style at the same time") + errUnsupportedBucketLookupType = errors.New("invalid S3 bucket lookup type") ) // HTTPConfig stores the http.Transport configuration for the s3 minio client. @@ -78,6 +89,7 @@ type Config struct { Insecure bool `yaml:"insecure" category:"advanced"` SignatureVersion string `yaml:"signature_version" category:"advanced"` ForcePathStyle bool `yaml:"force_path_style" category:"advanced"` + BucketLookupType string `yaml:"bucket_lookup_type" category:"advanced"` SSE SSEConfig `yaml:"sse"` HTTP HTTPConfig `yaml:"http"` @@ -96,7 +108,8 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { f.StringVar(&cfg.Region, prefix+"s3.region", "", "S3 region. If unset, the client will issue a S3 GetBucketLocation API call to autodetect it.") f.StringVar(&cfg.Endpoint, prefix+"s3.endpoint", "", "The S3 bucket endpoint. It could be an AWS S3 endpoint listed at https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an S3-compatible service in hostname:port format.") f.BoolVar(&cfg.Insecure, prefix+"s3.insecure", false, "If enabled, use http:// for the S3 endpoint instead of https://. This could be useful in local dev/test environments while using an S3-compatible backend storage, like Minio.") - f.BoolVar(&cfg.ForcePathStyle, prefix+"s3.force-path-style", false, "Set this to `true` to force the bucket lookup to be using path-style.") + f.BoolVar(&cfg.ForcePathStyle, prefix+"s3.force-path-style", false, "Deprecated, use s3.bucket-lookup-type instead. Set this to `true` to force the bucket lookup to be using path-style.") + f.StringVar(&cfg.BucketLookupType, prefix+"s3.bucket-lookup-type", AutoLookup, fmt.Sprintf("S3 bucket lookup style, use one of: %v", supportedBucketLookupTypes)) f.StringVar(&cfg.SignatureVersion, prefix+"s3.signature-version", SignatureVersionV4, fmt.Sprintf("The signature version to use for authenticating against S3. Supported values are: %s.", strings.Join(supportedSignatureVersions, ", "))) cfg.SSE.RegisterFlagsWithPrefix(prefix+"s3.sse.", f) cfg.HTTP.RegisterFlagsWithPrefix(prefix, f) @@ -108,6 +121,14 @@ func (cfg *Config) Validate() error { return errUnsupportedSignatureVersion } + if cfg.ForcePathStyle && cfg.BucketLookupType == VirtualHostedStyleLookup { + return errBucketLookupConfigConflict + } + + if !lo.Contains(supportedBucketLookupTypes, cfg.BucketLookupType) { + return errUnsupportedBucketLookupType + } + if err := cfg.SSE.Validate(); err != nil { return err } diff --git a/pkg/objstore/providers/s3/config_test.go b/pkg/objstore/providers/s3/config_test.go index 3a1d4a81f2..e827de30ab 100644 --- a/pkg/objstore/providers/s3/config_test.go +++ b/pkg/objstore/providers/s3/config_test.go @@ -117,3 +117,44 @@ func TestParseKMSEncryptionContext(t *testing.T) { assert.NoError(t, err) assert.Equal(t, expected, actual) } + +func TestConfig_Validate(t *testing.T) { + tests := map[string]struct { + setup func() *Config + expected error + }{ + "should pass with default config": { + setup: func() *Config { + cfg := &Config{} + flagext.DefaultValues(cfg) + + return cfg + }, + }, + "should fail on invalid bucket lookup style": { + setup: func() *Config { + cfg := &Config{} + flagext.DefaultValues(cfg) + cfg.BucketLookupType = "invalid" + return cfg + }, + expected: errUnsupportedBucketLookupType, + }, + "should fail if force-path-style conflicts with bucket-lookup-type": { + setup: func() *Config { + cfg := &Config{} + flagext.DefaultValues(cfg) + cfg.ForcePathStyle = true + cfg.BucketLookupType = VirtualHostedStyleLookup + return cfg + }, + expected: errBucketLookupConfigConflict, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + assert.Equal(t, testData.expected, testData.setup().Validate()) + }) + } +}