diff --git a/feature/s3/transfermanager/api_op_DownloadDirectory.go b/feature/s3/transfermanager/api_op_DownloadDirectory.go index 16c4e3e4cb0..f081bc366e1 100644 --- a/feature/s3/transfermanager/api_op_DownloadDirectory.go +++ b/feature/s3/transfermanager/api_op_DownloadDirectory.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -19,18 +20,14 @@ import ( // DownloadDirectoryInput represents a request to the DownloadDirectory() call type DownloadDirectoryInput struct { // Bucket where objects are downloaded from - Bucket string + Bucket *string // The destination directory to download - Destination string + Destination *string // The S3 key prefix to use for listing objects. If not provided, // all objects under a bucket will be retrieved - KeyPrefix string - - // The s3 delimiter used to convert keyname to local filepath if it - // is different from local file separator - S3Delimiter string + KeyPrefix *string // A callback func to allow users to fileter out unwanted objects // according to bool returned from the function @@ -39,6 +36,12 @@ type DownloadDirectoryInput struct { // A callback function to allow customers to update individual // GetObjectInput that the S3 Transfer Manager generates Callback GetRequestCallback + + // A callback function to allow users to control the download behavior + // when there are failed objects. The directory download will be terminated + // if its function returns non-nil error and will continue skipping current + // failed object if the function returns nil + FailurePolicy DownloadDirectoryFailurePolicy } // ObjectFilter is the callback to allow users to filter out unwanted objects. @@ -56,10 +59,41 @@ type GetRequestCallback interface { UpdateRequest(*GetObjectInput) } +// DownloadDirectoryFailurePolicy is a callback to allow users to control the +// download behavior when there are failed objects. It is invoked for every failed object. +// If the OnDownloadFailed returns non-nil error, downloader will cancel all ongoing +// single object download requests and terminate the download directory process, if it returns nil +// error, downloader will count the current request as a failed object downloaded but continue +// getting other objects. +type DownloadDirectoryFailurePolicy interface { + OnDownloadFailed(*DownloadDirectoryInput, *GetObjectInput, error) error +} + +// TerminateDownloadPolicy implements DownloadDirectoryFailurePolicy to cancel all other ongoing +// objects download and terminate the download directory call +type TerminateDownloadPolicy struct{} + +// OnDownloadFailed returns the initial err +func (TerminateDownloadPolicy) OnDownloadFailed(directoryInput *DownloadDirectoryInput, objectInput *GetObjectInput, err error) error { + return err +} + +// IgnoreDownloadFailurePolicy implements the DownloadDirectoryFailurePolicy to ignore single object download error +// and continue downloading other objects +type IgnoreDownloadFailurePolicy struct{} + +// OnDownloadFailed ignores input error and return nil +func (IgnoreDownloadFailurePolicy) OnDownloadFailed(*DownloadDirectoryInput, *GetObjectInput, error) error { + return nil +} + // DownloadDirectoryOutput represents a response from the DownloadDirectory() call type DownloadDirectoryOutput struct { // Total number of objects successfully downloaded - ObjectsDownloaded int + ObjectsDownloaded int64 + + // Total number of objects failed to download + ObjectsFailed int64 } type objectEntry struct { @@ -75,13 +109,13 @@ type objectEntry struct { // download. These options are copies of the original Options instance, the client of which DownloadDirectory is called from. // Modifying the options will not impact the original Client and Options instance. func (c *Client) DownloadDirectory(ctx context.Context, input *DownloadDirectoryInput, opts ...func(*Options)) (*DownloadDirectoryOutput, error) { - fileInfo, err := os.Stat(input.Destination) + fileInfo, err := os.Stat(aws.ToString(input.Destination)) if err != nil { if !errors.Is(err, fs.ErrNotExist) { return nil, fmt.Errorf("error when getting destination folder info: %v", err) } } else if !fileInfo.IsDir() { - return nil, fmt.Errorf("the destination path %s doesn't point to a valid directory", input.Destination) + return nil, fmt.Errorf("the destination path %s doesn't point to a valid directory", aws.ToString(input.Destination)) } @@ -94,11 +128,13 @@ func (c *Client) DownloadDirectory(ctx context.Context, input *DownloadDirectory } type directoryDownloader struct { - c *Client - options Options - in *DownloadDirectoryInput + c *Client + options Options + in *DownloadDirectoryInput + failurePolicy DownloadDirectoryFailurePolicy - objectsDownloaded int + objectsDownloaded int64 + objectsFailed int64 err error @@ -125,8 +161,8 @@ func (d *directoryDownloader) downloadDirectory(ctx context.Context) (*DownloadD break } listOutput, err := d.options.S3.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ - Bucket: aws.String(d.in.Bucket), - Prefix: nzstring(d.in.KeyPrefix), + Bucket: d.in.Bucket, + Prefix: d.in.KeyPrefix, ContinuationToken: nzstring(continuationToken), }) if err != nil { @@ -139,7 +175,7 @@ func (d *directoryDownloader) downloadDirectory(ctx context.Context) (*DownloadD break } key := aws.ToString(o.Key) - if strings.HasSuffix(key, "/") || strings.HasSuffix(key, d.in.S3Delimiter) { + if strings.HasSuffix(key, "/") { continue // skip folder object } if d.in.Filter != nil && !d.in.Filter.FilterObject(o) { @@ -167,6 +203,7 @@ func (d *directoryDownloader) downloadDirectory(ctx context.Context) (*DownloadD out := &DownloadDirectoryOutput{ ObjectsDownloaded: d.objectsDownloaded, + ObjectsFailed: d.objectsFailed, } d.emitter.Complete(ctx, out) @@ -175,26 +212,30 @@ func (d *directoryDownloader) downloadDirectory(ctx context.Context) (*DownloadD } func (d *directoryDownloader) init() { - if d.in.S3Delimiter == "" { - d.in.S3Delimiter = "/" + d.failurePolicy = TerminateDownloadPolicy{} + if d.in.FailurePolicy != nil { + d.failurePolicy = d.in.FailurePolicy } + d.emitter = &directoryObjectsProgressEmitter{ Listeners: d.options.DirectoryProgressListeners, } } func (d *directoryDownloader) getLocalPath(key string) (string, error) { - keyprefix := d.in.KeyPrefix - if keyprefix != "" && !strings.HasSuffix(keyprefix, d.in.S3Delimiter) { - keyprefix = keyprefix + d.in.S3Delimiter + keyprefix := aws.ToString(d.in.KeyPrefix) + delimiter := "/" + destination := aws.ToString(d.in.Destination) + if keyprefix != "" && !strings.HasSuffix(keyprefix, delimiter) { + keyprefix = keyprefix + delimiter } - path := filepath.Join(d.in.Destination, strings.ReplaceAll(strings.TrimPrefix(key, keyprefix), d.in.S3Delimiter, string(os.PathSeparator))) - relPath, err := filepath.Rel(d.in.Destination, path) + path := filepath.Join(destination, strings.ReplaceAll(strings.TrimPrefix(key, keyprefix), delimiter, string(os.PathSeparator))) + relPath, err := filepath.Rel(destination, path) if err != nil { return "", err } if relPath == "." || strings.Contains(relPath, "..") { - return "", fmt.Errorf("resolved local path %s is outside of destination %s", path, d.in.Destination) + return "", fmt.Errorf("resolved local path %s is outside of destination %s", path, destination) } return path, nil @@ -221,14 +262,19 @@ func (d *directoryDownloader) downloadObject(ctx context.Context, ch chan object input := &GetObjectInput{ Bucket: d.in.Bucket, - Key: data.key, + Key: aws.String(data.key), } if d.in.Callback != nil { d.in.Callback.UpdateRequest(input) } out, err := d.c.GetObject(ctx, input) if err != nil { - d.setErr(fmt.Errorf("error when downloading object %s: %v", data.key, err)) + err = d.failurePolicy.OnDownloadFailed(d.in, input, err) + if err != nil { + d.setErr(fmt.Errorf("error when heading info of object %s: %v", data.key, err)) + } else { + atomic.AddInt64(&d.objectsFailed, 1) + } continue } @@ -248,23 +294,22 @@ func (d *directoryDownloader) downloadObject(ctx context.Context, ch chan object } n, err := io.Copy(file, out.Body) if err != nil { - d.setErr(fmt.Errorf("error when writing to local file %s: %v", data.path, err)) + // where s3.GetObject is really called, must be handled by failure policy + err = d.failurePolicy.OnDownloadFailed(d.in, input, err) + if err != nil { + d.setErr(fmt.Errorf("error when getting object and writing to local file %s: %v", data.path, err)) + } else { + atomic.AddInt64(&d.objectsFailed, 1) + } os.Remove(data.path) continue } - d.incrObjectsDownloaded(1) + atomic.AddInt64(&d.objectsDownloaded, 1) d.emitter.ObjectsTransferred(ctx, n) } } -func (d *directoryDownloader) incrObjectsDownloaded(n int) { - d.mu.Lock() - defer d.mu.Unlock() - - d.objectsDownloaded += n -} - func (d *directoryDownloader) setErr(err error) { d.mu.Lock() defer d.mu.Unlock() diff --git a/feature/s3/transfermanager/api_op_DownloadDirectory_integ_test.go b/feature/s3/transfermanager/api_op_DownloadDirectory_integ_test.go index 1bfc71b29de..9656efe06a6 100644 --- a/feature/s3/transfermanager/api_op_DownloadDirectory_integ_test.go +++ b/feature/s3/transfermanager/api_op_DownloadDirectory_integ_test.go @@ -20,20 +20,6 @@ func TestInteg_DownloadDirectory(t *testing.T) { ExpectObjectsDownloaded: 3, ExpectFiles: []string{"bar", "oiibaz/zoo", "baz/zoo"}, }, - "multi file with prefix and custom delimiter": { - ObjectsSize: map[string]int64{ - "yee#bar": 2 * 1024 * 1024, - "yee#baz#": 0, - "yee#baz#zoo": 10 * 1024 * 1024, - "yee#oii@zoo": 10 * 1024 * 1024, - "yee#yee#..#bla": 2 * 1024 * 1024, - "ye": 20 * 1024 * 1024, - }, - KeyPrefix: "yee#", - Delimiter: "#", - ExpectObjectsDownloaded: 4, - ExpectFiles: []string{"bar", "baz/zoo", "oii@zoo", "bla"}, - }, } for name, c := range cases { diff --git a/feature/s3/transfermanager/api_op_DownloadObject.go b/feature/s3/transfermanager/api_op_DownloadObject.go index b0c545f5ffb..87e7b39271c 100644 --- a/feature/s3/transfermanager/api_op_DownloadObject.go +++ b/feature/s3/transfermanager/api_op_DownloadObject.go @@ -2,12 +2,15 @@ package transfermanager import ( "context" + "errors" "fmt" "io" "math" + "net/http" "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -22,10 +25,10 @@ import ( // of s3 GetObject input and destination WriterAt of object type DownloadObjectInput struct { // Bucket where the object is downloaded from - Bucket string + Bucket *string // Key of the object to get. - Key string + Key *string // Destination WriterAt which object parts are written to WriterAt io.WriterAt @@ -43,7 +46,7 @@ type DownloadObjectInput struct { // The account ID of the expected bucket owner. If the account ID that you provide // does not match the actual owner of the bucket, the request fails with the HTTP // status code 403 Forbidden (access denied). - ExpectedBucketOwner string + ExpectedBucketOwner *string // Return the object only if its entity tag (ETag) is the same as the one // specified in this header; otherwise, return a 412 Precondition Failed error. @@ -56,7 +59,7 @@ type DownloadObjectInput struct { // For more information about conditional requests, see [RFC 7232]. // // [RFC 7232]: https://tools.ietf.org/html/rfc7232 - IfMatch string + IfMatch *string // Return the object only if it has been modified since the specified time; // otherwise, return a 304 Not Modified error. @@ -69,7 +72,7 @@ type DownloadObjectInput struct { // For more information about conditional requests, see [RFC 7232]. // // [RFC 7232]: https://tools.ietf.org/html/rfc7232 - IfModifiedSince time.Time + IfModifiedSince *time.Time // Return the object only if its entity tag (ETag) is different from the one // specified in this header; otherwise, return a 304 Not Modified error. @@ -82,7 +85,7 @@ type DownloadObjectInput struct { // For more information about conditional requests, see [RFC 7232]. // // [RFC 7232]: https://tools.ietf.org/html/rfc7232 - IfNoneMatch string + IfNoneMatch *string // Return the object only if it has not been modified since the specified time; // otherwise, return a 412 Precondition Failed error. @@ -95,20 +98,7 @@ type DownloadObjectInput struct { // For more information about conditional requests, see [RFC 7232]. // // [RFC 7232]: https://tools.ietf.org/html/rfc7232 - IfUnmodifiedSince time.Time - - // Part number of the object being read. This is a positive integer between 1 and - // 10,000. Effectively performs a 'ranged' GET request for the part specified. - // Useful for downloading just a part of an object. - PartNumber int32 - - // Downloads the specified byte range of an object. For more information about the - // HTTP Range header, see [https://www.rfc-editor.org/rfc/rfc9110.html#name-range]. - // - // Amazon S3 doesn't support retrieving multiple ranges of data per GET request. - // - // [https://www.rfc-editor.org/rfc/rfc9110.html#name-range]: https://www.rfc-editor.org/rfc/rfc9110.html#name-range - Range string + IfUnmodifiedSince *time.Time // Confirms that the requester knows that they will be charged for the request. // Bucket owners need not specify this parameter in their requests. If either the @@ -123,22 +113,22 @@ type DownloadObjectInput struct { RequestPayer types.RequestPayer // Sets the Cache-Control header of the response. - ResponseCacheControl string + ResponseCacheControl *string // Sets the Content-Disposition header of the response. - ResponseContentDisposition string + ResponseContentDisposition *string // Sets the Content-Encoding header of the response. - ResponseContentEncoding string + ResponseContentEncoding *string // Sets the Content-Language header of the response. - ResponseContentLanguage string + ResponseContentLanguage *string // Sets the Content-Type header of the response. - ResponseContentType string + ResponseContentType *string // Sets the Expires header of the response. - ResponseExpires time.Time + ResponseExpires *time.Time // Specifies the algorithm to use when decrypting the object (for example, AES256 ). // @@ -157,7 +147,7 @@ type DownloadObjectInput struct { // This functionality is not supported for directory buckets. // // [Server-Side Encryption (Using Customer-Provided Encryption Keys)]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html - SSECustomerAlgorithm string + SSECustomerAlgorithm *string // Specifies the customer-provided encryption key that you originally provided for // Amazon S3 to encrypt the data before storing it. This value is used to decrypt @@ -180,7 +170,7 @@ type DownloadObjectInput struct { // This functionality is not supported for directory buckets. // // [Server-Side Encryption (Using Customer-Provided Encryption Keys)]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html - SSECustomerKey string + SSECustomerKey *string // Specifies the 128-bit MD5 digest of the customer-provided encryption key // according to RFC 1321. Amazon S3 uses this header for a message integrity check @@ -201,7 +191,7 @@ type DownloadObjectInput struct { // This functionality is not supported for directory buckets. // // [Server-Side Encryption (Using Customer-Provided Encryption Keys)]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html - SSECustomerKeyMD5 string + SSECustomerKeyMD5 *string // Version ID used to reference a specific version of the object. // @@ -224,13 +214,29 @@ type DownloadObjectInput struct { // For more information about versioning, see [PutBucketVersioning]. // // [PutBucketVersioning]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketVersioning.html - VersionID string + VersionID *string } func (i DownloadObjectInput) mapGetObjectInput(enableChecksumValidation bool) *s3.GetObjectInput { input := &s3.GetObjectInput{ - Bucket: aws.String(i.Bucket), - Key: aws.String(i.Key), + Bucket: i.Bucket, + Key: i.Key, + ExpectedBucketOwner: i.ExpectedBucketOwner, + IfMatch: i.IfMatch, + IfNoneMatch: i.IfNoneMatch, + IfModifiedSince: i.IfModifiedSince, + IfUnmodifiedSince: i.IfUnmodifiedSince, + RequestPayer: s3types.RequestPayer(i.RequestPayer), + ResponseCacheControl: i.ResponseCacheControl, + ResponseContentDisposition: i.ResponseContentDisposition, + ResponseContentEncoding: i.ResponseContentEncoding, + ResponseContentLanguage: i.ResponseContentLanguage, + ResponseContentType: i.ResponseContentType, + ResponseExpires: i.ResponseExpires, + SSECustomerAlgorithm: i.SSECustomerAlgorithm, + SSECustomerKey: i.SSECustomerKey, + SSECustomerKeyMD5: i.SSECustomerKeyMD5, + VersionId: i.VersionID, } if i.ChecksumMode != "" { @@ -239,26 +245,6 @@ func (i DownloadObjectInput) mapGetObjectInput(enableChecksumValidation bool) *s input.ChecksumMode = s3types.ChecksumModeEnabled } - if i.RequestPayer != "" { - input.RequestPayer = s3types.RequestPayer(i.RequestPayer) - } - - input.ExpectedBucketOwner = nzstring(i.ExpectedBucketOwner) - input.IfMatch = nzstring(i.IfMatch) - input.IfNoneMatch = nzstring(i.IfNoneMatch) - input.IfModifiedSince = nztime(i.IfModifiedSince) - input.IfUnmodifiedSince = nztime(i.IfUnmodifiedSince) - input.ResponseCacheControl = nzstring(i.ResponseCacheControl) - input.ResponseContentDisposition = nzstring(i.ResponseContentDisposition) - input.ResponseContentEncoding = nzstring(i.ResponseContentEncoding) - input.ResponseContentLanguage = nzstring(i.ResponseContentLanguage) - input.ResponseContentType = nzstring(i.ResponseContentType) - input.ResponseExpires = nztime(i.ResponseExpires) - input.SSECustomerAlgorithm = nzstring(i.SSECustomerAlgorithm) - input.SSECustomerKey = nzstring(i.SSECustomerKey) - input.SSECustomerKeyMD5 = nzstring(i.SSECustomerKeyMD5) - input.VersionId = nzstring(i.VersionID) - return input } @@ -266,14 +252,14 @@ func (i DownloadObjectInput) mapGetObjectInput(enableChecksumValidation bool) *s // of s3 GetObject output except Body which is replaced by WriterAt of input type DownloadObjectOutput struct { // Indicates that a range of bytes was specified in the request. - AcceptRanges string + AcceptRanges *string // Indicates whether the object uses an S3 Bucket Key for server-side encryption // with Key Management Service (KMS) keys (SSE-KMS). - BucketKeyEnabled bool + BucketKeyEnabled *bool // Specifies caching behavior along the request/reply chain. - CacheControl string + CacheControl *string // Specifies if the response checksum validation is enabled ChecksumMode types.ChecksumMode @@ -283,48 +269,63 @@ type DownloadObjectOutput struct { // Amazon S3 User Guide. // // [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html - ChecksumCRC32 string + ChecksumCRC32 *string // The base64-encoded, 32-bit CRC-32C checksum of the object. This will only be // present if it was uploaded with the object. For more information, see [Checking object integrity]in the // Amazon S3 User Guide. // // [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html - ChecksumCRC32C string + ChecksumCRC32C *string + + // The Base64 encoded, 64-bit CRC64NVME checksum of the object. For more + // information, see [Checking object integrity in the Amazon S3 User Guide]. + // + // [Checking object integrity in the Amazon S3 User Guide]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + ChecksumCRC64NVME *string // The base64-encoded, 160-bit SHA-1 digest of the object. This will only be // present if it was uploaded with the object. For more information, see [Checking object integrity]in the // Amazon S3 User Guide. // // [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html - ChecksumSHA1 string + ChecksumSHA1 *string // The base64-encoded, 256-bit SHA-256 digest of the object. This will only be // present if it was uploaded with the object. For more information, see [Checking object integrity]in the // Amazon S3 User Guide. // // [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html - ChecksumSHA256 string + ChecksumSHA256 *string + + // The checksum type, which determines how part-level checksums are combined to + // create an object-level checksum for multipart objects. You can use this header + // response to verify that the checksum type that is received is the same checksum + // type that was specified in the CreateMultipartUpload request. For more + // information, see [Checking object integrity]in the Amazon S3 User Guide. + // + // [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + ChecksumType types.ChecksumType // Specifies presentational information for the object. - ContentDisposition string + ContentDisposition *string // Indicates what content encodings have been applied to the object and thus what // decoding mechanisms must be applied to obtain the media-type referenced by the // Content-Type header field. - ContentEncoding string + ContentEncoding *string // The language the content is in. - ContentLanguage string + ContentLanguage *string // Size of the body in bytes. - ContentLength int64 + ContentLength *int64 // The portion of the object returned in the response. - ContentRange string + ContentRange *string // A standard MIME type describing the format of the object data. - ContentType string + ContentType *string // Indicates whether the object retrieved was (true) or was not (false) a Delete // Marker. If false, this response header does not appear in the response. @@ -336,11 +337,11 @@ type DownloadObjectOutput struct { // - If the specified version in the request is a delete marker, the response // returns a 405 Method Not Allowed error and the Last-Modified: timestamp // response header. - DeleteMarker bool + DeleteMarker *bool // An entity tag (ETag) is an opaque identifier assigned by a web server to a // specific version of a resource found at a URL. - ETag string + ETag *string // If the object expiration is configured (see [PutBucketLifecycleConfiguration]PutBucketLifecycleConfiguration ), // the response includes this header. It includes the expiry-date and rule-id @@ -350,18 +351,18 @@ type DownloadObjectOutput struct { // This functionality is not supported for directory buckets. // // [PutBucketLifecycleConfiguration]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html - Expiration string + Expiration *string // The date and time at which the object is no longer cacheable. // // Deprecated: This field is handled inconsistently across AWS SDKs. Prefer using // the ExpiresString field which contains the unparsed value from the service // response. - Expires time.Time + Expires *time.Time // The unparsed value of the Expires field from the service response. Prefer use // of this value over the normal Expires response field where possible. - ExpiresString string + ExpiresString *string // Date and time when the object was last modified. // @@ -369,7 +370,7 @@ type DownloadObjectOutput struct { // request, if the specified version in the request is a delete marker, the // response returns a 405 Method Not Allowed error and the Last-Modified: timestamp // response header. - LastModified time.Time + LastModified *time.Time // A map of metadata to store with the object in S3. // @@ -383,7 +384,7 @@ type DownloadObjectOutput struct { // headers. // // This functionality is not supported for directory buckets. - MissingMeta int32 + MissingMeta *int32 // Indicates whether this object has an active legal hold. This field is only // returned if you have permission to view an object's legal hold status. @@ -399,11 +400,11 @@ type DownloadObjectOutput struct { // The date and time when this object's Object Lock will expire. // // This functionality is not supported for directory buckets. - ObjectLockRetainUntilDate time.Time + ObjectLockRetainUntilDate *time.Time // The count of parts this object has. This value is only returned if you specify // partNumber in your request and the object was uploaded as a multipart upload. - PartsCount int32 + PartsCount *int32 // Amazon S3 can return this if your request involves a bucket that is either a // source or destination in a replication rule. @@ -422,24 +423,24 @@ type DownloadObjectOutput struct { // // This functionality is not supported for directory buckets. Only the S3 Express // One Zone storage class is supported by directory buckets to store objects. - Restore string + Restore *string // If server-side encryption with a customer-provided encryption key was // requested, the response will include this header to confirm the encryption // algorithm that's used. // // This functionality is not supported for directory buckets. - SSECustomerAlgorithm string + SSECustomerAlgorithm *string // If server-side encryption with a customer-provided encryption key was // requested, the response will include this header to provide the round-trip // message integrity verification of the customer-provided encryption key. // // This functionality is not supported for directory buckets. - SSECustomerKeyMD5 string + SSECustomerKeyMD5 *string // If present, indicates the ID of the KMS key that was used for object encryption. - SSEKMSKeyID string + SSEKMSKeyID *string // The server-side encryption algorithm used when you store this object in Amazon // S3. @@ -460,63 +461,65 @@ type DownloadObjectOutput struct { // This functionality is not supported for directory buckets. // // [GetObjectTagging]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectTagging.html - TagCount int32 + TagCount *int32 // Version ID of the object. // // This functionality is not supported for directory buckets. - VersionID string + VersionID *string // If the bucket is configured as a website, redirects requests for this object to // another object in the same bucket or to an external URL. Amazon S3 stores the // value of this header in the object metadata. // // This functionality is not supported for directory buckets. - WebsiteRedirectLocation string + WebsiteRedirectLocation *string // Metadata pertaining to the operation's result. ResultMetadata smithymiddleware.Metadata } func (o *DownloadObjectOutput) mapFromGetObjectOutput(out *s3.GetObjectOutput, checksumMode s3types.ChecksumMode) { - o.AcceptRanges = aws.ToString(out.AcceptRanges) - o.CacheControl = aws.ToString(out.CacheControl) + o.AcceptRanges = out.AcceptRanges + o.BucketKeyEnabled = out.BucketKeyEnabled + o.CacheControl = out.CacheControl o.ChecksumMode = types.ChecksumMode(checksumMode) - o.ChecksumCRC32 = aws.ToString(out.ChecksumCRC32) - o.ChecksumCRC32C = aws.ToString(out.ChecksumCRC32C) - o.ChecksumSHA1 = aws.ToString(out.ChecksumSHA1) - o.ChecksumSHA256 = aws.ToString(out.ChecksumSHA256) - o.ContentDisposition = aws.ToString(out.ContentDisposition) - o.ContentEncoding = aws.ToString(out.ContentEncoding) - o.ContentLanguage = aws.ToString(out.ContentLanguage) - o.ContentRange = aws.ToString(out.ContentRange) - o.ContentType = aws.ToString(out.ContentType) - o.ETag = aws.ToString(out.ETag) - o.Expiration = aws.ToString(out.Expiration) - o.ExpiresString = aws.ToString(out.ExpiresString) - o.Restore = aws.ToString(out.Restore) - o.SSECustomerAlgorithm = aws.ToString(out.SSECustomerAlgorithm) - o.SSECustomerKeyMD5 = aws.ToString(out.SSECustomerKeyMD5) - o.SSEKMSKeyID = aws.ToString(out.SSEKMSKeyId) - o.VersionID = aws.ToString(out.VersionId) - o.WebsiteRedirectLocation = aws.ToString(out.WebsiteRedirectLocation) - o.BucketKeyEnabled = aws.ToBool(out.BucketKeyEnabled) - o.DeleteMarker = aws.ToBool(out.DeleteMarker) - o.MissingMeta = aws.ToInt32(out.MissingMeta) - o.PartsCount = aws.ToInt32(out.PartsCount) - o.TagCount = aws.ToInt32(out.TagCount) - o.ContentLength = aws.ToInt64(out.ContentLength) - o.Expires = aws.ToTime(out.Expires) - o.LastModified = aws.ToTime(out.LastModified) - o.ObjectLockRetainUntilDate = aws.ToTime(out.ObjectLockRetainUntilDate) + o.ChecksumCRC32 = out.ChecksumCRC32 + o.ChecksumCRC32C = out.ChecksumCRC32C + o.ChecksumCRC64NVME = out.ChecksumCRC64NVME + o.ChecksumSHA1 = out.ChecksumSHA1 + o.ChecksumSHA256 = out.ChecksumSHA256 + o.ChecksumType = types.ChecksumType(out.ChecksumType) + o.ContentDisposition = out.ContentDisposition + o.ContentEncoding = out.ContentEncoding + o.ContentLanguage = out.ContentLanguage + o.ContentLength = out.ContentLength + o.ContentRange = out.ContentRange + o.ContentType = out.ContentType + o.DeleteMarker = out.DeleteMarker + o.ETag = out.ETag + o.Expiration = out.Expiration + o.Expires = out.Expires + o.ExpiresString = out.ExpiresString + o.LastModified = out.LastModified o.Metadata = out.Metadata + o.MissingMeta = out.MissingMeta o.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatus(out.ObjectLockLegalHoldStatus) o.ObjectLockMode = types.ObjectLockMode(out.ObjectLockMode) + o.ObjectLockRetainUntilDate = out.ObjectLockRetainUntilDate + o.PartsCount = out.PartsCount o.ReplicationStatus = types.ReplicationStatus(out.ReplicationStatus) o.RequestCharged = types.RequestCharged(out.RequestCharged) + o.Restore = out.Restore + o.SSECustomerAlgorithm = out.SSECustomerAlgorithm + o.SSECustomerKeyMD5 = out.SSECustomerKeyMD5 + o.SSEKMSKeyID = out.SSEKMSKeyId o.ServerSideEncryption = types.ServerSideEncryption(out.ServerSideEncryption) o.StorageClass = types.StorageClass(out.StorageClass) - o.ResultMetadata = out.ResultMetadata.Clone() + o.TagCount = out.TagCount + o.VersionID = out.VersionId + o.WebsiteRedirectLocation = out.WebsiteRedirectLocation + o.ResultMetadata = out.ResultMetadata } // DownloadObject downloads an object from S3, intelligently splitting large @@ -570,31 +573,23 @@ func (d *downloader) download(ctx context.Context) (*DownloadObjectOutput, error ) }} - if d.in.PartNumber > 0 { - return d.singleDownload(ctx, clientOptions...) - } - var output *DownloadObjectOutput if d.options.GetObjectType == types.GetObjectParts { - if d.in.Range != "" { - return d.singleDownload(ctx, clientOptions...) - } - output = d.getChunk(ctx, 1, "", clientOptions...) if d.err != nil { d.emitter.Failed(ctx, d.err) return output, d.err } - if output.PartsCount > 1 { - partSize := output.ContentLength + if aws.ToInt32(output.PartsCount) > 1 { + partSize := aws.ToInt64(output.ContentLength) ch := make(chan dlChunk, d.options.Concurrency) for i := 0; i < d.options.Concurrency; i++ { d.wg.Add(1) go d.downloadPart(ctx, ch, clientOptions...) } - for i := int32(2); i <= output.PartsCount; i++ { + for i := int32(2); i <= aws.ToInt32(output.PartsCount); i++ { if d.getErr() != nil { break } @@ -607,12 +602,25 @@ func (d *downloader) download(ctx context.Context) (*DownloadObjectOutput, error d.wg.Wait() } } else { - if d.in.Range != "" { - d.pos, d.totalBytes = d.getDownloadRange() - d.offset = d.pos - } - d.getChunk(ctx, 0, d.byteRange(), clientOptions...) + if d.err != nil { + // early check to see if error is caused by range download a zero object + // which will always return an invalid range error from s3 side + var responseError interface { + HTTPStatusCode() int + } + if errors.As(d.err, &responseError) { + if responseError.HTTPStatusCode() == http.StatusRequestedRangeNotSatisfiable { + out := &DownloadObjectOutput{ + ContentLength: aws.Int64(0), + } + d.emitter.Complete(ctx, out) + return out, nil + } + } + d.emitter.Failed(ctx, d.err) + return nil, d.err + } total := d.totalBytes ch := make(chan dlChunk, d.options.Concurrency) @@ -624,7 +632,7 @@ func (d *downloader) download(ctx context.Context) (*DownloadObjectOutput, error // Assign work for d.getErr() == nil { if d.pos >= total { - break // We're finished queuing chunks + break // We finish queuing chunks } // Queue the next range of bytes to read. @@ -644,8 +652,15 @@ func (d *downloader) download(ctx context.Context) (*DownloadObjectOutput, error d.emitter.Complete(ctx, d.out) - d.out.ContentLength = d.written - d.out.ContentRange = fmt.Sprintf("bytes=%d-%d", d.offset, d.totalBytes-1) + d.out.ContentRange = aws.String(fmt.Sprintf("bytes=%d-%d", d.offset, d.totalBytes-1)) + d.out.ContentLength = aws.Int64(d.written) + if d.out.ChecksumType == types.ChecksumTypeComposite { + d.out.ChecksumCRC32 = nil + d.out.ChecksumCRC32C = nil + d.out.ChecksumCRC64NVME = nil + d.out.ChecksumSHA1 = nil + d.out.ChecksumSHA256 = nil + } return d.out, nil } @@ -666,20 +681,6 @@ func (d *downloader) init() error { return nil } -func (d *downloader) singleDownload(ctx context.Context, clientOptions ...func(*s3.Options)) (*DownloadObjectOutput, error) { - chunk := dlChunk{w: d.in.WriterAt} - - // progress start is called idempotently on first response received - output, err := d.downloadChunk(ctx, chunk, clientOptions...) - if err != nil { - d.emitter.Failed(ctx, err) - return nil, err - } - - d.emitter.Complete(ctx, output) - return output, nil -} - func (d *downloader) downloadPart(ctx context.Context, ch chan dlChunk, clientOptions ...func(*s3.Options)) { defer d.wg.Done() for { @@ -700,7 +701,7 @@ func (d *downloader) downloadPart(ctx context.Context, ch chan dlChunk, clientOp } // getChunk grabs a chunk of data from the body. -// Not thread safe. Should only used when grabbing data on a single thread. +// Not thread safe. Should only be used when grabbing data on a single thread. func (d *downloader) getChunk(ctx context.Context, part int32, rng string, clientOptions ...func(*s3.Options)) *DownloadObjectOutput { chunk := dlChunk{w: d.in.WriterAt, start: d.pos - d.offset, part: part, withRange: rng} @@ -711,7 +712,7 @@ func (d *downloader) getChunk(ctx context.Context, part int32, rng string, clien } d.setOutput(output) - d.pos += output.ContentLength + d.pos += aws.ToInt64(output.ContentLength) return output } @@ -766,6 +767,21 @@ func (d *downloader) tryDownloadChunk(ctx context.Context, params *s3.GetObjectI return nil, err } + if params.Range != nil && out.ContentRange != nil { + reqStart, reqEnd, err := getReqRange(aws.ToString(params.Range)) + if err != nil { + return nil, err + } + respStart, respEnd, err := getRespRange(aws.ToString(out.ContentRange)) + if err != nil { + return nil, err + } + // don't validate first chunk since object size is unknown when getting that + if reqStart != 0 && (reqStart != respStart || reqEnd != respEnd) { + return nil, fmt.Errorf("range mismatch between request %d-%d and response %d-%d", reqStart, reqEnd, respStart, respEnd) + } + } + d.totalBytesOnce.Do(func() { d.setTotalBytes(out) d.emitter.Start(ctx, d.in, d.totalBytes) @@ -778,7 +794,7 @@ func (d *downloader) tryDownloadChunk(ctx context.Context, params *s3.GetObjectI return nil, &errReadingBody{err: err} } - d.incrWritten(n) + atomic.AddInt64(&d.written, n) d.emitter.BytesTransferred(ctx, n) return out, nil } @@ -827,32 +843,41 @@ func (d *downloader) setOutput(resp *DownloadObjectOutput) { d.out = resp } -// TODO this might be shared beteen get and download -func (d *downloader) getDownloadRange() (int64, int64) { - parts := strings.Split(strings.Split(d.in.Range, "=")[1], "-") +// byteRange returns a HTTP Byte-Range header value that should be used by the +// client to request a chunk range. +func (d *downloader) byteRange() string { + if d.totalBytes >= 0 { + return fmt.Sprintf("bytes=%d-%d", d.pos, int64(math.Min(float64(d.totalBytes-1), float64(d.pos+d.options.PartSizeBytes-1)))) + } + return fmt.Sprintf("bytes=%d-%d", d.pos, d.pos+d.options.PartSizeBytes-1) +} - start, err := strconv.ParseInt(parts[0], 10, 64) +func getReqRange(rng string) (int64, int64, error) { + // rng fmt "bytes=start-end" + ranges := strings.Split(strings.Split(rng, "=")[1], "-") + start, err := strconv.ParseInt(ranges[0], 10, 64) if err != nil { - d.err = err - return 0, 0 + return 0, 0, fmt.Errorf("error when parsing request start: %v", err) } - - end, err := strconv.ParseInt(parts[1], 10, 64) + end, err := strconv.ParseInt(ranges[1], 10, 64) if err != nil { - d.err = err - return 0, 0 + return 0, 0, fmt.Errorf("error when parsing request end: %v", err) } - - return start, end + 1 + return start, end, nil } -// byteRange returns a HTTP Byte-Range header value that should be used by the -// client to request a chunk range. -func (d *downloader) byteRange() string { - if d.totalBytes >= 0 { - return fmt.Sprintf("bytes=%d-%d", d.pos, int64(math.Min(float64(d.totalBytes-1), float64(d.pos+d.options.PartSizeBytes-1)))) +func getRespRange(rng string) (int64, int64, error) { + // rng format "bytes %d-%d/%d" + ranges := strings.Split(strings.Split(strings.Split(rng, " ")[1], "/")[0], "-") + start, err := strconv.ParseInt(ranges[0], 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("error when parsing response start: %v", err) } - return fmt.Sprintf("bytes=%d-%d", d.pos, d.pos+d.options.PartSizeBytes-1) + end, err := strconv.ParseInt(ranges[1], 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("error when parsing response end: %v", err) + } + return start, end, nil } func (d *downloader) getErr() error { diff --git a/feature/s3/transfermanager/api_op_DownloadObject_integ_test.go b/feature/s3/transfermanager/api_op_DownloadObject_integ_test.go index 00b7a020809..093c69f2674 100644 --- a/feature/s3/transfermanager/api_op_DownloadObject_integ_test.go +++ b/feature/s3/transfermanager/api_op_DownloadObject_integ_test.go @@ -25,8 +25,8 @@ func TestInteg_DownloadObject(t *testing.T) { }, }, "range get empty string body": { - Body: strings.NewReader(""), - ExpectError: "InvalidRange", + Body: strings.NewReader(""), + ExpectBody: []byte(""), OptFns: []func(*Options){ func(opt *Options) { opt.GetObjectType = types.GetObjectRanges diff --git a/feature/s3/transfermanager/api_op_GetObject.go b/feature/s3/transfermanager/api_op_GetObject.go index b3078d6cc78..b65284a01b3 100644 --- a/feature/s3/transfermanager/api_op_GetObject.go +++ b/feature/s3/transfermanager/api_op_GetObject.go @@ -6,6 +6,7 @@ import ( "io" "strconv" "strings" + "sync/atomic" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -36,10 +37,10 @@ func (e *errInvalidRange) Error() string { // of s3 GetObject input type GetObjectInput struct { // Bucket where the object is downloaded from - Bucket string + Bucket *string // Key of the object to get. - Key string + Key *string // To retrieve the checksum, this mode must be enabled. // @@ -54,7 +55,7 @@ type GetObjectInput struct { // The account ID of the expected bucket owner. If the account ID that you provide // does not match the actual owner of the bucket, the request fails with the HTTP // status code 403 Forbidden (access denied). - ExpectedBucketOwner string + ExpectedBucketOwner *string // Return the object only if its entity tag (ETag) is the same as the one // specified in this header; otherwise, return a 412 Precondition Failed error. @@ -67,7 +68,7 @@ type GetObjectInput struct { // For more information about conditional requests, see [RFC 7232]. // // [RFC 7232]: https://tools.ietf.org/html/rfc7232 - IfMatch string + IfMatch *string // Return the object only if it has been modified since the specified time; // otherwise, return a 304 Not Modified error. @@ -80,7 +81,7 @@ type GetObjectInput struct { // For more information about conditional requests, see [RFC 7232]. // // [RFC 7232]: https://tools.ietf.org/html/rfc7232 - IfModifiedSince time.Time + IfModifiedSince *time.Time // Return the object only if its entity tag (ETag) is different from the one // specified in this header; otherwise, return a 304 Not Modified error. @@ -93,7 +94,7 @@ type GetObjectInput struct { // For more information about conditional requests, see [RFC 7232]. // // [RFC 7232]: https://tools.ietf.org/html/rfc7232 - IfNoneMatch string + IfNoneMatch *string // Return the object only if it has not been modified since the specified time; // otherwise, return a 412 Precondition Failed error. @@ -106,20 +107,7 @@ type GetObjectInput struct { // For more information about conditional requests, see [RFC 7232]. // // [RFC 7232]: https://tools.ietf.org/html/rfc7232 - IfUnmodifiedSince time.Time - - // Part number of the object being read. This is a positive integer between 1 and - // 10,000. Effectively performs a 'ranged' GET request for the part specified. - // Useful for downloading just a part of an object. - PartNumber int32 - - // Downloads the specified byte range of an object. For more information about the - // HTTP Range header, see [https://www.rfc-editor.org/rfc/rfc9110.html#name-range]. - // - // Amazon S3 doesn't support retrieving multiple ranges of data per GET request. - // - // [https://www.rfc-editor.org/rfc/rfc9110.html#name-range]: https://www.rfc-editor.org/rfc/rfc9110.html#name-range - Range string + IfUnmodifiedSince *time.Time // Confirms that the requester knows that they will be charged for the request. // Bucket owners need not specify this parameter in their requests. If either the @@ -134,22 +122,22 @@ type GetObjectInput struct { RequestPayer types.RequestPayer // Sets the Cache-Control header of the response. - ResponseCacheControl string + ResponseCacheControl *string // Sets the Content-Disposition header of the response. - ResponseContentDisposition string + ResponseContentDisposition *string // Sets the Content-Encoding header of the response. - ResponseContentEncoding string + ResponseContentEncoding *string // Sets the Content-Language header of the response. - ResponseContentLanguage string + ResponseContentLanguage *string // Sets the Content-Type header of the response. - ResponseContentType string + ResponseContentType *string // Sets the Expires header of the response. - ResponseExpires time.Time + ResponseExpires *time.Time // Specifies the algorithm to use when decrypting the object (for example, AES256 ). // @@ -168,7 +156,7 @@ type GetObjectInput struct { // This functionality is not supported for directory buckets. // // [Server-Side Encryption (Using Customer-Provided Encryption Keys)]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html - SSECustomerAlgorithm string + SSECustomerAlgorithm *string // Specifies the customer-provided encryption key that you originally provided for // Amazon S3 to encrypt the data before storing it. This value is used to decrypt @@ -191,7 +179,7 @@ type GetObjectInput struct { // This functionality is not supported for directory buckets. // // [Server-Side Encryption (Using Customer-Provided Encryption Keys)]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html - SSECustomerKey string + SSECustomerKey *string // Specifies the 128-bit MD5 digest of the customer-provided encryption key // according to RFC 1321. Amazon S3 uses this header for a message integrity check @@ -212,7 +200,7 @@ type GetObjectInput struct { // This functionality is not supported for directory buckets. // // [Server-Side Encryption (Using Customer-Provided Encryption Keys)]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html - SSECustomerKeyMD5 string + SSECustomerKeyMD5 *string // Version ID used to reference a specific version of the object. // @@ -235,13 +223,29 @@ type GetObjectInput struct { // For more information about versioning, see [PutBucketVersioning]. // // [PutBucketVersioning]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketVersioning.html - VersionID string + VersionID *string } func (i GetObjectInput) mapGetObjectInput(enableChecksumValidation bool) *s3.GetObjectInput { input := &s3.GetObjectInput{ - Bucket: aws.String(i.Bucket), - Key: aws.String(i.Key), + Bucket: i.Bucket, + Key: i.Key, + ExpectedBucketOwner: i.ExpectedBucketOwner, + IfMatch: i.IfMatch, + IfNoneMatch: i.IfNoneMatch, + IfModifiedSince: i.IfModifiedSince, + IfUnmodifiedSince: i.IfUnmodifiedSince, + RequestPayer: s3types.RequestPayer(i.RequestPayer), + ResponseCacheControl: i.ResponseCacheControl, + ResponseContentDisposition: i.ResponseContentDisposition, + ResponseContentEncoding: i.ResponseContentEncoding, + ResponseContentLanguage: i.ResponseContentLanguage, + ResponseContentType: i.ResponseContentType, + ResponseExpires: i.ResponseExpires, + SSECustomerAlgorithm: i.SSECustomerAlgorithm, + SSECustomerKey: i.SSECustomerKey, + SSECustomerKeyMD5: i.SSECustomerKeyMD5, + VersionId: i.VersionID, } if i.ChecksumMode != "" { @@ -250,26 +254,6 @@ func (i GetObjectInput) mapGetObjectInput(enableChecksumValidation bool) *s3.Get input.ChecksumMode = s3types.ChecksumModeEnabled } - if i.RequestPayer != "" { - input.RequestPayer = s3types.RequestPayer(i.RequestPayer) - } - - input.ExpectedBucketOwner = nzstring(i.ExpectedBucketOwner) - input.IfMatch = nzstring(i.IfMatch) - input.IfNoneMatch = nzstring(i.IfNoneMatch) - input.IfModifiedSince = nztime(i.IfModifiedSince) - input.IfUnmodifiedSince = nztime(i.IfUnmodifiedSince) - input.ResponseCacheControl = nzstring(i.ResponseCacheControl) - input.ResponseContentDisposition = nzstring(i.ResponseContentDisposition) - input.ResponseContentEncoding = nzstring(i.ResponseContentEncoding) - input.ResponseContentLanguage = nzstring(i.ResponseContentLanguage) - input.ResponseContentType = nzstring(i.ResponseContentType) - input.ResponseExpires = nztime(i.ResponseExpires) - input.SSECustomerAlgorithm = nzstring(i.SSECustomerAlgorithm) - input.SSECustomerKey = nzstring(i.SSECustomerKey) - input.SSECustomerKeyMD5 = nzstring(i.SSECustomerKeyMD5) - input.VersionId = nzstring(i.VersionID) - return input } @@ -277,17 +261,17 @@ func (i GetObjectInput) mapGetObjectInput(enableChecksumValidation bool) *s3.Get // of s3 GetObject output type GetObjectOutput struct { // Indicates that a range of bytes was specified in the request. - AcceptRanges string + AcceptRanges *string // Object data. Body io.Reader // Indicates whether the object uses an S3 Bucket Key for server-side encryption // with Key Management Service (KMS) keys (SSE-KMS). - BucketKeyEnabled bool + BucketKeyEnabled *bool // Specifies caching behavior along the request/reply chain. - CacheControl string + CacheControl *string // Specifies if the response checksum validation is enabled ChecksumMode types.ChecksumMode @@ -297,48 +281,63 @@ type GetObjectOutput struct { // Amazon S3 User Guide. // // [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html - ChecksumCRC32 string + ChecksumCRC32 *string // The base64-encoded, 32-bit CRC-32C checksum of the object. This will only be // present if it was uploaded with the object. For more information, see [Checking object integrity]in the // Amazon S3 User Guide. // // [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html - ChecksumCRC32C string + ChecksumCRC32C *string + + // The Base64 encoded, 64-bit CRC64NVME checksum of the object. For more + // information, see [Checking object integrity in the Amazon S3 User Guide]. + // + // [Checking object integrity in the Amazon S3 User Guide]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + ChecksumCRC64NVME *string // The base64-encoded, 160-bit SHA-1 digest of the object. This will only be // present if it was uploaded with the object. For more information, see [Checking object integrity]in the // Amazon S3 User Guide. // // [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html - ChecksumSHA1 string + ChecksumSHA1 *string // The base64-encoded, 256-bit SHA-256 digest of the object. This will only be // present if it was uploaded with the object. For more information, see [Checking object integrity]in the // Amazon S3 User Guide. // // [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html - ChecksumSHA256 string + ChecksumSHA256 *string + + // The checksum type, which determines how part-level checksums are combined to + // create an object-level checksum for multipart objects. You can use this header + // response to verify that the checksum type that is received is the same checksum + // type that was specified in the CreateMultipartUpload request. For more + // information, see [Checking object integrity]in the Amazon S3 User Guide. + // + // [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + ChecksumType types.ChecksumType // Specifies presentational information for the object. - ContentDisposition string + ContentDisposition *string // Indicates what content encodings have been applied to the object and thus what // decoding mechanisms must be applied to obtain the media-type referenced by the // Content-Type header field. - ContentEncoding string + ContentEncoding *string // The language the content is in. - ContentLanguage string + ContentLanguage *string // Size of the body in bytes. - ContentLength int64 + ContentLength *int64 // The portion of the object returned in the response. - ContentRange string + ContentRange *string // A standard MIME type describing the format of the object data. - ContentType string + ContentType *string // Indicates whether the object retrieved was (true) or was not (false) a Delete // Marker. If false, this response header does not appear in the response. @@ -350,11 +349,11 @@ type GetObjectOutput struct { // - If the specified version in the request is a delete marker, the response // returns a 405 Method Not Allowed error and the Last-Modified: timestamp // response header. - DeleteMarker bool + DeleteMarker *bool // An entity tag (ETag) is an opaque identifier assigned by a web server to a // specific version of a resource found at a URL. - ETag string + ETag *string // If the object expiration is configured (see [PutBucketLifecycleConfiguration]PutBucketLifecycleConfiguration ), // the response includes this header. It includes the expiry-date and rule-id @@ -364,18 +363,18 @@ type GetObjectOutput struct { // This functionality is not supported for directory buckets. // // [PutBucketLifecycleConfiguration]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html - Expiration string + Expiration *string // The date and time at which the object is no longer cacheable. // // Deprecated: This field is handled inconsistently across AWS SDKs. Prefer using // the ExpiresString field which contains the unparsed value from the service // response. - Expires time.Time + Expires *time.Time // The unparsed value of the Expires field from the service response. Prefer use // of this value over the normal Expires response field where possible. - ExpiresString string + ExpiresString *string // Date and time when the object was last modified. // @@ -383,7 +382,7 @@ type GetObjectOutput struct { // request, if the specified version in the request is a delete marker, the // response returns a 405 Method Not Allowed error and the Last-Modified: timestamp // response header. - LastModified time.Time + LastModified *time.Time // A map of metadata to store with the object in S3. // @@ -397,7 +396,7 @@ type GetObjectOutput struct { // headers. // // This functionality is not supported for directory buckets. - MissingMeta int32 + MissingMeta *int32 // Indicates whether this object has an active legal hold. This field is only // returned if you have permission to view an object's legal hold status. @@ -413,11 +412,11 @@ type GetObjectOutput struct { // The date and time when this object's Object Lock will expire. // // This functionality is not supported for directory buckets. - ObjectLockRetainUntilDate time.Time + ObjectLockRetainUntilDate *time.Time // The count of parts this object has. This value is only returned if you specify // partNumber in your request and the object was uploaded as a multipart upload. - PartsCount int32 + PartsCount *int32 // Amazon S3 can return this if your request involves a bucket that is either a // source or destination in a replication rule. @@ -436,24 +435,24 @@ type GetObjectOutput struct { // // This functionality is not supported for directory buckets. Only the S3 Express // One Zone storage class is supported by directory buckets to store objects. - Restore string + Restore *string // If server-side encryption with a customer-provided encryption key was // requested, the response will include this header to confirm the encryption // algorithm that's used. // // This functionality is not supported for directory buckets. - SSECustomerAlgorithm string + SSECustomerAlgorithm *string // If server-side encryption with a customer-provided encryption key was // requested, the response will include this header to provide the round-trip // message integrity verification of the customer-provided encryption key. // // This functionality is not supported for directory buckets. - SSECustomerKeyMD5 string + SSECustomerKeyMD5 *string // If present, indicates the ID of the KMS key that was used for object encryption. - SSEKMSKeyID string + SSEKMSKeyID *string // The server-side encryption algorithm used when you store this object in Amazon // S3. @@ -474,108 +473,114 @@ type GetObjectOutput struct { // This functionality is not supported for directory buckets. // // [GetObjectTagging]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectTagging.html - TagCount int32 + TagCount *int32 // Version ID of the object. // // This functionality is not supported for directory buckets. - VersionID string + VersionID *string // If the bucket is configured as a website, redirects requests for this object to // another object in the same bucket or to an external URL. Amazon S3 stores the // value of this header in the object metadata. // // This functionality is not supported for directory buckets. - WebsiteRedirectLocation string + WebsiteRedirectLocation *string // Metadata pertaining to the operation's result. ResultMetadata smithymiddleware.Metadata } func (o *GetObjectOutput) mapFromGetObjectOutput(out *s3.GetObjectOutput, checksumMode s3types.ChecksumMode) { - o.AcceptRanges = aws.ToString(out.AcceptRanges) - o.CacheControl = aws.ToString(out.CacheControl) - o.ChecksumMode = types.ChecksumMode(checksumMode) - o.ChecksumCRC32 = aws.ToString(out.ChecksumCRC32) - o.ChecksumCRC32C = aws.ToString(out.ChecksumCRC32C) - o.ChecksumSHA1 = aws.ToString(out.ChecksumSHA1) - o.ChecksumSHA256 = aws.ToString(out.ChecksumSHA256) - o.ContentDisposition = aws.ToString(out.ContentDisposition) - o.ContentEncoding = aws.ToString(out.ContentEncoding) - o.ContentLanguage = aws.ToString(out.ContentLanguage) - o.ContentRange = aws.ToString(out.ContentRange) - o.ContentType = aws.ToString(out.ContentType) - o.ETag = aws.ToString(out.ETag) - o.Expiration = aws.ToString(out.Expiration) - o.ExpiresString = aws.ToString(out.ExpiresString) - o.Restore = aws.ToString(out.Restore) - o.SSECustomerAlgorithm = aws.ToString(out.SSECustomerAlgorithm) - o.SSECustomerKeyMD5 = aws.ToString(out.SSECustomerKeyMD5) - o.SSEKMSKeyID = aws.ToString(out.SSEKMSKeyId) - o.VersionID = aws.ToString(out.VersionId) - o.WebsiteRedirectLocation = aws.ToString(out.WebsiteRedirectLocation) - o.BucketKeyEnabled = aws.ToBool(out.BucketKeyEnabled) - o.DeleteMarker = aws.ToBool(out.DeleteMarker) - o.MissingMeta = aws.ToInt32(out.MissingMeta) - o.PartsCount = aws.ToInt32(out.PartsCount) - o.TagCount = aws.ToInt32(out.TagCount) - o.ContentLength = aws.ToInt64(out.ContentLength) o.Body = out.Body - o.Expires = aws.ToTime(out.Expires) - o.LastModified = aws.ToTime(out.LastModified) - o.ObjectLockRetainUntilDate = aws.ToTime(out.ObjectLockRetainUntilDate) + o.AcceptRanges = out.AcceptRanges + o.BucketKeyEnabled = out.BucketKeyEnabled + o.CacheControl = out.CacheControl + o.ChecksumMode = types.ChecksumMode(checksumMode) + o.ChecksumCRC32 = out.ChecksumCRC32 + o.ChecksumCRC32C = out.ChecksumCRC32C + o.ChecksumCRC64NVME = out.ChecksumCRC64NVME + o.ChecksumSHA1 = out.ChecksumSHA1 + o.ChecksumSHA256 = out.ChecksumSHA256 + o.ChecksumType = types.ChecksumType(out.ChecksumType) + o.ContentDisposition = out.ContentDisposition + o.ContentEncoding = out.ContentEncoding + o.ContentLanguage = out.ContentLanguage + o.ContentLength = out.ContentLength + o.ContentRange = out.ContentRange + o.ContentType = out.ContentType + o.DeleteMarker = out.DeleteMarker + o.ETag = out.ETag + o.Expiration = out.Expiration + o.Expires = out.Expires + o.ExpiresString = out.ExpiresString + o.LastModified = out.LastModified o.Metadata = out.Metadata + o.MissingMeta = out.MissingMeta o.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatus(out.ObjectLockLegalHoldStatus) o.ObjectLockMode = types.ObjectLockMode(out.ObjectLockMode) + o.ObjectLockRetainUntilDate = out.ObjectLockRetainUntilDate + o.PartsCount = out.PartsCount o.ReplicationStatus = types.ReplicationStatus(out.ReplicationStatus) o.RequestCharged = types.RequestCharged(out.RequestCharged) + o.Restore = out.Restore + o.SSECustomerAlgorithm = out.SSECustomerAlgorithm + o.SSECustomerKeyMD5 = out.SSECustomerKeyMD5 + o.SSEKMSKeyID = out.SSEKMSKeyId o.ServerSideEncryption = types.ServerSideEncryption(out.ServerSideEncryption) o.StorageClass = types.StorageClass(out.StorageClass) - o.ResultMetadata = out.ResultMetadata.Clone() + o.TagCount = out.TagCount + o.VersionID = out.VersionId + o.WebsiteRedirectLocation = out.WebsiteRedirectLocation + o.ResultMetadata = out.ResultMetadata } func (o *GetObjectOutput) mapFromHeadObjectOutput(out *s3.HeadObjectOutput, checksumMode types.ChecksumMode, enableChecksumValidation bool, body *concurrentReader) { - o.AcceptRanges = aws.ToString(out.AcceptRanges) - o.CacheControl = aws.ToString(out.CacheControl) + o.Body = body + o.AcceptRanges = out.AcceptRanges + o.BucketKeyEnabled = out.BucketKeyEnabled + o.CacheControl = out.CacheControl + o.ChecksumCRC32 = out.ChecksumCRC32 + o.ChecksumCRC32C = out.ChecksumCRC32C + o.ChecksumCRC64NVME = out.ChecksumCRC64NVME + o.ChecksumSHA1 = out.ChecksumSHA1 + o.ChecksumSHA256 = out.ChecksumSHA256 + o.ChecksumType = types.ChecksumType(out.ChecksumType) + o.ContentDisposition = out.ContentDisposition + o.ContentEncoding = out.ContentEncoding + o.ContentLanguage = out.ContentLanguage + o.ContentLength = aws.Int64(getTotalBytes(out)) + o.ContentRange = out.ContentRange + o.ContentType = out.ContentType if checksumMode != "" { o.ChecksumMode = checksumMode } else if enableChecksumValidation { o.ChecksumMode = types.ChecksumModeEnabled } - o.ChecksumCRC32 = aws.ToString(out.ChecksumCRC32) - o.ChecksumCRC32C = aws.ToString(out.ChecksumCRC32C) - o.ChecksumSHA1 = aws.ToString(out.ChecksumSHA1) - o.ChecksumSHA256 = aws.ToString(out.ChecksumSHA256) - o.ContentDisposition = aws.ToString(out.ContentDisposition) - o.ContentEncoding = aws.ToString(out.ContentEncoding) - o.ContentLanguage = aws.ToString(out.ContentLanguage) - o.ContentType = aws.ToString(out.ContentType) - o.ETag = aws.ToString(out.ETag) - o.Expiration = aws.ToString(out.Expiration) - o.ExpiresString = aws.ToString(out.ExpiresString) - o.Restore = aws.ToString(out.Restore) - o.SSECustomerAlgorithm = aws.ToString(out.SSECustomerAlgorithm) - o.SSECustomerKeyMD5 = aws.ToString(out.SSECustomerKeyMD5) - o.SSEKMSKeyID = aws.ToString(out.SSEKMSKeyId) - o.VersionID = aws.ToString(out.VersionId) - o.WebsiteRedirectLocation = aws.ToString(out.WebsiteRedirectLocation) - o.BucketKeyEnabled = aws.ToBool(out.BucketKeyEnabled) - o.DeleteMarker = aws.ToBool(out.DeleteMarker) - o.MissingMeta = aws.ToInt32(out.MissingMeta) - o.PartsCount = aws.ToInt32(out.PartsCount) - o.ContentLength = getTotalBytes(out) - o.Body = body - o.Expires = aws.ToTime(out.Expires) - o.LastModified = aws.ToTime(out.LastModified) - o.ObjectLockRetainUntilDate = aws.ToTime(out.ObjectLockRetainUntilDate) + o.DeleteMarker = out.DeleteMarker + o.ETag = out.ETag + o.Expiration = out.Expiration + o.Expires = out.Expires + o.ExpiresString = out.ExpiresString + o.LastModified = out.LastModified o.Metadata = out.Metadata + o.MissingMeta = out.MissingMeta o.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatus(out.ObjectLockLegalHoldStatus) o.ObjectLockMode = types.ObjectLockMode(out.ObjectLockMode) + o.ObjectLockRetainUntilDate = out.ObjectLockRetainUntilDate + o.PartsCount = out.PartsCount o.ReplicationStatus = types.ReplicationStatus(out.ReplicationStatus) o.RequestCharged = types.RequestCharged(out.RequestCharged) + o.Restore = out.Restore + o.SSECustomerAlgorithm = out.SSECustomerAlgorithm + o.SSECustomerKeyMD5 = out.SSECustomerKeyMD5 + o.SSEKMSKeyID = out.SSEKMSKeyId o.ServerSideEncryption = types.ServerSideEncryption(out.ServerSideEncryption) o.StorageClass = types.StorageClass(out.StorageClass) - o.ResultMetadata = out.ResultMetadata.Clone() + o.TagCount = out.TagCount + o.VersionID = out.VersionId + o.WebsiteRedirectLocation = out.WebsiteRedirectLocation + o.ResultMetadata = out.ResultMetadata } // GetObject downloads an object from S3, intelligently splitting large @@ -613,10 +618,6 @@ func (g *getter) get(ctx context.Context) (out *GetObjectOutput, err error) { ) }} - if g.in.PartNumber > 0 { - return g.singleDownload(ctx, clientOptions...) - } - r := &concurrentReader{ ctx: ctx, buf: make(map[int32]*outChunk), @@ -628,13 +629,10 @@ func (g *getter) get(ctx context.Context) (out *GetObjectOutput, err error) { output := &GetObjectOutput{} if g.options.GetObjectType == types.GetObjectParts { - if g.in.Range != "" { - return g.singleDownload(ctx, clientOptions...) - } // must know the part size before creating stream reader out, err := g.options.S3.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: aws.String(g.in.Bucket), - Key: aws.String(g.in.Key), + Bucket: g.in.Bucket, + Key: g.in.Key, PartNumber: aws.Int32(1), }, clientOptions...) if err != nil { @@ -643,7 +641,7 @@ func (g *getter) get(ctx context.Context) (out *GetObjectOutput, err error) { output.mapFromHeadObjectOutput(out, g.in.ChecksumMode, !g.options.DisableChecksumValidation, r) contentLength := getTotalBytes(out) - output.ContentRange = fmt.Sprintf("bytes=0-%d/%d", contentLength-1, contentLength) + output.ContentRange = aws.String(fmt.Sprintf("bytes=0-%d/%d", contentLength-1, contentLength)) partsCount := max(aws.ToInt32(out.PartsCount), 1) partSize := max(aws.ToInt64(out.ContentLength), 1) @@ -651,12 +649,12 @@ func (g *getter) get(ctx context.Context) (out *GetObjectOutput, err error) { capacity := sectionParts r.sectionParts = sectionParts r.partSize = partSize - r.setCapacity(min(capacity, partsCount)) + atomic.StoreInt32(&r.capacity, min(capacity, partsCount)) r.partsCount = partsCount } else { out, err := g.options.S3.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: aws.String(g.in.Bucket), - Key: aws.String(g.in.Key), + Bucket: g.in.Bucket, + Key: g.in.Key, }, clientOptions...) if err != nil { return nil, err @@ -665,36 +663,31 @@ func (g *getter) get(ctx context.Context) (out *GetObjectOutput, err error) { return g.singleDownload(ctx, clientOptions...) } total := aws.ToInt64(out.ContentLength) - var pos int64 - if g.in.Range != "" { - start, totalBytes, err := g.getDownloadRange() - if err != nil || start < 0 || start >= total || totalBytes > total || start >= totalBytes { - return nil, &errInvalidRange{ - max: total - 1, - } - } - pos = start - total = totalBytes - } - contentLength := total - pos + contentLength := total output.mapFromHeadObjectOutput(out, g.in.ChecksumMode, !g.options.DisableChecksumValidation, r) - output.ContentLength = contentLength - output.ContentRange = fmt.Sprintf("bytes=%d-%d/%d", pos, total-1, aws.ToInt64(out.ContentLength)) + output.ContentLength = aws.Int64(contentLength) + output.ContentRange = aws.String(fmt.Sprintf("bytes=0-%d/%d", total-1, aws.ToInt64(out.ContentLength))) partsCount := int32((contentLength-1)/g.options.PartSizeBytes + 1) sectionParts := int32(max(1, g.options.GetBufferSize/g.options.PartSizeBytes)) capacity := min(sectionParts, partsCount) r.partSize = g.options.PartSizeBytes - r.setCapacity(capacity) + atomic.StoreInt32(&r.capacity, capacity) r.partsCount = partsCount r.sectionParts = sectionParts r.totalBytes = total - r.pos = pos } r.etag = output.ETag output.Body = r + if output.ChecksumType == types.ChecksumTypeComposite { + output.ChecksumCRC32 = nil + output.ChecksumCRC32C = nil + output.ChecksumCRC64NVME = nil + output.ChecksumSHA1 = nil + output.ChecksumSHA256 = nil + } return output, nil } @@ -732,19 +725,3 @@ func getTotalBytes(resp *s3.HeadObjectOutput) int64 { } return total } - -func (g *getter) getDownloadRange() (int64, int64, error) { - parts := strings.Split(strings.Split(g.in.Range, "=")[1], "-") - - start, err := strconv.ParseInt(parts[0], 10, 64) - if err != nil { - return 0, 0, err - } - - end, err := strconv.ParseInt(parts[1], 10, 64) - if err != nil { - return 0, 0, err - } - - return start, end + 1, nil -} diff --git a/feature/s3/transfermanager/api_op_UploadDirectory.go b/feature/s3/transfermanager/api_op_UploadDirectory.go index f720a478c56..3ebca7271fb 100644 --- a/feature/s3/transfermanager/api_op_UploadDirectory.go +++ b/feature/s3/transfermanager/api_op_UploadDirectory.go @@ -5,29 +5,31 @@ import ( "fmt" "os" "path/filepath" - "strings" "sync" + "sync/atomic" + + "github.com/aws/aws-sdk-go-v2/aws" ) // UploadDirectoryInput represents a request to the UploadDirectory() call type UploadDirectoryInput struct { // Bucket where objects are uploaded to - Bucket string + Bucket *string // The source directory to upload - Source string + Source *string // Whether to follow symbolic links when traversing the file tree. - FollowSymbolicLinks bool + FollowSymbolicLinks *bool // Whether to recursively upload directories. If set to false by // default, only top level files under source folder will be uplaoded; // otherwise all files under subfolders will be uploaded - Recursive bool + Recursive *bool // The S3 key prefix to use for each object. If not provided, files // will be uploaded to the root of the bucket - KeyPrefix string + KeyPrefix *string // A callback func to allow users to filter out unwanted files // according to bool returned from the function @@ -37,10 +39,11 @@ type UploadDirectoryInput struct { // PutObjectInput that the S3 Transfer Manager generates. Callback PutRequestCallback - // The s3 delimeter contatenating each object key based on local file separator - // and file's relative path. If a non-defualt delimiter is used, it can not be - // included in any subfolders or files, which will cause error otherwise - S3Delimiter string + // A callback function to allow users to control the upload behavior + // when there are failed objects. The directory upload will be terminated + // if its function returns non-nil error and will continue skipping current + // failed object if the function returns nil + FailurePolicy UploadDirectoryFailurePolicy } // FileFilter is the callback to allow users to filter out unwanted files. @@ -58,10 +61,41 @@ type PutRequestCallback interface { UpdateRequest(*UploadObjectInput) } +// UploadDirectoryFailurePolicy is a callback to allow users to control the +// upload behavior when there are failed objects. It is invoked for every failed object. +// If the OnUploadFailed returns non-nil error, uploader will cancel all ongoing +// single object upload requests and terminate the upload directory process, if it returns nil +// error, uploader will count the current request as a failed object downloaded but continue +// uploading other objects. +type UploadDirectoryFailurePolicy interface { + OnUploadFailed(*UploadDirectoryInput, *UploadObjectInput, error) error +} + +// TerminateUploadPolicy implements UploadDirectoryFailurePolicy to cancel all other ongoing +// objects upload and terminate the upload directory call +type TerminateUploadPolicy struct{} + +// OnUploadFailed returns the initial err +func (TerminateUploadPolicy) OnUploadFailed(directoryInput *UploadDirectoryInput, objectInput *UploadObjectInput, err error) error { + return err +} + +// IgnoreUploadFailurePolicy implements the UploadDirectoryFailurePolicy to ignore single object upload error +// and continue uploading other objects +type IgnoreUploadFailurePolicy struct{} + +// OnUploadFailed ignores input error and return nil +func (IgnoreUploadFailurePolicy) OnUploadFailed(*UploadDirectoryInput, *UploadObjectInput, error) error { + return nil +} + // UploadDirectoryOutput represents a response from the UploadDirectory() call type UploadDirectoryOutput struct { // Total number of objects successfully uploaded - ObjectsUploaded int + ObjectsUploaded int64 + + // Total number of objects failed to upload + ObjectsFailed int64 } // UploadDirectory traverses a local directory recursively/non-recursively and intelligently @@ -72,12 +106,12 @@ type UploadDirectoryOutput struct { // upload. These options are copies of the original Options instance, the client of which UploadDirectory is called from. // Modifying the options will not impact the original Client and Options instance. func (c *Client) UploadDirectory(ctx context.Context, input *UploadDirectoryInput, opts ...func(*Options)) (*UploadDirectoryOutput, error) { - fileInfo, err := os.Stat(input.Source) + fileInfo, err := os.Stat(aws.ToString(input.Source)) if err != nil { return nil, fmt.Errorf("error when getting source info: %v", err) } if !fileInfo.IsDir() { - return nil, fmt.Errorf("the source path %s doesn't point to a valid directory", input.Source) + return nil, fmt.Errorf("the source path %s doesn't point to a valid directory", aws.ToString(input.Source)) } i := directoryUploader{c: c, in: input, options: c.options.Copy()} @@ -89,11 +123,13 @@ func (c *Client) UploadDirectory(ctx context.Context, input *UploadDirectoryInpu } type directoryUploader struct { - c *Client - options Options - in *UploadDirectoryInput + c *Client + options Options + in *UploadDirectoryInput + failurePolicy UploadDirectoryFailurePolicy - filesUploaded int + filesUploaded int64 + filesFailed int64 traversed map[string]interface{} err error @@ -114,10 +150,10 @@ func (u *directoryUploader) uploadDirectory(ctx context.Context) (*UploadDirecto go u.uploadFile(ctx, ch) } - if u.in.Recursive { - u.traverse(u.in.Source, u.in.KeyPrefix, ch) + if aws.ToBool(u.in.Recursive) { + u.traverse(aws.ToString(u.in.Source), aws.ToString(u.in.KeyPrefix), ch) } else { - files, err := u.traverseFolder(u.in.Source) + files, err := u.traverseFolder(aws.ToString(u.in.Source)) if err != nil { return nil, err } @@ -127,7 +163,7 @@ func (u *directoryUploader) uploadDirectory(ctx context.Context) (*UploadDirecto break } - path := filepath.Join(u.in.Source, f) + path := filepath.Join(aws.ToString(u.in.Source), f) absPath, err := u.getAbsPath(path) if err != nil { u.setErr(fmt.Errorf("error when getting abs path of file %s: %v", path, err)) @@ -148,14 +184,10 @@ func (u *directoryUploader) uploadDirectory(ctx context.Context) (*UploadDirecto if u.in.Filter != nil && !u.in.Filter.FilterFile(path) { continue } - if u.in.S3Delimiter != "/" && strings.Contains(f, u.in.S3Delimiter) { - u.setErr(fmt.Errorf("file %s contains delimiter %s", f, u.in.S3Delimiter)) - break - } - if u.in.KeyPrefix == "" { + if kp := aws.ToString(u.in.KeyPrefix); kp == "" { ch <- fileEntry{f, absPath} } else { - ch <- fileEntry{u.in.KeyPrefix + u.in.S3Delimiter + f, absPath} + ch <- fileEntry{kp + "/" + f, absPath} } } } @@ -169,18 +201,20 @@ func (u *directoryUploader) uploadDirectory(ctx context.Context) (*UploadDirecto out := &UploadDirectoryOutput{ ObjectsUploaded: u.filesUploaded, + ObjectsFailed: u.filesFailed, } u.emitter.Complete(ctx, out) return out, nil } func (u *directoryUploader) init() { - if u.in.S3Delimiter == "" { - u.in.S3Delimiter = "/" - } - u.traversed = make(map[string]interface{}) + u.failurePolicy = TerminateUploadPolicy{} + if u.in.FailurePolicy != nil { + u.failurePolicy = u.in.FailurePolicy + } + u.emitter = &directoryObjectsProgressEmitter{ Listeners: u.options.DirectoryProgressListeners, } @@ -207,12 +241,12 @@ func (u *directoryUploader) traverse(path, keyPrefix string, ch chan fileEntry) } var key string - if path == u.in.Source { + if path == aws.ToString(u.in.Source) { key = keyPrefix } else if keyPrefix == "" { key = filepath.Base(path) } else { - key = keyPrefix + u.in.S3Delimiter + filepath.Base(path) + key = keyPrefix + "/" + filepath.Base(path) } fileInfo, err := os.Lstat(absPath) if err != nil { @@ -226,22 +260,12 @@ func (u *directoryUploader) traverse(path, keyPrefix string, ch chan fileEntry) return } for _, f := range subFiles { - if d := u.in.S3Delimiter; d != "/" && strings.Contains(f, d) { - u.setErr(fmt.Errorf("file %s contains delimiter %s", f, d)) - return - } u.traverse(filepath.Join(path, f), key, ch) } } else { if u.in.Filter != nil && !u.in.Filter.FilterFile(path) { return } - if u.in.S3Delimiter != "/" { - if n, d := filepath.Base(path), u.in.S3Delimiter; strings.Contains(n, d) { - u.setErr(fmt.Errorf("file %s contains delimiter %s", n, d)) - return - } - } ch <- fileEntry{key, absPath} } } @@ -255,7 +279,7 @@ func (u *directoryUploader) getAbsPath(path string) (string, error) { } if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink { - if !u.in.FollowSymbolicLinks { + if !aws.ToBool(u.in.FollowSymbolicLinks) { return "", nil } path, err = u.traverseSymlink(path) @@ -342,7 +366,7 @@ func (u *directoryUploader) uploadFile(ctx context.Context, ch chan fileEntry) { } input := &UploadObjectInput{ Bucket: u.in.Bucket, - Key: data.key, + Key: aws.String(data.key), Body: f, } if u.in.Callback != nil { @@ -350,25 +374,24 @@ func (u *directoryUploader) uploadFile(ctx context.Context, ch chan fileEntry) { } out, err := u.c.UploadObject(ctx, input) if err != nil { - u.setErr(fmt.Errorf("error when uploading file %s: %v", data.path, err)) + err = u.failurePolicy.OnUploadFailed(u.in, input, err) + if err != nil { + u.setErr(fmt.Errorf("error when uploading file %s: %v", data.path, err)) + } else { + // this failed object is ignored, just increase the failure count + atomic.AddInt64(&u.filesFailed, 1) + } continue } u.progressOnce.Do(func() { u.emitter.Start(ctx, u.in) }) - u.incrFilesUploaded(1) - u.emitter.ObjectsTransferred(ctx, out.ContentLength) + atomic.AddInt64(&u.filesUploaded, 1) + u.emitter.ObjectsTransferred(ctx, aws.ToInt64(out.ContentLength)) } } -func (u *directoryUploader) incrFilesUploaded(n int) { - u.mu.Lock() - defer u.mu.Unlock() - - u.filesUploaded += n -} - func (u *directoryUploader) setErr(err error) { u.mu.Lock() defer u.mu.Unlock() diff --git a/feature/s3/transfermanager/api_op_UploadDirectory_integ_test.go b/feature/s3/transfermanager/api_op_UploadDirectory_integ_test.go index cff128ec93b..ed3dce5a405 100644 --- a/feature/s3/transfermanager/api_op_UploadDirectory_integ_test.go +++ b/feature/s3/transfermanager/api_op_UploadDirectory_integ_test.go @@ -40,20 +40,6 @@ func TestInteg_UploadDirectory(t *testing.T) { ExpectFilesUploaded: 3, ExpectKeys: []string{"bla/foo", "bla/to/bar", "bla/to/the/baz"}, }, - "multi file recursive with prefix and custom delimiter": { - FilesSize: map[string]int64{ - "foo": 2 * 1024 * 1024, - "to/bar": 10 * 1024 * 1024, - "to/the/baz": 20 * 1024 * 1024, - "too/the/zoo": 5 * 1024 * 1024, - }, - Source: "integ-dir", - Recursive: true, - KeyPrefix: "bla", - Delimiter: "#", - ExpectFilesUploaded: 4, - ExpectKeys: []string{"bla#foo", "bla#to#bar", "bla#to#the#baz", "bla#too#the#zoo"}, - }, } for name, c := range cases { diff --git a/feature/s3/transfermanager/api_op_UploadObject.go b/feature/s3/transfermanager/api_op_UploadObject.go index 847e3b4d087..6c578e02fd5 100644 --- a/feature/s3/transfermanager/api_op_UploadObject.go +++ b/feature/s3/transfermanager/api_op_UploadObject.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "io" + "log" "sort" "sync" "time" @@ -78,10 +79,10 @@ func (m *multipartUploadError) UploadID() string { // of s3 PutObject and CreateMultipartUpload input type UploadObjectInput struct { // Bucket the object is uploaded into - Bucket string + Bucket *string // Object key for which the PUT action was initiated - Key string + Key *string // Object data Body io.Reader @@ -124,13 +125,13 @@ type UploadObjectInput struct { // for S3 Bucket Key. // // This functionality is not supported for directory buckets. - BucketKeyEnabled bool + BucketKeyEnabled *bool // Can be used to specify caching behavior along the request/reply chain. For more // information, see [http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9]. // // [http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9]: http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 - CacheControl string + CacheControl *string // Indicates the algorithm used to create the checksum for the object when you use // the SDK. This header will not provide any additional functionality if you don't @@ -163,71 +164,144 @@ type UploadObjectInput struct { // [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html ChecksumAlgorithm types.ChecksumAlgorithm + // This header can be used as a data integrity check to verify that the data + // received is the same data that was originally sent. This header specifies the + // Base64 encoded, 32-bit CRC32 checksum of the object. For more information, see [Checking object integrity] + // in the Amazon S3 User Guide. + // + // [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + ChecksumCRC32 *string + + // This header can be used as a data integrity check to verify that the data + // received is the same data that was originally sent. This header specifies the + // Base64 encoded, 32-bit CRC32C checksum of the object. For more information, see [Checking object integrity] + // in the Amazon S3 User Guide. + // + // [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + ChecksumCRC32C *string + + // This header can be used as a data integrity check to verify that the data + // received is the same data that was originally sent. This header specifies the + // Base64 encoded, 64-bit CRC64NVME checksum of the object. The CRC64NVME checksum + // is always a full object checksum. For more information, see [Checking object integrity in the Amazon S3 User Guide]. + // + // [Checking object integrity in the Amazon S3 User Guide]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + ChecksumCRC64NVME *string + + // This header can be used as a data integrity check to verify that the data + // received is the same data that was originally sent. This header specifies the + // Base64 encoded, 160-bit SHA1 digest of the object. For more information, see [Checking object integrity] + // in the Amazon S3 User Guide. + // + // [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + ChecksumSHA1 *string + + // This header can be used as a data integrity check to verify that the data + // received is the same data that was originally sent. This header specifies the + // Base64 encoded, 256-bit SHA256 digest of the object. For more information, see [Checking object integrity] + // in the Amazon S3 User Guide. + // + // [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + ChecksumSHA256 *string + // Size of the body in bytes. This parameter is useful when the size of the body // cannot be determined automatically. For more information, see [https://www.rfc-editor.org/rfc/rfc9110.html#name-content-length]. // // [https://www.rfc-editor.org/rfc/rfc9110.html#name-content-length]: https://www.rfc-editor.org/rfc/rfc9110.html#name-content-length - ContentLength int64 + ContentLength *int64 // Specifies presentational information for the object. For more information, see [https://www.rfc-editor.org/rfc/rfc6266#section-4]. // // [https://www.rfc-editor.org/rfc/rfc6266#section-4]: https://www.rfc-editor.org/rfc/rfc6266#section-4 - ContentDisposition string + ContentDisposition *string // Specifies what content encodings have been applied to the object and thus what // decoding mechanisms must be applied to obtain the media-type referenced by the // Content-Type header field. For more information, see [https://www.rfc-editor.org/rfc/rfc9110.html#field.content-encoding]. // // [https://www.rfc-editor.org/rfc/rfc9110.html#field.content-encoding]: https://www.rfc-editor.org/rfc/rfc9110.html#field.content-encoding - ContentEncoding string + ContentEncoding *string // The language the content is in. - ContentLanguage string + ContentLanguage *string // A standard MIME type describing the format of the contents. For more // information, see [https://www.rfc-editor.org/rfc/rfc9110.html#name-content-type]. // // [https://www.rfc-editor.org/rfc/rfc9110.html#name-content-type]: https://www.rfc-editor.org/rfc/rfc9110.html#name-content-type - ContentType string + ContentType *string // The account ID of the expected bucket owner. If the account ID that you provide // does not match the actual owner of the bucket, the request fails with the HTTP // status code 403 Forbidden (access denied). - ExpectedBucketOwner string + ExpectedBucketOwner *string // The date and time at which the object is no longer cacheable. For more // information, see [https://www.rfc-editor.org/rfc/rfc7234#section-5.3]. // // [https://www.rfc-editor.org/rfc/rfc7234#section-5.3]: https://www.rfc-editor.org/rfc/rfc7234#section-5.3 - Expires time.Time + Expires *time.Time // Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object. // // - This functionality is not supported for directory buckets. // // - This functionality is not supported for Amazon S3 on Outposts. - GrantFullControl string + GrantFullControl *string // Allows grantee to read the object data and its metadata. // // - This functionality is not supported for directory buckets. // // - This functionality is not supported for Amazon S3 on Outposts. - GrantRead string + GrantRead *string // Allows grantee to read the object ACL. // // - This functionality is not supported for directory buckets. // // - This functionality is not supported for Amazon S3 on Outposts. - GrantReadACP string + GrantReadACP *string // Allows grantee to write the ACL for the applicable object. // // - This functionality is not supported for directory buckets. // // - This functionality is not supported for Amazon S3 on Outposts. - GrantWriteACP string + GrantWriteACP *string + + // Uploads the object only if the ETag (entity tag) value provided during the + // WRITE operation matches the ETag of the object in S3. If the ETag values do not + // match, the operation returns a 412 Precondition Failed error. + // + // If a conflicting operation occurs during the upload S3 returns a 409 + // ConditionalRequestConflict response. On a 409 failure you should fetch the + // object's ETag and retry the upload. + // + // Expects the ETag value as a string. + // + // For more information about conditional requests, see [RFC 7232], or [Conditional requests] in the Amazon S3 + // User Guide. + // + // [Conditional requests]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-requests.html + // [RFC 7232]: https://tools.ietf.org/html/rfc7232 + IfMatch *string + + // Uploads the object only if the object key name does not already exist in the + // bucket specified. Otherwise, Amazon S3 returns a 412 Precondition Failed error. + // + // If a conflicting operation occurs during the upload S3 returns a 409 + // ConditionalRequestConflict response. On a 409 failure you should retry the + // upload. + // + // Expects the '*' (asterisk) character. + // + // For more information about conditional requests, see [RFC 7232], or [Conditional requests] in the Amazon S3 + // User Guide. + // + // [Conditional requests]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-requests.html + // [RFC 7232]: https://tools.ietf.org/html/rfc7232 + IfNoneMatch *string // A map of metadata to store with the object in S3. Metadata map[string]string @@ -249,7 +323,7 @@ type UploadObjectInput struct { // formatted as a timestamp parameter. // // This functionality is not supported for directory buckets. - ObjectLockRetainUntilDate time.Time + ObjectLockRetainUntilDate *time.Time // Confirms that the requester knows that they will be charged for the request. // Bucket owners need not specify this parameter in their requests. If either the @@ -266,7 +340,7 @@ type UploadObjectInput struct { // Specifies the algorithm to use when encrypting the object (for example, AES256 ). // // This functionality is not supported for directory buckets. - SSECustomerAlgorithm string + SSECustomerAlgorithm *string // Specifies the customer-provided encryption key for Amazon S3 to use in // encrypting data. This value is used to store the object and then it is @@ -275,7 +349,14 @@ type UploadObjectInput struct { // x-amz-server-side-encryption-customer-algorithm header. // // This functionality is not supported for directory buckets. - SSECustomerKey string + SSECustomerKey *string + + // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. + // Amazon S3 uses this header for a message integrity check to ensure that the + // encryption key was transmitted without error. + // + // This functionality is not supported for directory buckets. + SSECustomerKeyMD5 *string // Specifies the Amazon Web Services KMS Encryption Context to use for object // encryption. The value of this header is a base64-encoded UTF-8 string holding @@ -285,7 +366,7 @@ type UploadObjectInput struct { // explicitly added during CopyObject operations. // // This functionality is not supported for directory buckets. - SSEKMSEncryptionContext string + SSEKMSEncryptionContext *string // If x-amz-server-side-encryption has a valid value of aws:kms or aws:kms:dsse , // this header specifies the ID (Key ID, Key ARN, or Key Alias) of the Key @@ -298,7 +379,7 @@ type UploadObjectInput struct { // and not just the ID. // // This functionality is not supported for directory buckets. - SSEKMSKeyID string + SSEKMSKeyID *string // The server-side encryption algorithm that was used when you store this object // in Amazon S3 (for example, AES256 , aws:kms , aws:kms:dsse ). @@ -336,7 +417,7 @@ type UploadObjectInput struct { // parameters. (For example, "Key1=Value1") // // This functionality is not supported for directory buckets. - Tagging string + Tagging *string // If the bucket is configured as a website, redirects requests for this object to // another object in the same bucket or to an external URL. Amazon S3 stores the @@ -361,7 +442,7 @@ type UploadObjectInput struct { // [How to Configure Website Page Redirects]: https://docs.aws.amazon.com/AmazonS3/latest/dev/how-to-page-redirect.html // [Hosting Websites on Amazon S3]: https://docs.aws.amazon.com/AmazonS3/latest/dev/WebsiteHosting.html // [Object Key and Metadata]: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html - WebsiteRedirectLocation string + WebsiteRedirectLocation *string } // map non-zero string to *string @@ -382,119 +463,111 @@ func nztime(t time.Time) *time.Time { func (i UploadObjectInput) mapSingleUploadInput(body io.Reader, checksumAlgorithm types.ChecksumAlgorithm) *s3.PutObjectInput { input := &s3.PutObjectInput{ - Bucket: aws.String(i.Bucket), - Key: aws.String(i.Key), - Body: body, - } - if i.ACL != "" { - input.ACL = s3types.ObjectCannedACL(i.ACL) + Bucket: i.Bucket, + Key: i.Key, + Body: body, + ACL: s3types.ObjectCannedACL(i.ACL), + BucketKeyEnabled: i.BucketKeyEnabled, + CacheControl: i.CacheControl, + ChecksumCRC32: i.ChecksumCRC32, + ChecksumCRC32C: i.ChecksumCRC32C, + ChecksumCRC64NVME: i.ChecksumCRC64NVME, + ChecksumSHA1: i.ChecksumSHA1, + ChecksumSHA256: i.ChecksumSHA256, + ContentDisposition: i.ContentDisposition, + ContentEncoding: i.ContentEncoding, + ContentLanguage: i.ContentLanguage, + ContentType: i.ContentType, + ExpectedBucketOwner: i.ExpectedBucketOwner, + Expires: i.Expires, + GrantFullControl: i.GrantFullControl, + GrantRead: i.GrantRead, + GrantReadACP: i.GrantReadACP, + GrantWriteACP: i.GrantWriteACP, + IfMatch: i.IfMatch, + IfNoneMatch: i.IfNoneMatch, + Metadata: i.Metadata, + ObjectLockLegalHoldStatus: s3types.ObjectLockLegalHoldStatus(i.ObjectLockLegalHoldStatus), + ObjectLockMode: s3types.ObjectLockMode(i.ObjectLockMode), + ObjectLockRetainUntilDate: i.ObjectLockRetainUntilDate, + RequestPayer: s3types.RequestPayer(i.RequestPayer), + SSECustomerAlgorithm: i.SSECustomerAlgorithm, + SSECustomerKey: i.SSECustomerKey, + SSECustomerKeyMD5: i.SSECustomerKeyMD5, + SSEKMSEncryptionContext: i.SSEKMSEncryptionContext, + SSEKMSKeyId: i.SSEKMSKeyID, + ServerSideEncryption: s3types.ServerSideEncryption(i.ServerSideEncryption), + StorageClass: s3types.StorageClass(i.StorageClass), + Tagging: i.Tagging, + WebsiteRedirectLocation: i.WebsiteRedirectLocation, } if i.ChecksumAlgorithm != "" { input.ChecksumAlgorithm = s3types.ChecksumAlgorithm(i.ChecksumAlgorithm) } else { input.ChecksumAlgorithm = s3types.ChecksumAlgorithm(checksumAlgorithm) } - if i.ObjectLockLegalHoldStatus != "" { - input.ObjectLockLegalHoldStatus = s3types.ObjectLockLegalHoldStatus(i.ObjectLockLegalHoldStatus) - } - if i.ObjectLockMode != "" { - input.ObjectLockMode = s3types.ObjectLockMode(i.ObjectLockMode) - } - if i.RequestPayer != "" { - input.RequestPayer = s3types.RequestPayer(i.RequestPayer) - } - if i.ServerSideEncryption != "" { - input.ServerSideEncryption = s3types.ServerSideEncryption(i.ServerSideEncryption) - } - if i.StorageClass != "" { - input.StorageClass = s3types.StorageClass(i.StorageClass) - } - input.BucketKeyEnabled = aws.Bool(i.BucketKeyEnabled) - input.CacheControl = nzstring(i.CacheControl) - input.ContentDisposition = nzstring(i.ContentDisposition) - input.ContentEncoding = nzstring(i.ContentEncoding) - input.ContentLanguage = nzstring(i.ContentLanguage) - input.ContentType = nzstring(i.ContentType) - input.ExpectedBucketOwner = nzstring(i.ExpectedBucketOwner) - input.GrantFullControl = nzstring(i.GrantFullControl) - input.GrantRead = nzstring(i.GrantRead) - input.GrantReadACP = nzstring(i.GrantReadACP) - input.GrantWriteACP = nzstring(i.GrantWriteACP) - input.Metadata = i.Metadata - input.SSECustomerAlgorithm = nzstring(i.SSECustomerAlgorithm) - input.SSECustomerKey = nzstring(i.SSECustomerKey) - input.SSEKMSEncryptionContext = nzstring(i.SSEKMSEncryptionContext) - input.SSEKMSKeyId = nzstring(i.SSEKMSKeyID) - input.Tagging = nzstring(i.Tagging) - input.WebsiteRedirectLocation = nzstring(i.WebsiteRedirectLocation) - input.Expires = nztime(i.Expires) - input.ObjectLockRetainUntilDate = nztime(i.ObjectLockRetainUntilDate) + return input } func (i UploadObjectInput) mapCreateMultipartUploadInput(checksumAlgorithm types.ChecksumAlgorithm) *s3.CreateMultipartUploadInput { input := &s3.CreateMultipartUploadInput{ - Bucket: aws.String(i.Bucket), - Key: aws.String(i.Key), - } - if i.ACL != "" { - input.ACL = s3types.ObjectCannedACL(i.ACL) + Bucket: i.Bucket, + Key: i.Key, + ACL: s3types.ObjectCannedACL(i.ACL), + BucketKeyEnabled: i.BucketKeyEnabled, + CacheControl: i.CacheControl, + ContentDisposition: i.ContentDisposition, + ContentEncoding: i.ContentEncoding, + ContentLanguage: i.ContentLanguage, + ContentType: i.ContentType, + ExpectedBucketOwner: i.ExpectedBucketOwner, + Expires: i.Expires, + GrantFullControl: i.GrantFullControl, + GrantRead: i.GrantRead, + GrantReadACP: i.GrantReadACP, + GrantWriteACP: i.GrantWriteACP, + Metadata: i.Metadata, + ObjectLockLegalHoldStatus: s3types.ObjectLockLegalHoldStatus(i.ObjectLockLegalHoldStatus), + ObjectLockMode: s3types.ObjectLockMode(i.ObjectLockMode), + ObjectLockRetainUntilDate: i.ObjectLockRetainUntilDate, + RequestPayer: s3types.RequestPayer(i.RequestPayer), + SSECustomerAlgorithm: i.SSECustomerAlgorithm, + SSECustomerKey: i.SSECustomerKey, + SSECustomerKeyMD5: i.SSECustomerKeyMD5, + SSEKMSEncryptionContext: i.SSEKMSEncryptionContext, + SSEKMSKeyId: i.SSEKMSKeyID, + ServerSideEncryption: s3types.ServerSideEncryption(i.ServerSideEncryption), + StorageClass: s3types.StorageClass(i.StorageClass), + Tagging: i.Tagging, + WebsiteRedirectLocation: i.WebsiteRedirectLocation, } if i.ChecksumAlgorithm != "" { input.ChecksumAlgorithm = s3types.ChecksumAlgorithm(i.ChecksumAlgorithm) } else { input.ChecksumAlgorithm = s3types.ChecksumAlgorithm(checksumAlgorithm) } - if i.ObjectLockLegalHoldStatus != "" { - input.ObjectLockLegalHoldStatus = s3types.ObjectLockLegalHoldStatus(i.ObjectLockLegalHoldStatus) - } - if i.ObjectLockMode != "" { - input.ObjectLockMode = s3types.ObjectLockMode(i.ObjectLockMode) - } - if i.RequestPayer != "" { - input.RequestPayer = s3types.RequestPayer(i.RequestPayer) - } - if i.ServerSideEncryption != "" { - input.ServerSideEncryption = s3types.ServerSideEncryption(i.ServerSideEncryption) - } - if i.StorageClass != "" { - input.StorageClass = s3types.StorageClass(i.StorageClass) - } - input.BucketKeyEnabled = aws.Bool(i.BucketKeyEnabled) - input.CacheControl = nzstring(i.CacheControl) - input.ContentDisposition = nzstring(i.ContentDisposition) - input.ContentEncoding = nzstring(i.ContentEncoding) - input.ContentLanguage = nzstring(i.ContentLanguage) - input.ContentType = nzstring(i.ContentType) - input.ExpectedBucketOwner = nzstring(i.ExpectedBucketOwner) - input.GrantFullControl = nzstring(i.GrantFullControl) - input.GrantRead = nzstring(i.GrantRead) - input.GrantReadACP = nzstring(i.GrantReadACP) - input.GrantWriteACP = nzstring(i.GrantWriteACP) - input.Metadata = i.Metadata - input.SSECustomerAlgorithm = nzstring(i.SSECustomerAlgorithm) - input.SSECustomerKey = nzstring(i.SSECustomerKey) - input.SSEKMSEncryptionContext = nzstring(i.SSEKMSEncryptionContext) - input.SSEKMSKeyId = nzstring(i.SSEKMSKeyID) - input.Tagging = nzstring(i.Tagging) - input.WebsiteRedirectLocation = nzstring(i.WebsiteRedirectLocation) - input.Expires = nztime(i.Expires) - input.ObjectLockRetainUntilDate = nztime(i.ObjectLockRetainUntilDate) return input } func (i UploadObjectInput) mapCompleteMultipartUploadInput(uploadID *string, completedParts completedParts) *s3.CompleteMultipartUploadInput { input := &s3.CompleteMultipartUploadInput{ - Bucket: aws.String(i.Bucket), - Key: aws.String(i.Key), - UploadId: uploadID, - } - if i.RequestPayer != "" { - input.RequestPayer = s3types.RequestPayer(i.RequestPayer) + Bucket: i.Bucket, + Key: i.Key, + UploadId: uploadID, + ChecksumCRC32: i.ChecksumCRC32, + ChecksumCRC32C: i.ChecksumCRC32C, + ChecksumCRC64NVME: i.ChecksumCRC64NVME, + ChecksumSHA1: i.ChecksumSHA1, + ChecksumSHA256: i.ChecksumSHA256, + ExpectedBucketOwner: i.ExpectedBucketOwner, + IfMatch: i.IfMatch, + IfNoneMatch: i.IfNoneMatch, + RequestPayer: s3types.RequestPayer(i.RequestPayer), + SSECustomerAlgorithm: i.SSECustomerAlgorithm, + SSECustomerKey: i.SSECustomerKey, + SSECustomerKeyMD5: i.SSECustomerKeyMD5, } - input.ExpectedBucketOwner = nzstring(i.ExpectedBucketOwner) - input.SSECustomerAlgorithm = nzstring(i.SSECustomerAlgorithm) - input.SSECustomerKey = nzstring(i.SSECustomerKey) var parts []s3types.CompletedPart for _, part := range completedParts { parts = append(parts, part.MapCompletedPart()) @@ -507,11 +580,16 @@ func (i UploadObjectInput) mapCompleteMultipartUploadInput(uploadID *string, com func (i UploadObjectInput) mapUploadPartInput(body io.Reader, partNum *int32, uploadID *string, checksumAlgorithm types.ChecksumAlgorithm) *s3.UploadPartInput { input := &s3.UploadPartInput{ - Bucket: aws.String(i.Bucket), - Key: aws.String(i.Key), - Body: body, - PartNumber: partNum, - UploadId: uploadID, + Bucket: i.Bucket, + Key: i.Key, + Body: body, + PartNumber: partNum, + UploadId: uploadID, + ExpectedBucketOwner: i.ExpectedBucketOwner, + RequestPayer: s3types.RequestPayer(i.RequestPayer), + SSECustomerAlgorithm: i.SSECustomerAlgorithm, + SSECustomerKey: i.SSECustomerKey, + SSECustomerKeyMD5: i.SSECustomerKeyMD5, } if i.ChecksumAlgorithm != "" { input.ChecksumAlgorithm = s3types.ChecksumAlgorithm(i.ChecksumAlgorithm) @@ -519,20 +597,16 @@ func (i UploadObjectInput) mapUploadPartInput(body io.Reader, partNum *int32, up input.ChecksumAlgorithm = s3types.ChecksumAlgorithm(checksumAlgorithm) } - if i.RequestPayer != "" { - input.RequestPayer = s3types.RequestPayer(i.RequestPayer) - } - input.ExpectedBucketOwner = nzstring(i.ExpectedBucketOwner) - input.SSECustomerAlgorithm = nzstring(i.SSECustomerAlgorithm) - input.SSECustomerKey = nzstring(i.SSECustomerKey) return input } func (i *UploadObjectInput) mapAbortMultipartUploadInput(uploadID *string) *s3.AbortMultipartUploadInput { input := &s3.AbortMultipartUploadInput{ - Bucket: aws.String(i.Bucket), - Key: aws.String(i.Key), - UploadId: uploadID, + Bucket: i.Bucket, + Key: i.Key, + UploadId: uploadID, + ExpectedBucketOwner: i.ExpectedBucketOwner, + RequestPayer: s3types.RequestPayer(i.RequestPayer), } return input } @@ -540,57 +614,94 @@ func (i *UploadObjectInput) mapAbortMultipartUploadInput(uploadID *string) *s3.A // UploadObjectOutput represents a response from the PutObject() call. It contains common fields // of s3 PutObject and CompleteMultipartUpload output type UploadObjectOutput struct { - // The ID for a multipart upload to S3. In the case of an error the error - // can be cast to the MultiUploadFailure interface to extract the upload ID. - // Will be empty string if multipart upload was not used, and the object - // was uploaded as a single PutObject call. - UploadID string + // The bucket where the newly created object is put + Bucket *string - // The list of parts that were uploaded and their checksums. Will be empty - // if multipart upload was not used, and the object was uploaded as a - // single PutObject call. - CompletedParts []types.CompletedPart + // The object key of the newly created object. + Key *string // Indicates whether the uploaded object uses an S3 Bucket Key for server-side // encryption with Amazon Web Services KMS (SSE-KMS). - BucketKeyEnabled bool + BucketKeyEnabled *bool // The base64-encoded, 32-bit CRC32 checksum of the object. - ChecksumCRC32 string + ChecksumCRC32 *string // The base64-encoded, 32-bit CRC32C checksum of the object. - ChecksumCRC32C string + ChecksumCRC32C *string + + // The Base64 encoded, 64-bit CRC64NVME checksum of the object. + ChecksumCRC64NVME *string // The base64-encoded, 160-bit SHA-1 digest of the object. - ChecksumSHA1 string + ChecksumSHA1 *string // The base64-encoded, 256-bit SHA-256 digest of the object. - ChecksumSHA256 string - - // Total length of the object - ContentLength int64 + ChecksumSHA256 *string + + // This header specifies the checksum type of the object, which determines how + // part-level checksums are combined to create an object-level checksum for + // multipart objects. For PutObject uploads, the checksum type is always + // FULL_OBJECT . You can use this header as a data integrity check to verify that + // the checksum type that is received is the same checksum that was specified. For + // more information, see [Checking object integrity]in the Amazon S3 User Guide. + // + // [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + ChecksumType types.ChecksumType // Entity tag for the uploaded object. - ETag string + ETag *string // If the object expiration is configured, this will contain the expiration date // (expiry-date) and rule ID (rule-id). The value of rule-id is URL encoded. - Expiration string + Expiration *string - // The bucket where the newly created object is put - Bucket string + // The URI that identifies the newly created object. + Location *string - // The object key of the newly created object. - Key string + // The ID for a multipart upload to S3. In the case of an error the error + // can be cast to the MultiUploadFailure interface to extract the upload ID. + // Will be empty string if multipart upload was not used, and the object + // was uploaded as a single PutObject call. + UploadID *string + + // The list of parts that were uploaded and their checksums. Will be empty + // if multipart upload was not used, and the object was uploaded as a + // single PutObject call. + CompletedParts []types.CompletedPart + + // Total length of the object + ContentLength *int64 // If present, indicates that the requester was successfully charged for the // request. RequestCharged types.RequestCharged + // If server-side encryption with a customer-provided encryption key was + // requested, the response will include this header to confirm the encryption + // algorithm that's used. + // + // This functionality is not supported for directory buckets. + SSECustomerAlgorithm *string + + // If server-side encryption with a customer-provided encryption key was + // requested, the response will include this header to provide the round-trip + // message integrity verification of the customer-provided encryption key. + // + // This functionality is not supported for directory buckets. + SSECustomerKeyMD5 *string + + // If present, indicates the Amazon Web Services KMS Encryption Context to use for + // object encryption. The value of this header is a Base64 encoded string of a + // UTF-8 encoded JSON, which contains the encryption context as key-value pairs. + // This value is stored as object metadata and automatically gets passed on to + // Amazon Web Services KMS for future GetObject operations on this object. + SSEKMSEncryptionContext *string + // If present, specifies the ID of the Amazon Web Services Key Management Service // (Amazon Web Services KMS) symmetric customer managed customer master key (CMK) // that was used for the object. - SSEKMSKeyID string + SSEKMSKeyID *string // If you specified server-side encryption either with an Amazon S3-managed // encryption key or an Amazon Web Services KMS customer master key (CMK) in your @@ -601,47 +712,55 @@ type UploadObjectOutput struct { // The version of the object that was uploaded. Will only be populated if // the S3 Bucket is versioned. If the bucket is not versioned this field // will not be set. - VersionID string + VersionID *string // Metadata pertaining to the operation's result. ResultMetadata smithymiddleware.Metadata } -func (o *UploadObjectOutput) mapFromPutObjectOutput(out *s3.PutObjectOutput, bucket, key string, contentLength int64) { - o.BucketKeyEnabled = aws.ToBool(out.BucketKeyEnabled) - o.ChecksumCRC32 = aws.ToString(out.ChecksumCRC32) - o.ChecksumCRC32C = aws.ToString(out.ChecksumCRC32C) - o.ChecksumSHA1 = aws.ToString(out.ChecksumSHA1) - o.ChecksumSHA256 = aws.ToString(out.ChecksumSHA256) - o.ContentLength = contentLength - o.ETag = aws.ToString(out.ETag) - o.Expiration = aws.ToString(out.Expiration) +func (o *UploadObjectOutput) mapFromPutObjectOutput(out *s3.PutObjectOutput, bucket, key *string, contentLength int64) { o.Bucket = bucket o.Key = key + o.BucketKeyEnabled = out.BucketKeyEnabled + o.ChecksumCRC32 = out.ChecksumCRC32 + o.ChecksumCRC32C = out.ChecksumCRC32C + o.ChecksumCRC64NVME = out.ChecksumCRC64NVME + o.ChecksumSHA1 = out.ChecksumSHA1 + o.ChecksumSHA256 = out.ChecksumSHA256 + o.ChecksumType = types.ChecksumType(out.ChecksumType) + o.ContentLength = aws.Int64(contentLength) + o.ETag = out.ETag + o.Expiration = out.Expiration o.RequestCharged = types.RequestCharged(out.RequestCharged) - o.SSEKMSKeyID = aws.ToString(out.SSEKMSKeyId) + o.SSECustomerAlgorithm = out.SSECustomerAlgorithm + o.SSECustomerKeyMD5 = out.SSECustomerKeyMD5 + o.SSEKMSEncryptionContext = out.SSEKMSEncryptionContext + o.SSEKMSKeyID = out.SSEKMSKeyId o.ServerSideEncryption = types.ServerSideEncryption(out.ServerSideEncryption) - o.VersionID = aws.ToString(out.VersionId) - o.ResultMetadata = out.ResultMetadata.Clone() + o.VersionID = out.VersionId + o.ResultMetadata = out.ResultMetadata } -func (o *UploadObjectOutput) mapFromCompleteMultipartUploadOutput(out *s3.CompleteMultipartUploadOutput, bucket, uploadID string, contentLength int64, completedParts completedParts) { +func (o *UploadObjectOutput) mapFromCompleteMultipartUploadOutput(out *s3.CompleteMultipartUploadOutput, bucket, uploadID *string, contentLength int64, completedParts completedParts) { + o.Bucket = bucket + o.Key = out.Key o.UploadID = uploadID o.CompletedParts = completedParts - o.BucketKeyEnabled = aws.ToBool(out.BucketKeyEnabled) - o.ChecksumCRC32 = aws.ToString(out.ChecksumCRC32) - o.ChecksumCRC32C = aws.ToString(out.ChecksumCRC32C) - o.ChecksumSHA1 = aws.ToString(out.ChecksumSHA1) - o.ChecksumSHA256 = aws.ToString(out.ChecksumSHA256) - o.ContentLength = contentLength - o.ETag = aws.ToString(out.ETag) - o.Expiration = aws.ToString(out.Expiration) - o.Bucket = bucket - o.Key = aws.ToString(out.Key) + o.BucketKeyEnabled = out.BucketKeyEnabled + o.ChecksumCRC32 = out.ChecksumCRC32 + o.ChecksumCRC32C = out.ChecksumCRC32C + o.ChecksumCRC64NVME = out.ChecksumCRC64NVME + o.ChecksumSHA1 = out.ChecksumSHA1 + o.ChecksumSHA256 = out.ChecksumSHA256 + o.ChecksumType = types.ChecksumType(out.ChecksumType) + o.ContentLength = aws.Int64(contentLength) + o.ETag = out.ETag + o.Expiration = out.Expiration + o.Location = out.Location o.RequestCharged = types.RequestCharged(out.RequestCharged) - o.SSEKMSKeyID = aws.ToString(out.SSEKMSKeyId) + o.SSEKMSKeyID = out.SSEKMSKeyId o.ServerSideEncryption = types.ServerSideEncryption(out.ServerSideEncryption) - o.VersionID = aws.ToString(out.VersionId) + o.VersionID = out.VersionId o.ResultMetadata = out.ResultMetadata } @@ -729,7 +848,7 @@ func (u *uploader) initSize() error { } u.objectSize = n default: - if l := u.in.ContentLength; l > 0 { + if l := aws.ToInt64(u.in.ContentLength); l > 0 { u.objectSize = l } } @@ -887,7 +1006,7 @@ func (u *multiUploader) upload(ctx context.Context, firstBuf io.Reader, firstBuf } var out UploadObjectOutput - out.mapFromCompleteMultipartUploadOutput(completeOut, aws.ToString(params.Bucket), aws.ToString(u.uploadID), u.progressEmitter.bytesTransferred.Load(), u.parts) + out.mapFromCompleteMultipartUploadOutput(completeOut, params.Bucket, u.uploadID, u.progressEmitter.bytesTransferred.Load(), u.parts) u.progressEmitter.Complete(ctx, &out) return &out, nil @@ -994,6 +1113,7 @@ func (u *multiUploader) complete(ctx context.Context, clientOptions ...func(*s3. resp, err := u.options.S3.CompleteMultipartUpload(ctx, params, clientOptions...) if err != nil { u.seterr(err) + log.Printf("failed to complete multipart upload for upload ID %v: %v", u.uploadID, err) u.fail(ctx) } diff --git a/feature/s3/transfermanager/api_op_UploadObject_test.go b/feature/s3/transfermanager/api_op_UploadObject_test.go index d6856292678..ef1da3cdb42 100644 --- a/feature/s3/transfermanager/api_op_UploadObject_test.go +++ b/feature/s3/transfermanager/api_op_UploadObject_test.go @@ -36,12 +36,12 @@ func TestUploadOrderMulti(t *testing.T) { mgr := New(c, Options{}) resp, err := mgr.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key - value", + Bucket: aws.String("Bucket"), + Key: aws.String("Key - value"), Body: bytes.NewReader(buf20MB), ServerSideEncryption: "aws:kms", - SSEKMSKeyID: "KmsId", - ContentType: "content/type", + SSEKMSKeyID: aws.String("KmsId"), + ContentType: aws.String("content/type"), }) if err != nil { @@ -52,12 +52,12 @@ func TestUploadOrderMulti(t *testing.T) { t.Errorf(diff) } - if "UPLOAD-ID" != resp.UploadID { - t.Errorf("expect %q, got %q", "UPLOAD-ID", resp.UploadID) + if "UPLOAD-ID" != aws.ToString(resp.UploadID) { + t.Errorf("expect %q, got %q", "UPLOAD-ID", aws.ToString(resp.UploadID)) } - if "VERSION-ID" != resp.VersionID { - t.Errorf("expect %q, got %q", "VERSION-ID", resp.VersionID) + if "VERSION-ID" != aws.ToString(resp.VersionID) { + t.Errorf("expect %q, got %q", "VERSION-ID", aws.ToString(resp.VersionID)) } // Validate input values @@ -115,8 +115,8 @@ func TestUploadOrderMultiDifferentPartSize(t *testing.T) { }) _, err := mgr.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), Body: bytes.NewReader(buf20MB), }) @@ -144,8 +144,8 @@ func TestUploadFailIfPartSizeTooSmall(t *testing.T) { o.PartSizeBytes = 5 }) resp, err := mgr.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), Body: bytes.NewReader(buf20MB), }) @@ -165,12 +165,12 @@ func TestUploadOrderSingle(t *testing.T) { c, invocations, params := s3testing.NewUploadLoggingClient(nil) mgr := New(c, Options{}) resp, err := mgr.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key - value", + Bucket: aws.String("Bucket"), + Key: aws.String("Key - value"), Body: bytes.NewReader(buf2MB), ServerSideEncryption: "aws:kms", - SSEKMSKeyID: "KmsId", - ContentType: "content/type", + SSEKMSKeyID: aws.String("KmsId"), + ContentType: aws.String("content/type"), }) if err != nil { @@ -181,12 +181,12 @@ func TestUploadOrderSingle(t *testing.T) { t.Error(diff) } - if e := "VERSION-ID"; e != resp.VersionID { - t.Errorf("expect %q, got %q", e, resp.VersionID) + if e := "VERSION-ID"; e != aws.ToString(resp.VersionID) { + t.Errorf("expect %q, got %q", e, aws.ToString(resp.VersionID)) } - if len(resp.UploadID) > 0 { - t.Errorf("expect empty string, got %q", resp.UploadID) + if len(aws.ToString(resp.UploadID)) > 0 { + t.Errorf("expect empty string, got %q", aws.ToString(resp.UploadID)) } putObjectInput := (*params)[0].(*s3.PutObjectInput) @@ -213,8 +213,8 @@ func TestUploadSingleFailure(t *testing.T) { mgr := New(c, Options{}) resp, err := mgr.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), Body: bytes.NewReader(buf2MB), }) @@ -235,8 +235,8 @@ func TestUploadOrderZero(t *testing.T) { c, invocations, params := s3testing.NewUploadLoggingClient(nil) mgr := New(c, Options{}) resp, err := mgr.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), Body: bytes.NewReader(make([]byte, 0)), }) @@ -248,8 +248,8 @@ func TestUploadOrderZero(t *testing.T) { t.Error(diff) } - if len(resp.UploadID) > 0 { - t.Errorf("expect empty string, got %q", resp.UploadID) + if len(aws.ToString(resp.UploadID)) > 0 { + t.Errorf("expect empty string, got %q", aws.ToString(resp.UploadID)) } if e, a := int64(0), getReaderLength((*params)[0].(*s3.PutObjectInput).Body); e != a { @@ -271,8 +271,8 @@ func TestUploadOrderMultiFailure(t *testing.T) { o.Concurrency = 1 }) _, err := mgr.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), Body: bytes.NewReader(buf20MB), }) @@ -296,8 +296,8 @@ func TestUploadOrderMultiFailureOnComplete(t *testing.T) { o.Concurrency = 1 }) _, err := mgr.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), Body: bytes.NewReader(buf20MB), }) @@ -320,8 +320,8 @@ func TestUploadOrderMultiFailureOnCreate(t *testing.T) { mgr := New(c, Options{}) _, err := mgr.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), Body: bytes.NewReader(make([]byte, 1024*1024*12)), }) @@ -351,8 +351,8 @@ func TestUploadOrderReadFail1(t *testing.T) { c, invocations, _ := s3testing.NewUploadLoggingClient(nil) mgr := New(c, Options{}) _, err := mgr.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), Body: &failreader{times: 1}, }) if err == nil { @@ -374,8 +374,8 @@ func TestUploadOrderReadFail2(t *testing.T) { o.Concurrency = 1 }) _, err := mgr.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), Body: &failreader{times: 2}, }) if err == nil { @@ -418,8 +418,8 @@ func TestUploadOrderMultiBufferedReader(t *testing.T) { c, invocations, params := s3testing.NewUploadLoggingClient(nil) mgr := New(c, Options{}) _, err := mgr.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), Body: &sizedReader{size: 1024 * 1024 * 21}, }) if err != nil { @@ -449,8 +449,8 @@ func TestUploadOrderMultiBufferedReaderPartial(t *testing.T) { c, invocations, params := s3testing.NewUploadLoggingClient(nil) mgr := New(c, Options{}) _, err := mgr.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), Body: &sizedReader{size: 1024 * 1024 * 21, err: io.EOF}, }) if err != nil { @@ -482,8 +482,8 @@ func TestUploadOrderMultiBufferedReaderEOF(t *testing.T) { c, invocations, params := s3testing.NewUploadLoggingClient(nil) mgr := New(c, Options{}) _, err := mgr.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), Body: &sizedReader{size: 1024 * 1024 * 16, err: io.EOF}, }) @@ -513,8 +513,8 @@ func TestUploadOrderSingleBufferedReader(t *testing.T) { c, invocations, _ := s3testing.NewUploadLoggingClient(nil) mgr := New(c, Options{}) resp, err := mgr.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), Body: &sizedReader{size: 1024 * 1024 * 2}, }) @@ -526,8 +526,8 @@ func TestUploadOrderSingleBufferedReader(t *testing.T) { t.Error(diff) } - if len(resp.UploadID) > 0 { - t.Errorf("expect no value, got %q", resp.UploadID) + if len(aws.ToString(resp.UploadID)) > 0 { + t.Errorf("expect no value, got %q", aws.ToString(resp.UploadID)) } } @@ -536,8 +536,8 @@ func TestUploadZeroLenObject(t *testing.T) { mgr := New(c, Options{}) resp, err := mgr.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), Body: strings.NewReader(""), }) @@ -548,8 +548,8 @@ func TestUploadZeroLenObject(t *testing.T) { t.Errorf("expect request to have been made, but was not, %v", diff) } - if len(resp.UploadID) > 0 { - t.Errorf("expect empty string, but received %q", resp.UploadID) + if len(aws.ToString(resp.UploadID)) > 0 { + t.Errorf("expect empty string, but received %q", aws.ToString(resp.UploadID)) } } @@ -565,8 +565,8 @@ func TestProgressListener_SingleUpload_SeekableBody(t *testing.T) { body := "foobarbaz" in := &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), Body: strings.NewReader(body), } out, err := mgr.UploadObject(context.Background(), in) @@ -593,8 +593,8 @@ func TestProgressListener_SingleUpload_UnseekableBody(t *testing.T) { body := "foobarbaz" in := &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), Body: bytes.NewBuffer([]byte(body)), } out, err := mgr.UploadObject(context.Background(), in) @@ -621,8 +621,8 @@ func TestProgressListener_MultiUpload(t *testing.T) { mgr := New(c, opts) in := &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), Body: bytes.NewReader(buf40MB), } out, err := mgr.UploadObject(context.Background(), in) @@ -667,8 +667,8 @@ func TestUploadUnexpectedEOF(t *testing.T) { o.PartSizeBytes = minPartSizeBytes }) _, err := mgr.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), Body: &testIncompleteReader{ Size: minPartSizeBytes + 1, }, @@ -706,10 +706,10 @@ func TestSSE(t *testing.T) { }) _, err := mgr.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", - SSECustomerAlgorithm: "AES256", - SSECustomerKey: "foo", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), + SSECustomerAlgorithm: aws.String("AES256"), + SSECustomerKey: aws.String("foo"), Body: bytes.NewBuffer(make([]byte, 1024*1024*10)), }) @@ -730,8 +730,8 @@ func TestUploadWithContextCanceled(t *testing.T) { close(ctx.DoneCh) _, err := u.UploadObject(ctx, &UploadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), Body: bytes.NewReader(make([]byte, 0)), }) if err == nil { @@ -794,8 +794,8 @@ func TestUploadRetry(t *testing.T) { uploader := New(client, Options{}) _, err := uploader.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: "bucket", - Key: "key", + Bucket: aws.String("bucket"), + Key: aws.String("key"), Body: c.Body, }) diff --git a/feature/s3/transfermanager/concurrent_reader.go b/feature/s3/transfermanager/concurrent_reader.go index 68d80e545f8..e4c2587fcd0 100644 --- a/feature/s3/transfermanager/concurrent_reader.go +++ b/feature/s3/transfermanager/concurrent_reader.go @@ -10,6 +10,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "io" "sync" + "sync/atomic" ) // concurrentReader receives object parts from working goroutines, composes those chunks in order and read @@ -35,7 +36,7 @@ type concurrentReader struct { written int64 partSize int64 invocations int32 - etag string + etag *string ctx context.Context m sync.Mutex @@ -81,7 +82,7 @@ func (r *concurrentReader) Read(p []byte) (int, error) { break } - if r.index == r.getCapacity() { + if r.index == atomic.LoadInt32(&r.capacity) { continue } @@ -136,7 +137,7 @@ func (r *concurrentReader) downloadChunk(ctx context.Context, chunk getChunk, cl params.Range = aws.String(chunk.withRange) } if params.VersionId == nil { - params.IfMatch = aws.String(r.etag) + params.IfMatch = r.etag } out, err := r.options.S3.GetObject(ctx, params, clientOptions...) @@ -144,6 +145,21 @@ func (r *concurrentReader) downloadChunk(ctx context.Context, chunk getChunk, cl return nil, err } + if params.Range != nil && out.ContentRange != nil { + reqStart, reqEnd, err := getReqRange(aws.ToString(params.Range)) + if err != nil { + return nil, err + } + respStart, respEnd, err := getRespRange(aws.ToString(out.ContentRange)) + if err != nil { + return nil, err + } + // don't validate first chunk since object size is unknown when getting that + if reqStart != 0 && (reqStart != respStart || reqEnd != respEnd) { + return nil, fmt.Errorf("range mismatch between request %d-%d and response %d-%d", reqStart, reqEnd, respStart, respEnd) + } + } + defer out.Body.Close() buf, err := io.ReadAll(out.Body) @@ -187,7 +203,7 @@ func (r *concurrentReader) read(p []byte) (int, error) { partSize := r.partSize minIndex := int32(r.written / partSize) - maxIndex := min(int32((r.written+int64(cap(p))-1)/partSize), r.getCapacity()-1) + maxIndex := min(int32((r.written+int64(cap(p))-1)/partSize), atomic.LoadInt32(&r.capacity)-1) for i := minIndex; i <= maxIndex; i++ { if e := r.getErr(); e != nil && e != io.EOF { r.clean() @@ -208,9 +224,9 @@ func (r *concurrentReader) read(p []byte) (int, error) { if c.cur >= c.length { r.readCount++ delete(r.buf, i) - if r.readCount == r.getCapacity() { - capacity := min(r.getCapacity()+r.sectionParts, r.partsCount) - r.setCapacity(capacity) + if r.readCount == atomic.LoadInt32(&r.capacity) { + capacity := min(atomic.LoadInt32(&r.capacity)+r.sectionParts, r.partsCount) + atomic.StoreInt32(&r.capacity, capacity) } if r.readCount >= r.partsCount { r.setErr(io.EOF) @@ -219,7 +235,7 @@ func (r *concurrentReader) read(p []byte) (int, error) { } } - for r.receiveCount < r.getCapacity() { + for r.receiveCount < atomic.LoadInt32(&r.capacity) { if e := r.getErr(); e != nil && e != io.EOF { r.clean() return written, e @@ -248,9 +264,9 @@ func (r *concurrentReader) read(p []byte) (int, error) { r.buf[oc.index] = &oc } else { r.readCount++ - if r.readCount == r.getCapacity() { - capacity := min(r.getCapacity()+r.sectionParts, r.partsCount) - r.setCapacity(capacity) + if r.readCount == atomic.LoadInt32(&r.capacity) { + capacity := min(atomic.LoadInt32(&r.capacity)+r.sectionParts, r.partsCount) + atomic.StoreInt32(&r.capacity, capacity) } if r.readCount >= r.partsCount { r.setErr(io.EOF) @@ -261,20 +277,6 @@ func (r *concurrentReader) read(p []byte) (int, error) { return written, r.getErr() } -func (r *concurrentReader) setCapacity(n int32) { - r.m.Lock() - defer r.m.Unlock() - - r.capacity = n -} - -func (r *concurrentReader) getCapacity() int32 { - r.m.Lock() - defer r.m.Unlock() - - return r.capacity -} - func (r *concurrentReader) setDone(done bool) { r.m.Lock() defer r.m.Unlock() diff --git a/feature/s3/transfermanager/concurrent_reader_test.go b/feature/s3/transfermanager/concurrent_reader_test.go index 0077f8d0dee..d20f90bde48 100644 --- a/feature/s3/transfermanager/concurrent_reader_test.go +++ b/feature/s3/transfermanager/concurrent_reader_test.go @@ -3,6 +3,7 @@ package transfermanager import ( "bytes" "context" + "github.com/aws/aws-sdk-go-v2/aws" s3testing "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager/internal/testing" "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager/types" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -173,8 +174,8 @@ func TestConcurrentReader(t *testing.T) { sectionParts: c.sectionParts, options: c.options, in: &GetObjectInput{ - Bucket: "bucket", - Key: "key", + Bucket: aws.String("bucket"), + Key: aws.String("key"), }, capacity: int32(math.Min(float64(c.sectionParts), float64(c.partsCount))), buf: make(map[int32]*outChunk), diff --git a/feature/s3/transfermanager/download_directory_test.go b/feature/s3/transfermanager/download_directory_test.go index 8eff969eae4..861916b9164 100644 --- a/feature/s3/transfermanager/download_directory_test.go +++ b/feature/s3/transfermanager/download_directory_test.go @@ -36,8 +36,8 @@ type objectkeyCallback struct { } func (oc *objectkeyCallback) UpdateRequest(in *GetObjectInput) { - if in.Key == oc.keyword { - in.Key = in.Key + "gotyou" + if key := aws.ToString(in.Key); key == oc.keyword { + in.Key = aws.String(key + "gotyou") } } @@ -51,15 +51,16 @@ func TestDownloadDirectory(t *testing.T) { objectsLists [][]s3types.Object continuationTokens []string filter ObjectFilter - s3Delimiter string concurrency int callback GetRequestCallback + failurePolicy DownloadDirectoryFailurePolicy getobjectFn func(*s3testing.TransferManagerLoggingClient, *s3.GetObjectInput) (*s3.GetObjectOutput, error) expectTokens []string expectKeys []string expectFiles []string expectErr string - expectObjectsDownloaded int + expectObjectsDownloaded int64 + expectObjectsFailed int64 listenerValidationFn func(*testing.T, *mockDirectoryListener, any, any, error) }{ "single object": { @@ -252,32 +253,6 @@ func TestDownloadDirectory(t *testing.T) { l.expectComplete(t, in, out, 5) }, }, - "multiple objects with keyprefix with customized delimiter suffix": { - destination: "multiple-objects-with-keyprefix-customized-delimiter", - objectsLists: [][]s3types.Object{ - { - { - Key: aws.String("ab/c*d"), - }, - { - Key: aws.String("ab/c/e"), - }, - { - Key: aws.String("ab/c*f*g"), - }, - }, - }, - keyPrefix: "ab/c", - s3Delimiter: "*", - expectTokens: []string{""}, - expectKeys: []string{"ab/c*d", "ab/c/e", "ab/c*f*g"}, - expectFiles: []string{"d", "ab/c/e", "f*g"}, - expectObjectsDownloaded: 3, - listenerValidationFn: func(t *testing.T, l *mockDirectoryListener, in, out any, err error) { - l.expectStart(t, in) - l.expectComplete(t, in, out, 3) - }, - }, "error when path resolved from objects key out of destination scope": { destination: "error-bucket", concurrency: 1, @@ -396,60 +371,58 @@ func TestDownloadDirectory(t *testing.T) { l.expectComplete(t, in, out, 4) }, }, - "multiple objects paginated with keyprefix, delimiter, filter and callback": { - destination: "multiple-objects-with-keyprefix-delimiter-filter-callback", + "error when getting object": { + destination: "error-bucket", objectsLists: [][]s3types.Object{ { { - Key: aws.String("a&"), - }, - { - Key: aws.String("a&b"), + Key: aws.String("foo/bar"), }, { - Key: aws.String("a@b"), + Key: aws.String("baz"), }, }, { { - Key: aws.String("a&foo&bar"), - }, - { - Key: aws.String("ac"), + Key: aws.String("foo/zoo/bar"), }, { - Key: aws.String("ac@d&e"), + Key: aws.String("foo/zoo/oii/bababoii"), }, }, { { - Key: aws.String("ac/d/unwanted"), + Key: aws.String("foo/zoo/baz"), }, { - Key: aws.String("a&k.b"), + Key: aws.String("foo/zoo/oii/yee"), }, }, }, - continuationTokens: []string{"token1", "token2"}, - s3Delimiter: "&", - keyPrefix: "a", - filter: &objectkeyFilter{"unwanted"}, - callback: &objectkeyCallback{"a&k.b"}, - expectTokens: []string{"", "token1", "token2"}, - expectKeys: []string{"a&b", "a@b", "a&foo&bar", "ac", "ac@d&e", "a&k.bgotyou"}, - expectFiles: []string{"b", "a@b", "foo&bar", "ac", "ac@d&e", "k.b"}, - expectObjectsDownloaded: 6, + concurrency: 1, + continuationTokens: []string{"token1", "token2"}, + getobjectFn: func(c *s3testing.TransferManagerLoggingClient, in *s3.GetObjectInput) (*s3.GetObjectOutput, error) { + if aws.ToString(in.Key) == "foo/zoo/bar" { + return nil, fmt.Errorf("mocking error") + } + return &s3.GetObjectOutput{ + Body: ioutil.NopCloser(bytes.NewReader(c.Data)), + ContentLength: aws.Int64(int64(len(c.Data))), + PartsCount: aws.Int32(c.PartsCount), + ETag: aws.String(etag), + }, nil + }, + expectErr: "mocking error", listenerValidationFn: func(t *testing.T, l *mockDirectoryListener, in, out any, err error) { - l.expectStart(t, in) - l.expectComplete(t, in, out, 6) + l.expectFailed(t, in, err) }, }, - "error when getting object": { - destination: "error-bucket", + "specified getting object failure ignored by failure policy": { + destination: "error-ignored", objectsLists: [][]s3types.Object{ { { - Key: aws.String("foo/bar"), + Key: aws.String("fo/"), }, { Key: aws.String("baz"), @@ -473,9 +446,10 @@ func TestDownloadDirectory(t *testing.T) { }, }, concurrency: 1, + failurePolicy: IgnoreDownloadFailurePolicy{}, continuationTokens: []string{"token1", "token2"}, getobjectFn: func(c *s3testing.TransferManagerLoggingClient, in *s3.GetObjectInput) (*s3.GetObjectOutput, error) { - if aws.ToString(in.Key) == "foo/zoo/bar" { + if key := aws.ToString(in.Key); key == "foo/zoo/bar" || key == "baz" { return nil, fmt.Errorf("mocking error") } return &s3.GetObjectOutput{ @@ -485,9 +459,14 @@ func TestDownloadDirectory(t *testing.T) { ETag: aws.String(etag), }, nil }, - expectErr: "mocking error", + expectTokens: []string{"", "token1", "token2"}, + expectKeys: []string{"baz", "foo/zoo/bar", "foo/zoo/oii/bababoii", "foo/zoo/baz", "foo/zoo/oii/yee"}, + expectFiles: []string{"foo/zoo/oii/bababoii", "foo/zoo/baz", "foo/zoo/oii/yee"}, + expectObjectsDownloaded: 3, + expectObjectsFailed: 2, listenerValidationFn: func(t *testing.T, l *mockDirectoryListener, in, out any, err error) { - l.expectFailed(t, in, err) + l.expectStart(t, in) + l.expectComplete(t, in, out, 3) }, }, } @@ -510,12 +489,12 @@ func TestDownloadDirectory(t *testing.T) { defer os.RemoveAll(dstPath) req := &DownloadDirectoryInput{ - Bucket: "mock-bucket", - Destination: dstPath, - KeyPrefix: c.keyPrefix, - S3Delimiter: c.s3Delimiter, - Filter: c.filter, - Callback: c.callback, + Bucket: aws.String("mock-bucket"), + Destination: aws.String(dstPath), + KeyPrefix: nzstring(c.keyPrefix), + Filter: c.filter, + Callback: c.callback, + FailurePolicy: c.failurePolicy, } listener := &mockDirectoryListener{} @@ -547,6 +526,9 @@ func TestDownloadDirectory(t *testing.T) { if e, a := c.expectObjectsDownloaded, resp.ObjectsDownloaded; e != a { t.Errorf("expect %d objects downloaded, got %d", e, a) } + if e, a := c.expectObjectsFailed, resp.ObjectsFailed; e != a { + t.Errorf("expect %d objects failed, got %d", e, a) + } var actualTokens []string var actualKeys []string @@ -570,12 +552,8 @@ func TestDownloadDirectory(t *testing.T) { t.Errorf("expect downloaded keys to be %v, got %v", e, a) } - delimiter := c.s3Delimiter - if delimiter == "" { - delimiter = "/" - } for _, file := range c.expectFiles { - path := filepath.Join(dstPath, strings.ReplaceAll(file, delimiter, string(os.PathSeparator))) + path := filepath.Join(dstPath, strings.ReplaceAll(file, "/", string(os.PathSeparator))) _, err := os.Stat(path) if os.IsNotExist(err) { t.Errorf("expect %s to be downloaded, got none", path) @@ -676,8 +654,8 @@ func TestDownloadDirectoryObjectsTransferred(t *testing.T) { defer os.RemoveAll(dstPath) req := &DownloadDirectoryInput{ - Bucket: "mock-bucket", - Destination: dstPath, + Bucket: aws.String("mock-bucket"), + Destination: aws.String(dstPath), } listener := &mockDirectoryListener{} @@ -710,8 +688,8 @@ func TestDownloadDirectoryWithContextCanceled(t *testing.T) { close(ctx.DoneCh) _, err := u.DownloadDirectory(ctx, &DownloadDirectoryInput{ - Bucket: "mock-bucket", - Destination: dstPath, + Bucket: aws.String("mock-bucket"), + Destination: aws.String(dstPath), }) if err == nil { t.Fatalf("expect error, got nil") diff --git a/feature/s3/transfermanager/downloadobject_test.go b/feature/s3/transfermanager/downloadobject_test.go index 6e11d5c01b8..89ecba7efc5 100644 --- a/feature/s3/transfermanager/downloadobject_test.go +++ b/feature/s3/transfermanager/downloadobject_test.go @@ -27,15 +27,14 @@ func TestDownloadObject(t *testing.T) { errReaders []s3testing.TestErrReader getObjectFn func(*s3testing.TransferManagerLoggingClient, *s3.GetObjectInput) (*s3.GetObjectOutput, error) options Options - downloadRange string expectInvocations int expectRanges []string - partNumber int32 versionID string partsCount int32 expectParts []int32 expectVersions []string expectETags []string + expectComposite bool expectErr string dataValidationFn func(*testing.T, *types.WriteAtBuffer) listenerValidationFn func(*testing.T, *mockListener, any, any, error) @@ -104,7 +103,7 @@ func TestDownloadObject(t *testing.T) { l.expectFailed(t, in, err) }, }, - "range download with mismatch error": { + "range download with content mismatch error": { data: buf20MB, getObjectFn: s3testing.MismatchRangeGetObjectFn, options: Options{ @@ -118,6 +117,20 @@ func TestDownloadObject(t *testing.T) { l.expectFailed(t, in, err) }, }, + "range download with resp range mismatch error": { + data: buf20MB, + getObjectFn: s3testing.WrongRangeGetObjectFn, + options: Options{ + Concurrency: 1, + GetObjectType: types.GetObjectRanges, + }, + expectInvocations: 2, + expectErr: "range mismatch between request", + listenerValidationFn: func(t *testing.T, l *mockListener, in, out any, err error) { + l.expectStartTotalBytes(t, 20*megabyte) + l.expectFailed(t, in, err) + }, + }, "content length download single chunk": { data: buf2MB, getObjectFn: s3testing.NonRangeGetObjectFn, @@ -229,41 +242,25 @@ func TestDownloadObject(t *testing.T) { l.expectFailed(t, in, err) }, }, - "range download a range of object": { - data: buf20MB, - getObjectFn: s3testing.RangeGetObjectFn, - options: Options{ - Concurrency: 1, - GetObjectType: types.GetObjectRanges, - }, - downloadRange: "bytes=1-10485759", - expectInvocations: 2, - expectRanges: []string{"bytes=1-8388608", "bytes=8388609-10485759"}, - expectETags: []string{"", etag}, - listenerValidationFn: func(t *testing.T, l *mockListener, in, out any, err error) { - l.expectComplete(t, in, out) - l.expectByteTransfers(t, 8*megabyte, 10*megabyte-1) - }, - }, - "range download a range of object with version ID": { - data: buf20MB, - getObjectFn: s3testing.RangeGetObjectFn, + "parts download in order": { + data: buf2MB, + getObjectFn: s3testing.PartGetObjectFn, options: Options{ - Concurrency: 1, - GetObjectType: types.GetObjectRanges, + Concurrency: 1, }, - downloadRange: "bytes=0-10485759", + partsCount: 3, versionID: vID, - expectInvocations: 2, - expectVersions: []string{vID, vID}, + expectInvocations: 3, + expectVersions: []string{vID, vID, vID}, + expectParts: []int32{1, 2, 3}, listenerValidationFn: func(t *testing.T, l *mockListener, in, out any, err error) { l.expectComplete(t, in, out) - l.expectByteTransfers(t, 8*megabyte, 10*megabyte) + l.expectByteTransfers(t, 2*megabyte, 4*megabyte, 6*megabyte) }, }, - "parts download in order": { + "parts download in order with composite checksum type": { data: buf2MB, - getObjectFn: s3testing.PartGetObjectFn, + getObjectFn: s3testing.CompositePartGetObjectFn, options: Options{ Concurrency: 1, }, @@ -272,6 +269,7 @@ func TestDownloadObject(t *testing.T) { expectInvocations: 3, expectVersions: []string{vID, vID, vID}, expectParts: []int32{1, 2, 3}, + expectComposite: true, listenerValidationFn: func(t *testing.T, l *mockListener, in, out any, err error) { l.expectComplete(t, in, out) l.expectByteTransfers(t, 2*megabyte, 4*megabyte, 6*megabyte) @@ -397,40 +395,6 @@ func TestDownloadObject(t *testing.T) { l.expectFailed(t, in, err) }, }, - "parts download with range input": { - data: []byte("123"), - getObjectFn: s3testing.PartGetObjectFn, - options: Options{}, - downloadRange: "bytes=0-100", - partsCount: 3, - expectInvocations: 1, - dataValidationFn: func(t *testing.T, w *types.WriteAtBuffer) { - if e, a := "123", string(w.Bytes()); e != a { - t.Errorf("expect %q response, got %q", e, a) - } - }, - listenerValidationFn: func(t *testing.T, l *mockListener, in, out any, err error) { - l.expectComplete(t, in, out) - l.expectByteTransfers(t, 3) - }, - }, - "parts download with part number input": { - data: []byte("ab"), - getObjectFn: s3testing.PartGetObjectFn, - options: Options{}, - partsCount: 3, - partNumber: 5, - expectInvocations: 1, - dataValidationFn: func(t *testing.T, w *types.WriteAtBuffer) { - if e, a := "ab", string(w.Bytes()); e != a { - t.Errorf("expect %q response, got %q", e, a) - } - }, - listenerValidationFn: func(t *testing.T, l *mockListener, in, out any, err error) { - l.expectComplete(t, in, out) - l.expectByteTransfers(t, 2) - }, - }, } for name, c := range cases { @@ -444,12 +408,10 @@ func TestDownloadObject(t *testing.T) { w := types.NewWriteAtBuffer(make([]byte, 0)) input := &DownloadObjectInput{ - Bucket: "bucket", - Key: "key", - WriterAt: w, - Range: c.downloadRange, - PartNumber: c.partNumber, - VersionID: c.versionID, + Bucket: aws.String("bucket"), + Key: aws.String("key"), + WriterAt: w, + VersionID: nzstring(c.versionID), } listener := &mockListener{} @@ -503,6 +465,15 @@ func TestDownloadObject(t *testing.T) { } } + if c.expectComposite { + if output.ChecksumCRC32 != nil || output.ChecksumCRC32C != nil || output.ChecksumCRC64NVME != nil || + output.ChecksumSHA1 != nil || output.ChecksumSHA256 != nil { + t.Errorf("expect all composite checksum value to be empty in output, got non-empty value: %s, %s, %s, %s, %s", + aws.ToString(output.ChecksumCRC32), aws.ToString(output.ChecksumCRC32C), aws.ToString(output.ChecksumCRC64NVME), + aws.ToString(output.ChecksumSHA1), aws.ToString(output.ChecksumSHA256)) + } + } + if c.dataValidationFn != nil { c.dataValidationFn(t, w) } @@ -522,7 +493,6 @@ func TestDownloadAsyncWithFailure(t *testing.T) { for name, c := range cases { t.Run(name, func(t *testing.T) { - startingByte := 0 reqCount := int64(0) s3Client := &s3testing.TransferManagerLoggingClient{} @@ -533,15 +503,20 @@ func TestDownloadAsyncWithFailure(t *testing.T) { time.Sleep(1 * time.Second) err = fmt.Errorf("some connection error") default: + var start, end int64 + if params.Range != nil { + start, end, err = getReqRange(aws.ToString(params.Range)) + if err != nil { + return + } + } body := bytes.NewReader(make([]byte, minPartSizeBytes)) out = &s3.GetObjectOutput{ Body: ioutil.NopCloser(body), ContentLength: aws.Int64(int64(body.Len())), - ContentRange: aws.String(fmt.Sprintf("bytes %d-%d/%d", startingByte, body.Len()-1, body.Len()*10)), + ContentRange: aws.String(fmt.Sprintf("bytes %d-%d/%d", start, end, body.Len()*10)), PartsCount: aws.Int32(10), } - - startingByte += body.Len() if reqCount > 0 { // sleep here to ensure context switching between goroutines time.Sleep(25 * time.Millisecond) @@ -560,8 +535,8 @@ func TestDownloadAsyncWithFailure(t *testing.T) { // Expect this request to exit quickly after failure _, err := d.DownloadObject(context.Background(), &DownloadObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), WriterAt: w, }) if err == nil { @@ -602,8 +577,8 @@ func TestDownloadObjectWithContextCanceled(t *testing.T) { w := types.NewWriteAtBuffer(make([]byte, 0)) _, err := d.DownloadObject(ctx, &DownloadObjectInput{ - Bucket: "bucket", - Key: "Key", + Bucket: aws.String("bucket"), + Key: aws.String("Key"), WriterAt: w, }) if err == nil { diff --git a/feature/s3/transfermanager/getobject_test.go b/feature/s3/transfermanager/getobject_test.go index c400e1dd2ce..569098f486a 100644 --- a/feature/s3/transfermanager/getobject_test.go +++ b/feature/s3/transfermanager/getobject_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" "io" "io/ioutil" "reflect" @@ -28,13 +29,12 @@ func TestGetObject(t *testing.T) { errReaders []s3testing.TestErrReader getObjectFn func(*s3testing.TransferManagerLoggingClient, *s3.GetObjectInput) (*s3.GetObjectOutput, error) options Options - downloadRange string versionID string + checksumType s3types.ChecksumType expectInvocations int expectRanges []string expectVersions []string expectETags []string - partNumber int32 partsCount int32 expectParts []int32 expectGetErr string @@ -83,7 +83,7 @@ func TestGetObject(t *testing.T) { expectInvocations: 2, expectReadErr: "s3 service error", }, - "range download with mismatch error": { + "range download with content mismatch error": { data: buf20MB, getObjectFn: s3testing.MismatchRangeGetObjectFn, options: Options{ @@ -93,6 +93,16 @@ func TestGetObject(t *testing.T) { expectInvocations: 2, expectReadErr: "PreconditionFailed", }, + "range download with resp range mismatch error": { + data: buf20MB, + getObjectFn: s3testing.WrongRangeGetObjectFn, + options: Options{ + GetObjectType: types.GetObjectRanges, + Concurrency: 1, + }, + expectInvocations: 2, + expectReadErr: "range mismatch between request", + }, "content length download single chunk": { data: buf2MB, getObjectFn: s3testing.NonRangeGetObjectFn, @@ -160,44 +170,24 @@ func TestGetObject(t *testing.T) { expectInvocations: 1, expectReadErr: "unexpected EOF", }, - "range download a range of object": { - data: buf20MB, - getObjectFn: s3testing.RangeGetObjectFn, - options: Options{ - GetObjectType: types.GetObjectRanges, - Concurrency: 1, - }, - downloadRange: "bytes=1-10485759", - expectInvocations: 2, - expectETags: []string{etag, etag}, - expectRanges: []string{"bytes=1-8388608", "bytes=8388609-10485759"}, - }, - "range download a range of object with part number": { - data: buf20MB, - getObjectFn: s3testing.NonRangeGetObjectFn, - options: Options{ - GetObjectType: types.GetObjectRanges, - }, - downloadRange: "bytes=1-10485759", - partNumber: 5, - expectInvocations: 1, - }, - "range download invalid range": { - data: buf20MB, - getObjectFn: s3testing.RangeGetObjectFn, + "parts download in order": { + data: buf2MB, + getObjectFn: s3testing.PartGetObjectFn, options: Options{ - GetObjectType: types.GetObjectRanges, - Concurrency: 1, + Concurrency: 1, }, - downloadRange: "bytes=1--1", - expectGetErr: "invalid input range", + partsCount: 3, + expectInvocations: 3, + expectETags: []string{etag, etag, etag}, + expectParts: []int32{1, 2, 3}, }, - "parts download in order": { + "parts download with composite checksum type": { data: buf2MB, getObjectFn: s3testing.PartGetObjectFn, options: Options{ Concurrency: 1, }, + checksumType: s3types.ChecksumTypeComposite, partsCount: 3, expectInvocations: 3, expectETags: []string{etag, etag, etag}, @@ -283,32 +273,6 @@ func TestGetObject(t *testing.T) { } }, }, - "parts download with range input": { - data: []byte("123"), - getObjectFn: s3testing.PartGetObjectFn, - options: Options{}, - downloadRange: "bytes=0-100", - partsCount: 3, - expectInvocations: 1, - dataValidationFn: func(t *testing.T, bytes []byte) { - if e, a := "123", string(bytes); e != a { - t.Errorf("expect %q response, got %q", e, a) - } - }, - }, - "parts download with part number input": { - data: []byte("ab"), - getObjectFn: s3testing.PartGetObjectFn, - options: Options{}, - partsCount: 3, - partNumber: 5, - expectInvocations: 1, - dataValidationFn: func(t *testing.T, bytes []byte) { - if e, a := "ab", string(bytes); e != a { - t.Errorf("expect %q response, got %q", e, a) - } - }, - }, } for name, c := range cases { @@ -318,15 +282,14 @@ func TestGetObject(t *testing.T) { s3Client.GetObjectFn = c.getObjectFn s3Client.ErrReaders = c.errReaders s3Client.PartsCount = c.partsCount + s3Client.ChecksumType = c.checksumType mgr := New(s3Client, c.options) input := &GetObjectInput{ - Bucket: "bucket", - Key: "key", + Bucket: aws.String("bucket"), + Key: aws.String("key"), } - input.Range = c.downloadRange - input.PartNumber = c.partNumber - input.VersionID = c.versionID + input.VersionID = nzstring(c.versionID) out, err := mgr.GetObject(context.Background(), input) @@ -384,6 +347,15 @@ func TestGetObject(t *testing.T) { } } + if c.checksumType == s3types.ChecksumTypeComposite { + if out.ChecksumCRC32 != nil || out.ChecksumCRC32C != nil || out.ChecksumCRC64NVME != nil || + out.ChecksumSHA1 != nil || out.ChecksumSHA256 != nil { + t.Errorf("expect all composite checksum value to be empty in output, got non-empty value: %s, %s, %s, %s, %s", + aws.ToString(out.ChecksumCRC32), aws.ToString(out.ChecksumCRC32C), aws.ToString(out.ChecksumCRC64NVME), + aws.ToString(out.ChecksumSHA1), aws.ToString(out.ChecksumSHA256)) + } + } + if c.dataValidationFn != nil { c.dataValidationFn(t, actualBuf) } @@ -403,12 +375,11 @@ func TestGetAsyncWithFailure(t *testing.T) { for name, c := range cases { t.Run(name, func(t *testing.T) { - startingByte := 0 reqCount := int64(0) s3Client := &s3testing.TransferManagerLoggingClient{} s3Client.PartsCount = 10 - s3Client.Data = buf40MB + s3Client.Data = buf80MB s3Client.GetObjectFn = func(c *s3testing.TransferManagerLoggingClient, params *s3.GetObjectInput) (out *s3.GetObjectOutput, err error) { switch atomic.LoadInt64(&reqCount) { case 1: @@ -416,19 +387,26 @@ func TestGetAsyncWithFailure(t *testing.T) { time.Sleep(1 * time.Second) err = fmt.Errorf("some connection error") default: + var start, end int64 + if params.Range != nil { + start, end, err = getReqRange(aws.ToString(params.Range)) + if err != nil { + return + } + } body := bytes.NewReader(make([]byte, minPartSizeBytes)) out = &s3.GetObjectOutput{ Body: ioutil.NopCloser(body), ContentLength: aws.Int64(int64(body.Len())), - ContentRange: aws.String(fmt.Sprintf("bytes %d-%d/%d", startingByte, body.Len()-1, body.Len()*10)), + ContentRange: aws.String(fmt.Sprintf("bytes %d-%d/%d", start, end, body.Len()*10)), } - startingByte += body.Len() if reqCount > 0 { // sleep here to ensure context switching between goroutines time.Sleep(25 * time.Millisecond) } } + atomic.AddInt64(&reqCount, 1) return out, err } @@ -440,8 +418,8 @@ func TestGetAsyncWithFailure(t *testing.T) { // Expect this request to exit quickly after failure out, err := mgr.GetObject(context.Background(), &GetObjectInput{ - Bucket: "Bucket", - Key: "Key", + Bucket: aws.String("Bucket"), + Key: aws.String("Key"), }) _, err = io.ReadAll(out.Body) @@ -481,8 +459,8 @@ func TestGetObjectWithContextCanceled(t *testing.T) { close(ctx.DoneCh) _, err := mgr.GetObject(ctx, &GetObjectInput{ - Bucket: "bucket", - Key: "Key", + Bucket: aws.String("bucket"), + Key: aws.String("Key"), }) if err == nil { diff --git a/feature/s3/transfermanager/internal/testing/client.go b/feature/s3/transfermanager/internal/testing/client.go index 66e1c537ed9..2c6b947dd99 100644 --- a/feature/s3/transfermanager/internal/testing/client.go +++ b/feature/s3/transfermanager/internal/testing/client.go @@ -45,6 +45,7 @@ type TransferManagerLoggingClient struct { RetrievedParts []int32 Versions []string Etags []string + ChecksumType s3types.ChecksumType ErrReaders []TestErrReader @@ -244,9 +245,15 @@ func (c *TransferManagerLoggingClient) HeadObject(ctx context.Context, params *s defer c.m.Unlock() return &s3.HeadObjectOutput{ - PartsCount: aws.Int32(c.PartsCount), - ContentLength: aws.Int64(int64(len(c.Data))), - ETag: aws.String(etag), + PartsCount: aws.Int32(c.PartsCount), + ContentLength: aws.Int64(int64(len(c.Data))), + ETag: aws.String(etag), + ChecksumType: c.ChecksumType, + ChecksumCRC32: aws.String("crc32"), + ChecksumCRC32C: aws.String("crc32c"), + ChecksumCRC64NVME: aws.String("crc64nvme"), + ChecksumSHA1: aws.String("sha1"), + ChecksumSHA256: aws.String("sha256"), }, nil } @@ -304,18 +311,21 @@ var RangeGetObjectFn = func(c *TransferManagerLoggingClient, params *s3.GetObjec start, fin := parseRange(aws.ToString(params.Range)) fin++ - if fin >= int64(len(c.Data)) { + if fin > int64(len(c.Data)) { fin = int64(len(c.Data)) } bodyBytes := c.Data[start:fin] - return &s3.GetObjectOutput{ + out := &s3.GetObjectOutput{ Body: ioutil.NopCloser(bytes.NewReader(bodyBytes)), - ContentRange: aws.String(fmt.Sprintf("bytes %d-%d/%d", start, fin-1, len(c.Data))), ContentLength: aws.Int64(int64(len(bodyBytes))), ETag: aws.String(etag), - }, nil + } + if len(bodyBytes) != len(c.Data) { + out.ContentRange = aws.String(fmt.Sprintf("bytes %d-%d/%d", start, fin-1, len(c.Data))) + } + return out, nil } // ErrRangeGetObjectFn mocks getobject behavior of s3 client to return service error when certain number of range get is called from s3 client @@ -338,6 +348,16 @@ var MismatchRangeGetObjectFn = func(c *TransferManagerLoggingClient, params *s3. return out, err } +// WrongRangeGetObjectFn mocks getobject behavior of s3 client to return wrong content range during ranges GET +var WrongRangeGetObjectFn = func(c *TransferManagerLoggingClient, params *s3.GetObjectInput) (*s3.GetObjectOutput, error) { + out, err := RangeGetObjectFn(c, params) + c.index++ + if c.index > 1 { + out.ContentRange = aws.String("bytes 0-1/100") // first chunk is never validated, so this resp start will always mismatch + } + return out, err +} + // NonRangeGetObjectFn mocks getobject behavior of s3 client to return the whole object var NonRangeGetObjectFn = func(c *TransferManagerLoggingClient, params *s3.GetObjectInput) (*s3.GetObjectOutput, error) { return &s3.GetObjectOutput{ @@ -379,6 +399,22 @@ var ReaderPartGetObjectFn = func(c *TransferManagerLoggingClient, params *s3.Get }, nil } +// CompositePartGetObjectFn mocks getobject behavior of s3 client to return object with composite checksum type and checksum value +var CompositePartGetObjectFn = func(c *TransferManagerLoggingClient, params *s3.GetObjectInput) (*s3.GetObjectOutput, error) { + return &s3.GetObjectOutput{ + Body: ioutil.NopCloser(bytes.NewReader(c.Data)), + ContentLength: aws.Int64(int64(len(c.Data))), + PartsCount: aws.Int32(c.PartsCount), + ETag: aws.String(etag), + ChecksumCRC32: aws.String("crc32"), + ChecksumCRC32C: aws.String("crc32c"), + ChecksumCRC64NVME: aws.String("crc64nvme"), + ChecksumSHA1: aws.String("sha1"), + ChecksumSHA256: aws.String("sha256"), + ChecksumType: s3types.ChecksumTypeComposite, + }, nil +} + // ErrPartGetObjectFn mocks getobject behavior of s3 client to return service error when certain number of part get is called from s3 client var ErrPartGetObjectFn = func(c *TransferManagerLoggingClient, params *s3.GetObjectInput) (*s3.GetObjectOutput, error) { out, err := PartGetObjectFn(c, params) diff --git a/feature/s3/transfermanager/mapping_reference_test.go b/feature/s3/transfermanager/mapping_reference_test.go new file mode 100644 index 00000000000..d4f146ab6fc --- /dev/null +++ b/feature/s3/transfermanager/mapping_reference_test.go @@ -0,0 +1,704 @@ +package transfermanager + +import ( + "encoding/json" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager/types" + "github.com/aws/aws-sdk-go-v2/service/s3" + "reflect" + "strings" + "testing" + "time" +) + +// all tests use a single package-level instance of this spec +// +// legacy name mappings may mutate some of the field names but it doesn't break +// anything from test to test right now +var mappingReference struct { + Definition struct { + UploadRequest struct { + PutObjectRequest []string + } + UploadResponse struct { + PutObjectResponse []string + } + DownloadRequest struct { + GetObjectRequest []string + } + DownloadResponse struct { + GetObjectResponse []string + } + } + Conversion struct { + UploadRequest struct { + PutObjectRequest []string + CreateMultipartRequest []string + UploadPartRequest []string + CompleteMultipartRequest []string + AbortMultipartRequest []string + } + CompleteMultipartResponse struct { + UploadResponse []string + } + PutObjectResponse struct { + UploadResponse []string + } + GetObjectResponse struct { + DownloadResponse []string + } + } +} + +const mappingReferenceJSON = ` +{ + "Definition": { + "UploadRequest": { + "PutObjectRequest": [ + "ACL", + "Bucket", + "BucketKeyEnabled", + "CacheControl", + "ChecksumAlgorithm", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ContentDisposition", + "ContentEncoding", + "ContentLanguage", + "ContentType", + "ExpectedBucketOwner", + "Expires", + "GrantFullControl", + "GrantRead", + "GrantReadACP", + "GrantWriteACP", + "IfMatch", + "IfNoneMatch", + "Key", + "Metadata", + "ObjectLockLegalHoldStatus", + "ObjectLockMode", + "ObjectLockRetainUntilDate", + "RequestPayer", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5", + "SSEKMSEncryptionContext", + "SSEKMSKeyId", + "ServerSideEncryption", + "StorageClass", + "Tagging", + "WebsiteRedirectLocation" + ] + }, + "UploadResponse": { + "PutObjectResponse": [ + "BucketKeyEnabled", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ChecksumType", + "ETag", + "Expiration", + "RequestCharged", + "SSEKMSKeyId", + "ServerSideEncryption", + "VersionId" + ] + }, + "DownloadRequest": { + "GetObjectRequest": [ + "Bucket", + "ChecksumMode", + "ExpectedBucketOwner", + "IfMatch", + "IfModifiedSince", + "IfNoneMatch", + "IfUnmodifiedSince", + "Key", + "RequestPayer", + "ResponseCacheControl", + "ResponseContentDisposition", + "ResponseContentEncoding", + "ResponseContentLanguage", + "ResponseContentType", + "ResponseExpires", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5", + "VersionId" + ] + }, + "DownloadResponse": { + "GetObjectResponse": [ + "AcceptRanges", + "BucketKeyEnabled", + "CacheControl", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ChecksumType", + "ContentDisposition", + "ContentEncoding", + "ContentLanguage", + "ContentLength", + "ContentRange", + "ContentType", + "DeleteMarker", + "ETag", + "Expiration", + "Expires", + "LastModified", + "Metadata", + "MissingMeta", + "ObjectLockLegalHoldStatus", + "ObjectLockMode", + "ObjectLockRetainUntilDate", + "PartsCount", + "ReplicationStatus", + "RequestCharged", + "Restore", + "SSECustomerAlgorithm", + "SSECustomerKeyMD5", + "SSEKMSKeyId", + "ServerSideEncryption", + "StorageClass", + "TagCount", + "VersionId", + "WebsiteRedirectLocation" + ] + } + }, + "Conversion": { + "UploadRequest": { + "PutObjectRequest": [ + "ACL", + "Bucket", + "BucketKeyEnabled", + "CacheControl", + "ChecksumAlgorithm", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ContentDisposition", + "ContentEncoding", + "ContentLanguage", + "ContentType", + "ExpectedBucketOwner", + "Expires", + "GrantFullControl", + "GrantRead", + "GrantReadACP", + "GrantWriteACP", + "IfMatch", + "IfNoneMatch", + "Key", + "Metadata", + "ObjectLockLegalHoldStatus", + "ObjectLockMode", + "ObjectLockRetainUntilDate", + "RequestPayer", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5", + "SSEKMSEncryptionContext", + "SSEKMSKeyID", + "ServerSideEncryption", + "StorageClass", + "Tagging", + "WebsiteRedirectLocation" + ], + "CreateMultipartRequest": [ + "ACL", + "Bucket", + "BucketKeyEnabled", + "CacheControl", + "ChecksumAlgorithm", + "ContentDisposition", + "ContentEncoding", + "ContentLanguage", + "ContentType", + "ExpectedBucketOwner", + "Expires", + "GrantFullControl", + "GrantRead", + "GrantReadACP", + "GrantWriteACP", + "Key", + "Metadata", + "ObjectLockLegalHoldStatus", + "ObjectLockMode", + "ObjectLockRetainUntilDate", + "RequestPayer", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5", + "SSEKMSEncryptionContext", + "SSEKMSKeyID", + "ServerSideEncryption", + "StorageClass", + "Tagging", + "WebsiteRedirectLocation" + ], + "UploadPartRequest": [ + "Bucket", + "ChecksumAlgorithm", + "ExpectedBucketOwner", + "Key", + "RequestPayer", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5" + ], + "CompleteMultipartRequest": [ + "Bucket", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ExpectedBucketOwner", + "IfMatch", + "IfNoneMatch", + "Key", + "RequestPayer", + "SSECustomerAlgorithm", + "SSECustomerKey", + "SSECustomerKeyMD5" + ], + "AbortMultipartRequest": [ + "Bucket", + "ExpectedBucketOwner", + "Key", + "RequestPayer" + ] + }, + "CompleteMultipartResponse": { + "UploadResponse": [ + "BucketKeyEnabled", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ChecksumType", + "ETag", + "Expiration", + "RequestCharged", + "SSEKMSKeyId", + "ServerSideEncryption", + "VersionId" + ] + }, + "PutObjectResponse": { + "UploadResponse": [ + "BucketKeyEnabled", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ChecksumType", + "ETag", + "Expiration", + "RequestCharged", + "SSECustomerAlgorithm", + "SSECustomerKeyMD5", + "SSEKMSEncryptionContext", + "SSEKMSKeyId", + "ServerSideEncryption", + "VersionId" + ] + }, + "GetObjectResponse": { + "DownloadResponse": [ + "AcceptRanges", + "BucketKeyEnabled", + "CacheControl", + "ChecksumCRC32", + "ChecksumCRC32C", + "ChecksumCRC64NVME", + "ChecksumSHA1", + "ChecksumSHA256", + "ChecksumType", + "ContentDisposition", + "ContentEncoding", + "ContentLanguage", + "ContentLength", + "ContentRange", + "ContentType", + "DeleteMarker", + "ETag", + "Expiration", + "Expires", + "ExpiresString", + "LastModified", + "Metadata", + "MissingMeta", + "ObjectLockLegalHoldStatus", + "ObjectLockMode", + "ObjectLockRetainUntilDate", + "PartsCount", + "ReplicationStatus", + "RequestCharged", + "Restore", + "SSECustomerAlgorithm", + "SSECustomerKeyMD5", + "SSEKMSKeyId", + "ServerSideEncryption", + "StorageClass", + "TagCount", + "VersionId", + "WebsiteRedirectLocation" + ] + } + } +} +` + +func init() { + if err := json.Unmarshal([]byte(mappingReferenceJSON), &mappingReference); err != nil { + panic(err) + } +} + +func TestDefinition_UploadRequest(t *testing.T) { + legacyMappings := map[string]string{ + "SSEKMSKeyId": "SSEKMSKeyID", + } + + rtype := reflect.TypeOf(UploadObjectInput{}) + + for _, field := range mappingReference.Definition.UploadRequest.PutObjectRequest { + if mapped, ok := legacyMappings[field]; ok { + field = mapped + } + + _, ok := rtype.FieldByName(field) + if !ok { + t.Errorf("UploadInput: missing field %q", field) + } + } +} + +func TestDefinition_UploadResponse(t *testing.T) { + legacyMappings := map[string]string{ + "VersionId": "VersionID", + "SSEKMSKeyId": "SSEKMSKeyID", + } + + rtype := reflect.TypeOf(UploadObjectOutput{}) + + for _, field := range mappingReference.Definition.UploadResponse.PutObjectResponse { + if mapped, ok := legacyMappings[field]; ok { + field = mapped + } + + _, ok := rtype.FieldByName(field) + if !ok { + t.Errorf("UploadOutput: missing field %q", field) + } + } +} + +func TestDefinition_DownloadRequest(t *testing.T) { + legacyMappings := map[string]string{ + "VersionId": "VersionID", + } + + rtype := reflect.TypeOf(DownloadObjectInput{}) + + for _, field := range mappingReference.Definition.DownloadRequest.GetObjectRequest { + if mapped, ok := legacyMappings[field]; ok { + field = mapped + } + + _, ok := rtype.FieldByName(field) + if !ok { + t.Errorf("DownloadInput: missing field %q", field) + } + } +} + +func TestDefinition_DownloadResponse(t *testing.T) { + legacyMappings := map[string]string{ + "VersionId": "VersionID", + "SSEKMSKeyId": "SSEKMSKeyID", + } + + rtype := reflect.TypeOf(DownloadObjectOutput{}) + + for _, field := range mappingReference.Definition.DownloadResponse.GetObjectResponse { + if mapped, ok := legacyMappings[field]; ok { + field = mapped + } + + _, ok := rtype.FieldByName(field) + if !ok { + t.Errorf("DownloadOutput: missing field %q", field) + } + } +} + +func TestDefinition_GetRequest(t *testing.T) { + legacyMappings := map[string]string{ + "VersionId": "VersionID", + } + + rtype := reflect.TypeOf(GetObjectInput{}) + + for _, field := range mappingReference.Definition.DownloadRequest.GetObjectRequest { + if mapped, ok := legacyMappings[field]; ok { + field = mapped + } + + _, ok := rtype.FieldByName(field) + if !ok { + t.Errorf("GetInput: missing field %q", field) + } + } +} + +func TestDefinition_GetResponse(t *testing.T) { + legacyMappings := map[string]string{ + "VersionId": "VersionID", + "SSEKMSKeyId": "SSEKMSKeyID", + } + + rtype := reflect.TypeOf(GetObjectOutput{}) + + for _, field := range mappingReference.Definition.DownloadResponse.GetObjectResponse { + if mapped, ok := legacyMappings[field]; ok { + field = mapped + } + + _, ok := rtype.FieldByName(field) + if !ok { + t.Errorf("DownloadOutput: missing field %q", field) + } + } +} + +func TestConversion_UploadRequest_PutObjectRequest(t *testing.T) { + src := UploadObjectInput{} + + now := time.Now() + mockFields(&src, mappingReference.Conversion.UploadRequest.PutObjectRequest, map[string]interface{}{ + "Time": &now, + }) + + dst := src.mapSingleUploadInput(strings.NewReader(""), types.ChecksumAlgorithmCrc32) + + expectConvertedFields(t, dst, &src, + mappingReference.Conversion.UploadRequest.PutObjectRequest, + map[string]string{ + "SSEKMSKeyID": "SSEKMSKeyId", + }) +} + +func TestConversion_UploadRequest_CreateMultipartUploadRequest(t *testing.T) { + src := UploadObjectInput{} + + now := time.Now() + mockFields(&src, mappingReference.Conversion.UploadRequest.CreateMultipartRequest, map[string]interface{}{ + "Time": &now, + }) + + dst := src.mapCreateMultipartUploadInput(types.ChecksumAlgorithmCrc32) + + expectConvertedFields(t, dst, &src, + mappingReference.Conversion.UploadRequest.CreateMultipartRequest, + map[string]string{ + "SSEKMSKeyID": "SSEKMSKeyId", + }) +} + +func TestConversion_UploadRequest_UploadPartRequest(t *testing.T) { + src := UploadObjectInput{} + + mockFields(&src, mappingReference.Conversion.UploadRequest.UploadPartRequest, map[string]interface{}{}) + + dst := src.mapUploadPartInput(strings.NewReader(""), aws.Int32(1), aws.String(""), types.ChecksumAlgorithmCrc32) + + expectConvertedFields(t, dst, &src, + mappingReference.Conversion.UploadRequest.UploadPartRequest, + map[string]string{}) +} + +func TestConversion_UploadRequest_CompleteMultipartUploadRequest(t *testing.T) { + src := UploadObjectInput{} + + mockFields(&src, mappingReference.Conversion.UploadRequest.CompleteMultipartRequest, map[string]interface{}{}) + + dst := src.mapCompleteMultipartUploadInput(aws.String(""), []types.CompletedPart{}) + + expectConvertedFields(t, dst, &src, + mappingReference.Conversion.UploadRequest.CompleteMultipartRequest, + map[string]string{}) +} + +func TestConversion_UploadRequest_AbortMultipartUploadRequest(t *testing.T) { + src := UploadObjectInput{} + + mockFields(&src, mappingReference.Conversion.UploadRequest.AbortMultipartRequest, map[string]interface{}{}) + + dst := src.mapAbortMultipartUploadInput(aws.String("")) + + expectConvertedFields(t, dst, &src, + mappingReference.Conversion.UploadRequest.AbortMultipartRequest, + map[string]string{}) +} + +func TestConversion_CompleteMultipartUploadResponse_UploadResponse(t *testing.T) { + dst := UploadObjectOutput{} + src := s3.CompleteMultipartUploadOutput{} + + mockFields(&src, mappingReference.Conversion.CompleteMultipartResponse.UploadResponse, map[string]interface{}{}) + + dst.mapFromCompleteMultipartUploadOutput(&src, aws.String(""), aws.String(""), 0, []types.CompletedPart{}) + + expectConvertedFields(t, &dst, &src, + mappingReference.Conversion.CompleteMultipartResponse.UploadResponse, + map[string]string{ + "VersionId": "VersionID", + "SSEKMSKeyId": "SSEKMSKeyID", + }) +} + +func TestConversion_PutObjectResponse_UploadResponse(t *testing.T) { + dst := UploadObjectOutput{} + src := s3.PutObjectOutput{} + + mockFields(&src, mappingReference.Conversion.PutObjectResponse.UploadResponse, map[string]interface{}{}) + + dst.mapFromPutObjectOutput(&src, aws.String(""), aws.String(""), 0) + + expectConvertedFields(t, &dst, &src, + mappingReference.Conversion.PutObjectResponse.UploadResponse, + map[string]string{ + "VersionId": "VersionID", + "SSEKMSKeyId": "SSEKMSKeyID", + }) +} + +func TestConversion_GetObjectResponse_DownloadResponse(t *testing.T) { + dst := DownloadObjectOutput{} + src := s3.GetObjectOutput{} + + now := time.Now() + mockFields(&src, mappingReference.Conversion.GetObjectResponse.DownloadResponse, map[string]interface{}{ + "Time": &now, + }) + + dst.mapFromGetObjectOutput(&src, "") + + expectConvertedFields(t, &dst, &src, + mappingReference.Conversion.GetObjectResponse.DownloadResponse, + map[string]string{ + "VersionId": "VersionID", + "SSEKMSKeyId": "SSEKMSKeyID", + }) +} + +func TestConversion_GetObjectResponse_GetResponse(t *testing.T) { + dst := GetObjectOutput{} + src := s3.GetObjectOutput{} + + now := time.Now() + mockFields(&src, mappingReference.Conversion.GetObjectResponse.DownloadResponse, map[string]interface{}{ + "Time": &now, + }) + + dst.mapFromGetObjectOutput(&src, "") + + expectConvertedFields(t, &dst, &src, + mappingReference.Conversion.GetObjectResponse.DownloadResponse, + map[string]string{ + "VersionId": "VersionID", + "SSEKMSKeyId": "SSEKMSKeyID", + }) +} + +func expectConvertedFields(t *testing.T, dst, src any, fields []string, legacyNames map[string]string) { + t.Helper() + + dstv := reflect.ValueOf(dst).Elem() + srcv := reflect.ValueOf(src).Elem() + for _, srcField := range fields { + dstField := srcField + if legacy, ok := legacyNames[srcField]; ok { + dstField = legacy + } + + // indirect for any fields that happen to different in pointerness + dstf := reflect.Indirect(dstv.FieldByName(dstField)) + srcf := reflect.Indirect(srcv.FieldByName(srcField)) + if !dstf.IsValid() { + t.Fatalf("dst is missing field %q, do you need a legacy name mapping?", srcField) + } + + expect := srcf.Interface() + actual := dstf.Interface() + if dstf.Type() != srcf.Type() { + // s3 and tm use their own string type, which is always deeply unequal + // so their underlying string value is compared directly + expect = srcf.String() + actual = dstf.String() + } + if !reflect.DeepEqual(expect, actual) { + t.Errorf("field %q: %v != %v", srcField, expect, actual) + } + } +} + +// v must be a pointer to a struct +func mockFields(v any, fields []string, legacyTypes map[string]interface{}) { + rv := reflect.ValueOf(v).Elem() + for _, field := range fields { + mockValue(rv.FieldByName(field), field, legacyTypes) + } +} + +func mockValue(v reflect.Value, field string, legacyTypes map[string]interface{}) { + switch v.Kind() { + case reflect.Pointer: + switch v.Type().Elem().Kind() { + case reflect.Bool: + vv := true + v.Set(reflect.ValueOf(&vv)) + case reflect.String: + v.Set(reflect.ValueOf(&field)) + case reflect.Int64: + vv := int64(1) + v.Set(reflect.ValueOf(&vv)) + case reflect.Int32: + vv := int32(2) + v.Set(reflect.ValueOf(&vv)) + case reflect.Struct: + vv, ok := legacyTypes[v.Type().Elem().Name()] + if !ok { + panic(fmt.Sprintf("need to handle %v for field %s", v.Type().Elem().Kind(), field)) + } + v.Set(reflect.ValueOf(vv)) + default: + panic(fmt.Sprintf("need to handle %v", v.Type().Elem().Kind())) + } + case reflect.String: + // Convert() because it's probably a string enum + v.Set(reflect.ValueOf(field).Convert(v.Type())) + case reflect.Map: + v.Set(reflect.ValueOf(map[string]string{"a": "b"})) + default: + panic(fmt.Sprintf("need to handle %v", v.Type().Elem().Kind())) + } +} diff --git a/feature/s3/transfermanager/setup_integ_test.go b/feature/s3/transfermanager/setup_integ_test.go index 12e6db28912..bfa13487ba0 100644 --- a/feature/s3/transfermanager/setup_integ_test.go +++ b/feature/s3/transfermanager/setup_integ_test.go @@ -251,8 +251,8 @@ func testPutObject(t *testing.T, bucket string, testData putObjectTestData, opts _, err := s3TransferManagerClient.UploadObject(context.Background(), &UploadObjectInput{ - Bucket: bucket, - Key: key, + Bucket: aws.String(bucket), + Key: aws.String(key), Body: testData.Body, }, opts...) if err != nil { @@ -302,8 +302,8 @@ func testGetObject(t *testing.T, bucket string, testData getObjectTestData) { out, err := s3TransferManagerClient.GetObject(context.Background(), &GetObjectInput{ - Bucket: bucket, - Key: key, + Bucket: aws.String(bucket), + Key: aws.String(key), }, testData.OptFns...) if err != nil { @@ -359,8 +359,8 @@ func testDownloadObject(t *testing.T, bucket string, testData downloadObjectTest w := types.NewWriteAtBuffer(make([]byte, 0)) _, err = s3TransferManagerClient.DownloadObject(context.Background(), &DownloadObjectInput{ - Bucket: bucket, - Key: key, + Bucket: aws.String(bucket), + Key: aws.String(key), WriterAt: w, }, testData.OptFns...) if err != nil { @@ -388,9 +388,8 @@ type uploadDirectoryTestData struct { FilesSize map[string]int64 Source string Recursive bool - Delimiter string KeyPrefix string - ExpectFilesUploaded int + ExpectFilesUploaded int64 ExpectKeys []string ExpectError string } @@ -399,9 +398,6 @@ func testUploadDirectory(t *testing.T, bucket string, testData uploadDirectoryTe _, filename, _, _ := runtime.Caller(0) root := filepath.Join(filepath.Dir(filename), "testdata") delimiter := "/" - if testData.Delimiter != "" { - delimiter = testData.Delimiter - } expectObjects := map[string][]byte{} source := filepath.Join(root, testData.Source) if err := os.MkdirAll(source, os.ModePerm); err != nil { @@ -434,11 +430,10 @@ func testUploadDirectory(t *testing.T, bucket string, testData uploadDirectoryTe } out, err := s3TransferManagerClient.UploadDirectory(context.Background(), &UploadDirectoryInput{ - Bucket: bucket, - Source: source, - Recursive: testData.Recursive, - S3Delimiter: testData.Delimiter, - KeyPrefix: testData.KeyPrefix, + Bucket: aws.String(bucket), + Source: aws.String(source), + Recursive: aws.Bool(testData.Recursive), + KeyPrefix: aws.String(testData.KeyPrefix), }) if err != nil { if len(testData.ExpectError) == 0 { @@ -482,9 +477,8 @@ func testUploadDirectory(t *testing.T, bucket string, testData uploadDirectoryTe type downloadDirectoryTestData struct { ObjectsSize map[string]int64 - Delimiter string KeyPrefix string - ExpectObjectsDownloaded int + ExpectObjectsDownloaded int64 ExpectFiles []string ExpectError string } @@ -494,10 +488,7 @@ func testDownloadDirectory(t *testing.T, bucket string, testData downloadDirecto dst := filepath.Join(filepath.Dir(filename), "testdata", "integ") defer os.RemoveAll(dst) - delimiter := testData.Delimiter - if delimiter == "" { - delimiter = "/" - } + delimiter := "/" keyprefix := testData.KeyPrefix if keyprefix != "" && !strings.HasSuffix(keyprefix, delimiter) { keyprefix = keyprefix + delimiter @@ -518,15 +509,14 @@ func testDownloadDirectory(t *testing.T, bucket string, testData downloadDirecto if err != nil { t.Fatalf("error when putting object %s", key) } - file := filepath.Join(strings.ReplaceAll(strings.TrimPrefix(key, keyprefix), delimiter, string(os.PathSeparator))) + file := strings.ReplaceAll(strings.TrimPrefix(key, keyprefix), delimiter, string(os.PathSeparator)) expectFiles[file] = fileBuf } out, err := s3TransferManagerClient.DownloadDirectory(context.Background(), &DownloadDirectoryInput{ - Bucket: bucket, - Destination: dst, - KeyPrefix: testData.KeyPrefix, - S3Delimiter: testData.Delimiter, + Bucket: aws.String(bucket), + Destination: aws.String(dst), + KeyPrefix: aws.String(testData.KeyPrefix), }) if err != nil { if len(testData.ExpectError) == 0 { @@ -548,7 +538,7 @@ func testDownloadDirectory(t *testing.T, bucket string, testData downloadDirecto t.Errorf("expect %d objects downloaded, got %d", e, a) } for _, file := range testData.ExpectFiles { - f := strings.ReplaceAll(file, "/", string(os.PathSeparator)) + f := strings.ReplaceAll(file, delimiter, string(os.PathSeparator)) path := filepath.Join(dst, f) b, err := os.ReadFile(path) if err != nil { diff --git a/feature/s3/transfermanager/shared_test.go b/feature/s3/transfermanager/shared_test.go index 1c62ecab25b..c19b83a28ad 100644 --- a/feature/s3/transfermanager/shared_test.go +++ b/feature/s3/transfermanager/shared_test.go @@ -3,3 +3,4 @@ package transfermanager var buf20MB = make([]byte, 1024*1024*20) var buf2MB = make([]byte, 1024*1024*2) var buf40MB = make([]byte, 1024*1024*40) +var buf80MB = make([]byte, 1024*1024*80) diff --git a/feature/s3/transfermanager/types/types.go b/feature/s3/transfermanager/types/types.go index 26ab2719e81..be93f8bb2d2 100644 --- a/feature/s3/transfermanager/types/types.go +++ b/feature/s3/transfermanager/types/types.go @@ -425,3 +425,12 @@ func (b *WriteAtBuffer) Bytes() []byte { defer b.m.Unlock() return b.buf } + +// ChecksumType represents the transfer checksum type +type ChecksumType string + +// Enum values for ChecksumType +const ( + ChecksumTypeComposite ChecksumType = "COMPOSITE" + ChecksumTypeFullObject ChecksumType = "FULL_OBJECT" +) diff --git a/feature/s3/transfermanager/upload_directory_test.go b/feature/s3/transfermanager/upload_directory_test.go index cbc0b07c85e..842ef7907bf 100644 --- a/feature/s3/transfermanager/upload_directory_test.go +++ b/feature/s3/transfermanager/upload_directory_test.go @@ -34,8 +34,8 @@ type keynameCallback struct { } func (kc *keynameCallback) UpdateRequest(in *UploadObjectInput) { - if in.Key == kc.keyword { - in.Key = in.Key + "/gotyou" + if k := aws.ToString(in.Key); k == kc.keyword { + *in.Key = k + "/gotyou" } } @@ -49,13 +49,14 @@ func TestUploadDirectory(t *testing.T) { recursive bool keyPrefix string filter FileFilter - s3Delimiter string callback PutRequestCallback + failurePolicy UploadDirectoryFailurePolicy putobjectFunc func(*s3testing.TransferManagerLoggingClient, *s3.PutObjectInput) (*s3.PutObjectOutput, error) preprocessFunc func(string) (func() error, error) expectKeys []string expectErr string - expectFilesUploaded int + expectFilesUploaded int64 + expectFilesFailed int64 listenerValidationFn func(*testing.T, *mockDirectoryListener, any, any, error) }{ "single file recursively": { @@ -269,22 +270,39 @@ func TestUploadDirectory(t *testing.T) { l.expectFailed(t, in, err) }, }, - "error when a file contains customized delimiter": { - source: filepath.Join(root, "file-contains-non-default-delimiter"), - recursive: true, - s3Delimiter: "@", - expectErr: "contains delimiter @", + "specified files uploads failure ignored by failure policy for recursive upload": { + source: filepath.Join(root, "multi-file-with-subdir"), + recursive: true, + putobjectFunc: func(svc *s3testing.TransferManagerLoggingClient, param *s3.PutObjectInput) (*s3.PutObjectOutput, error) { + if key := aws.ToString(param.Key); key == "zoo/oii/yee" || key == "foo" { + return nil, fmt.Errorf("banned key") + } + return &s3.PutObjectOutput{}, nil + }, + failurePolicy: IgnoreUploadFailurePolicy{}, + expectKeys: []string{"zoo/baz", "bar", "zoo/oii/yee", "foo"}, + expectFilesUploaded: 2, + expectFilesFailed: 2, listenerValidationFn: func(t *testing.T, l *mockDirectoryListener, in, out any, err error) { - l.expectFailed(t, in, err) + l.expectStart(t, in) + l.expectComplete(t, in, out, 2) }, }, - "error when a sub-folder contains customized delimiter": { - source: filepath.Join(root, "folder-contains-non-default-delimiter"), - recursive: true, - s3Delimiter: "@", - expectErr: "contains delimiter @", + "specified files uploads failure ignored by failure policy for non-recursive upload": { + source: filepath.Join(root, "multi-file-with-subdir"), + putobjectFunc: func(svc *s3testing.TransferManagerLoggingClient, param *s3.PutObjectInput) (*s3.PutObjectOutput, error) { + if key := aws.ToString(param.Key); key == "zoo/oii/yee" || key == "foo" { + return nil, fmt.Errorf("banned key") + } + return &s3.PutObjectOutput{}, nil + }, + failurePolicy: IgnoreUploadFailurePolicy{}, + expectKeys: []string{"bar", "foo"}, + expectFilesUploaded: 1, + expectFilesFailed: 1, listenerValidationFn: func(t *testing.T, l *mockDirectoryListener, in, out any, err error) { - l.expectFailed(t, in, err) + l.expectStart(t, in) + l.expectComplete(t, in, out, 1) }, }, "error when a symlink refers to its upper dir": { @@ -404,65 +422,6 @@ func TestUploadDirectory(t *testing.T) { l.expectComplete(t, in, out, 2) }, }, - "folder containing both file and symlink with keyprefix and custome delimiter": { - source: filepath.Join(root, "multi-file-contain-symlink"), - followSymLinks: true, - recursive: true, - keyPrefix: "bla", - s3Delimiter: "#", - preprocessFunc: func(root string) (func() error, error) { - symlinkPath1 := filepath.Join(root, "multi-file-contain-symlink", "to", "the", "symFoo") - symlinkPath2 := filepath.Join(root, "multi-file-contain-symlink", "to", "symBar") - postprocessFunc := func() error { - os.Remove(symlinkPath1) - os.Remove(symlinkPath2) - return nil - } - if err := os.Symlink(filepath.Join(root, "dstFile1"), symlinkPath1); err != nil { - return postprocessFunc, err - } - if err := os.Symlink(filepath.Join(root, "dstDir1"), symlinkPath2); err != nil { - return postprocessFunc, err - } - return postprocessFunc, nil - }, - expectKeys: []string{"bla#foo", "bla#bar", "bla#to#baz", "bla#to#the#symFoo", "bla#to#symBar#foo", "bla#to#the#yee"}, - expectFilesUploaded: 6, - listenerValidationFn: func(t *testing.T, l *mockDirectoryListener, in, out any, err error) { - l.expectStart(t, in) - l.expectComplete(t, in, out, 6) - }, - }, - "folder containing both file and symlink with keyprefix, custome delimiter and request callback": { - source: filepath.Join(root, "multi-file-contain-symlink"), - followSymLinks: true, - recursive: true, - keyPrefix: "bla", - s3Delimiter: "#", - callback: &keynameCallback{"bla#to#baz"}, - preprocessFunc: func(root string) (func() error, error) { - symlinkPath1 := filepath.Join(root, "multi-file-contain-symlink", "to", "the", "symFoo") - symlinkPath2 := filepath.Join(root, "multi-file-contain-symlink", "to", "symBar") - postprocessFunc := func() error { - os.Remove(symlinkPath1) - os.Remove(symlinkPath2) - return nil - } - if err := os.Symlink(filepath.Join(root, "dstFile1"), symlinkPath1); err != nil { - return postprocessFunc, err - } - if err := os.Symlink(filepath.Join(root, "dstDir1"), symlinkPath2); err != nil { - return postprocessFunc, err - } - return postprocessFunc, nil - }, - expectKeys: []string{"bla#foo", "bla#bar", "bla#to#baz/gotyou", "bla#to#the#symFoo", "bla#to#symBar#foo", "bla#to#the#yee"}, - expectFilesUploaded: 6, - listenerValidationFn: func(t *testing.T, l *mockDirectoryListener, in, out any, err error) { - l.expectStart(t, in) - l.expectComplete(t, in, out, 6) - }, - }, } for name, c := range cases { @@ -480,14 +439,14 @@ func TestUploadDirectory(t *testing.T) { } req := &UploadDirectoryInput{ - Bucket: "mock-bucket", - Source: c.source, - FollowSymbolicLinks: c.followSymLinks, - Recursive: c.recursive, - KeyPrefix: c.keyPrefix, + Bucket: aws.String("mock-bucket"), + Source: aws.String(c.source), + FollowSymbolicLinks: aws.Bool(c.followSymLinks), + Recursive: aws.Bool(c.recursive), + KeyPrefix: aws.String(c.keyPrefix), Filter: c.filter, Callback: c.callback, - S3Delimiter: c.s3Delimiter, + FailurePolicy: c.failurePolicy, } listener := &mockDirectoryListener{} @@ -517,6 +476,9 @@ func TestUploadDirectory(t *testing.T) { if e, a := c.expectFilesUploaded, resp.ObjectsUploaded; e != a { t.Errorf("expect %d objects uploaded, got %d", e, a) } + if e, a := c.expectFilesFailed, resp.ObjectsFailed; e != a { + t.Errorf("expect %d objects failed, got %d", e, a) + } var actualKeys []string for _, param := range *params { @@ -569,9 +531,9 @@ func TestUploadDirectoryObjectsTransferred(t *testing.T) { mgr := New(s3Client, Options{}) req := &UploadDirectoryInput{ - Bucket: "mock-bucket", - Source: c.source, - Recursive: c.recursive, + Bucket: aws.String("mock-bucket"), + Source: aws.String(c.source), + Recursive: aws.Bool(c.recursive), } listener := &mockDirectoryListener{} @@ -603,9 +565,9 @@ func TestUploadDirectoryWithContextCanceled(t *testing.T) { close(ctx.DoneCh) _, err := u.UploadDirectory(ctx, &UploadDirectoryInput{ - Bucket: "mock-bucket", - Source: filepath.Join(root, "multi-file-contain-symlink"), - Recursive: true, + Bucket: aws.String("mock-bucket"), + Source: aws.String(filepath.Join(root, "multi-file-contain-symlink")), + Recursive: aws.Bool(true), }) if err == nil { t.Fatalf("expect error, got nil")