diff --git a/lib/api/apiUtils/object/sourceChecksum.js b/lib/api/apiUtils/object/sourceChecksum.js new file mode 100644 index 0000000000..c717289325 --- /dev/null +++ b/lib/api/apiUtils/object/sourceChecksum.js @@ -0,0 +1,89 @@ +const { PassThrough } = require('stream'); +const async = require('async'); +const { jsutil } = require('arsenal'); + +const { data } = require('../../../data/wrapper'); +const ChecksumWritable = require('../../../auth/streamingV4/ChecksumWritable'); + +/** + * Sequentially GET the ordered source `dataLocator` parts into a single + * readable stream, reading them in order through `data.get`. Returns the + * PassThrough immediately; consumers should pipe it onward and observe + * `error` for any read failure along the way. + * + * @param {Array} dataLocator - ordered source parts + * @param {object} log - request logger + * @return {PassThrough} + */ +function buildSourcePartsStream(dataLocator, log) { + const passthrough = new PassThrough(); + const wrapErr = (err, part) => + Object.assign(err, { + copyPart: { key: part.key, dataStoreName: part.dataStoreName, dataStoreType: part.dataStoreType }, + }); + async.eachSeries( + dataLocator, + (part, cb) => { + const done = jsutil.once(cb); + if (part.dataStoreType === 'azure') { + // Azure's data.get writes part bytes into the provided writable + // instead of returning a Readable. Pipe a per-part PassThrough + // into the master passthrough and use its 'end' as the completion + // signal — same pattern arsenal's data.copyObject uses. + const perPart = new PassThrough(); + perPart.once('error', err => done(wrapErr(err, part))); + perPart.once('end', () => done()); + perPart.pipe(passthrough, { end: false }); + return data.get(part, perPart, log, err => { + if (err) { + perPart.destroy(err); + done(wrapErr(err, part)); + } + }); + } + return data.get(part, null, log, (err, partStream) => { + if (err) { + return done(wrapErr(err, part)); + } + partStream.once('error', err => done(wrapErr(err, part))); + partStream.once('end', () => done()); + partStream.pipe(passthrough, { end: false }); + return undefined; + }); + }, + err => { + if (err) { + passthrough.destroy(err); + } else { + passthrough.end(); + } + }, + ); + return passthrough; +} + +/** + * Compute the checksum of the (range-adjusted) source bytes by streaming them + * through a ChecksumWritable sink. An empty `dataLocator` ends the stream + * immediately, yielding the empty-input digest. + * + * @param {Array} dataLocator - ordered source parts + * @param {string} algorithm - lowercase checksum algorithm name + * @param {object} log - request logger + * @param {function} cb - cb(err, { algorithm, value }) + * @return {undefined} + */ +function computeChecksumFromDataLocator(dataLocator, algorithm, log, cb) { + const onceCb = jsutil.once(cb); + const sourceStream = buildSourcePartsStream(dataLocator || [], log); + const checksumSink = new ChecksumWritable(algorithm, log); + sourceStream.once('error', err => { + checksumSink.destroy(err); + onceCb(err); + }); + checksumSink.once('error', onceCb); + checksumSink.once('finish', () => onceCb(null, { algorithm, value: checksumSink.digest })); + sourceStream.pipe(checksumSink); +} + +module.exports = { buildSourcePartsStream, computeChecksumFromDataLocator }; diff --git a/lib/api/completeMultipartUpload.js b/lib/api/completeMultipartUpload.js index a79c4b5a74..044104d741 100644 --- a/lib/api/completeMultipartUpload.js +++ b/lib/api/completeMultipartUpload.js @@ -52,15 +52,19 @@ const allChecksumXmlTags = Object.values(checksumAlgorithms).map(algo => algo.xm * does not match the stored part's ChecksumValue, return InvalidPart. * - If checksumType === 'COMPOSITE' and checksumIsDefault is false, every part * in the request body MUST include the matching Checksum field; - * missing → InvalidRequest. + * missing → InvalidRequest. (Relaxed for external backends, which store no + * per-part checksum - but a checksum the client does submit is still checked, + * and rejected, since there is no stored value to match.) * * @param {object} jsonList - parsed CompleteMultipartUpload XML * @param {array} storedParts - parts as returned by services.getMPUparts * @param {string} mpuSplitter - splitter used in part keys * @param {object} mpuChecksum - { algorithm, type, isDefault } + * @param {boolean} isExternal - external-backend MPU; relax the COMPOSITE + * per-part requirement (external parts carry no stored checksum) * @returns {Error|null} */ -function validatePerPartChecksums(jsonList, storedParts, mpuSplitter, mpuChecksum) { +function validatePerPartChecksums(jsonList, storedParts, mpuSplitter, mpuChecksum, isExternal) { const mpuAlgo = mpuChecksum.algorithm; if (!mpuAlgo) { // Legacy / pre-checksums MPU, no algorithm tracked, nothing to validate. @@ -68,7 +72,10 @@ function validatePerPartChecksums(jsonList, storedParts, mpuSplitter, mpuChecksu } const expectedTag = checksumAlgorithms[mpuAlgo] ? checksumAlgorithms[mpuAlgo].xmlTag : null; // Skip enforcement if the MPU's algorithm is unknown (shouldn't happen). - const requireForEachPart = mpuChecksum.type === 'COMPOSITE' && !mpuChecksum.isDefault && expectedTag !== null; + // External backends store no per-part checksum, so don't require one; a + // checksum the client does submit is still rejected below (no stored value). + const requireForEachPart = + mpuChecksum.type === 'COMPOSITE' && !mpuChecksum.isDefault && expectedTag !== null && !isExternal; const storedByPartNumber = new Map(); storedParts.forEach(item => { @@ -462,7 +469,14 @@ function completeMultipartUpload(authInfo, request, log, callback) { type: storedMetadata.checksumType, isDefault: storedMetadata.checksumIsDefault, }; - const checksumErr = validatePerPartChecksums(jsonList, storedParts, splitter, mpuChecksum); + const isExternalMpu = !!constants.externalBackends[config.getLocationConstraintType(location)]; + const checksumErr = validatePerPartChecksums( + jsonList, + storedParts, + splitter, + mpuChecksum, + isExternalMpu, + ); if (checksumErr) { log.debug('per-part checksum validation failed', { error: checksumErr, diff --git a/lib/api/objectCopy.js b/lib/api/objectCopy.js index 13b9628cf8..fa0b237ebf 100644 --- a/lib/api/objectCopy.js +++ b/lib/api/objectCopy.js @@ -1,5 +1,4 @@ const async = require('async'); -const { PassThrough } = require('stream'); const { errors, errorInstances, jsutil, versioning, s3middleware, s3routes } = require('arsenal'); const { validateObjectKeyLength } = s3routes.routesUtils; @@ -33,6 +32,7 @@ const { } = require('./apiUtils/integrity/validateChecksums'); const ChecksumTransform = require('../auth/streamingV4/ChecksumTransform'); const ChecksumWritable = require('../auth/streamingV4/ChecksumWritable'); +const { buildSourcePartsStream } = require('./apiUtils/object/sourceChecksum'); const kms = require('../kms/wrapper'); const versionIdUtils = versioning.VersionID; @@ -66,63 +66,6 @@ function _orphanedDataLocations(dataToDelete, newDataGetInfo) { return orphans.length > 0 ? orphans : null; } -/** - * Concatenate the source object's parts into a single Readable stream by - * reading them sequentially through `data.get`. Returns the PassThrough - * immediately; consumers should pipe it to the next stage and observe - * `error` on this stream for any read failure along the way. - * - * @param {Array} dataLocator - ordered source parts - * @param {object} log - request logger - * @return {PassThrough} - */ -function _pipeSourcePartsThrough(dataLocator, log) { - const passthrough = new PassThrough(); - const wrapErr = (err, part) => - Object.assign(err, { - copyPart: { key: part.key, dataStoreName: part.dataStoreName, dataStoreType: part.dataStoreType }, - }); - async.eachSeries( - dataLocator, - (part, cb) => { - const done = jsutil.once(cb); - if (part.dataStoreType === 'azure') { - // Azure's data.get writes part bytes into the provided writable - // instead of returning a Readable. Pipe a per-part PassThrough - // into the master passthrough and use its 'end' as the completion - // signal — same pattern arsenal's data.copyObject uses. - const perPart = new PassThrough(); - perPart.once('error', err => done(wrapErr(err, part))); - perPart.once('end', () => done()); - perPart.pipe(passthrough, { end: false }); - return data.get(part, perPart, log, err => { - if (err) { - perPart.destroy(err); - done(wrapErr(err, part)); - } - }); - } - return data.get(part, null, log, (err, partStream) => { - if (err) { - return done(wrapErr(err, part)); - } - partStream.once('error', err => done(wrapErr(err, part))); - partStream.once('end', () => done()); - partStream.pipe(passthrough, { end: false }); - return undefined; - }); - }, - err => { - if (err) { - passthrough.destroy(err); - } else { - passthrough.end(); - } - }, - ); - return passthrough; -} - /** * Decide whether the destination's checksum needs to be recomputed by * streaming the source bytes through a ChecksumTransform. @@ -207,7 +150,7 @@ function _recomputeChecksumAndStore( algorithm: algoName, size: storeMetadataParams.size, }); - const sourceStream = _pipeSourcePartsThrough(dataLocator, log); + const sourceStream = buildSourcePartsStream(dataLocator, log); const checksumSink = new ChecksumWritable(algoName, log); const finish = jsutil.once(err => { if (err) { @@ -236,7 +179,7 @@ function _recomputeChecksumAndStore( // Stream source bytes through a ChecksumTransform and write them out as a single put. log.debug('recomputing checksum on CopyObject', { algorithm: algoName, size: storeMetadataParams.size }); - const sourceStream = _pipeSourcePartsThrough(dataLocator, log); + const sourceStream = buildSourcePartsStream(dataLocator, log); const checksumStream = new ChecksumTransform(algoName, undefined, false, log); const done = jsutil.once((err, results) => { if (err) { @@ -362,7 +305,10 @@ function _prepMetadata( }; } // Cannot copy from same source and destination if no MD - // changed and no source version id + // changed and no source version id. A checksum-algorithm header is NOT a + // change here: AWS rejects a COPY-directive self-copy regardless of any + // requested checksum (verified). Recomputing a checksum in place requires + // MetadataDirective=REPLACE, which is a metadata change and so is allowed. if ( sourceIsDestination && whichMetadata === 'COPY' && diff --git a/lib/api/objectPutCopyPart.js b/lib/api/objectPutCopyPart.js index 3f3999b212..ccba422c71 100644 --- a/lib/api/objectPutCopyPart.js +++ b/lib/api/objectPutCopyPart.js @@ -1,12 +1,11 @@ const async = require('async'); -const { errors, errorInstances, versioning, s3middleware } = require('arsenal'); +const { errors, errorInstances, versioning, s3middleware, models, jsutil } = require('arsenal'); const validateHeaders = s3middleware.validateConditionalHeaders; const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const constants = require('../../constants'); const { data } = require('../data/wrapper'); -const locationConstraintCheck = - require('./apiUtils/object/locationConstraintCheck'); +const locationConstraintCheck = require('./apiUtils/object/locationConstraintCheck'); const metadata = require('../metadata/wrapper'); const { pushMetric } = require('../utapi/utilities'); const services = require('../services'); @@ -17,11 +16,109 @@ const { verifyColdObjectAvailable } = require('./apiUtils/object/coldStorage'); const { validateQuotas } = require('./apiUtils/quotas/quotaUtils'); const { setSSEHeaders } = require('./apiUtils/object/sseHeaders'); const { initializeInternalLogRequestQueue, queueInternalLogRequest } = require('../utilities/serverAccessLogger'); +const { algorithms } = require('./apiUtils/integrity/validateChecksums'); +const { buildSourcePartsStream, computeChecksumFromDataLocator } = require('./apiUtils/object/sourceChecksum'); +const { config } = require('../Config'); +const kms = require('../kms/wrapper'); +const ChecksumTransform = require('../auth/streamingV4/ChecksumTransform'); const versionIdUtils = versioning.VersionID; +const { BackendInfo } = models; const skipError = new Error('skip'); +function _shouldRecomputeChecksum(request, sourceChecksum, algo) { + if (request.headers['x-amz-copy-source-range']) { + return true; + } + return !( + sourceChecksum && + sourceChecksum.checksumType === 'FULL_OBJECT' && + sourceChecksum.checksumAlgorithm === algo + ); +} + +/** + * Copy a part and calculate its checksum in a single streaming pass. + * + * @param {Array} dataLocator - ordered (range-adjusted) source parts + * @param {number} size - copied part size in bytes + * @param {object|null} sse - server-side encryption config for the dest MPU + * @param {string} destLocationConstraint - destination MPU location constraint + * @param {object} dataStoreContext - destination object data-store context + * @param {string} algo - checksum algorithm to compute + * @param {object} log - request logger + * @param {function} cb - cb(err, { locations, totalHash, checksum }) + * @return {undefined} + */ +function _copyPartStreamingWithChecksum( + dataLocator, + size, + sse, + destLocationConstraint, + dataStoreContext, + algo, + log, + cb, +) { + log.debug('recomputing checksum on UploadPartCopy', { algorithm: algo, size }); + const wrapChecksumErr = err => Object.assign(err, { checksumStream: { algorithm: algo } }); + const backendInfo = new BackendInfo(config, destLocationConstraint); + const sourceStream = buildSourcePartsStream(dataLocator, log); + const checksumStream = new ChecksumTransform(algo, undefined, false, log); + const done = jsutil.once((err, result) => { + if (err) { + sourceStream.destroy(err); + checksumStream.destroy(err); + return cb(err); + } + return cb(null, result); + }); + sourceStream.once('error', done); + checksumStream.once('error', err => done(wrapChecksumErr(err))); + sourceStream.pipe(checksumStream); + const doPut = cipherBundle => + data.put( + cipherBundle, + checksumStream, + size, + dataStoreContext, + backendInfo, + log, + (err, dataRetrievalInfo, hashedStream) => { + if (err) { + return done(err); + } + const location = { + key: dataRetrievalInfo.key, + dataStoreName: dataRetrievalInfo.dataStoreName, + dataStoreETag: hashedStream.completedHash, + size, + }; + if (cipherBundle) { + location.sseCryptoScheme = cipherBundle.cryptoScheme; + location.sseCipheredDataKey = cipherBundle.cipheredDataKey; + location.sseAlgorithm = cipherBundle.algorithm; + location.sseMasterKeyId = cipherBundle.masterKeyId; + } + return done(null, { + locations: [location], + totalHash: hashedStream.completedHash, + checksum: { algorithm: algo, value: checksumStream.digest }, + }); + }, + ); + if (sse && sse.algorithm) { + return kms.createCipherBundle(sse, log, (err, cipherBundle) => { + if (err) { + return done(err); + } + return doPut(cipherBundle); + }); + } + return doPut(null); +} + /** * PUT Part Copy during a multipart upload. * @param {AuthInfo} authInfo - Instance of AuthInfo class with @@ -35,8 +132,7 @@ const skipError = new Error('skip'); * @param {function} callback - final callback to call with the result * @return {undefined} */ -function objectPutCopyPart(authInfo, request, sourceBucket, - sourceObject, reqVersionId, log, callback) { +function objectPutCopyPart(authInfo, request, sourceBucket, sourceObject, reqVersionId, log, callback) { log.debug('processing request', { method: 'objectPutCopyPart' }); const destBucketName = request.bucketName; const destObjectKey = request.objectKey; @@ -62,8 +158,7 @@ function objectPutCopyPart(authInfo, request, sourceBucket, const partNumber = Number.parseInt(request.query.partNumber, 10); // AWS caps partNumbers at 10,000 if (partNumber > 10000 || !Number.isInteger(partNumber) || partNumber < 1) { - monitoring.promMetrics('PUT', destBucketName, 400, - 'putObjectCopyPart'); + monitoring.promMetrics('PUT', destBucketName, 400, 'putObjectCopyPart'); return callback(errors.InvalidArgument); } // We pad the partNumbers so that the parts will be sorted @@ -72,7 +167,9 @@ function objectPutCopyPart(authInfo, request, sourceBucket, // Note that keys in the query object retain their case, so // request.query.uploadId must be called with that exact // capitalization - const { query: { uploadId } } = request; + const { + query: { uploadId }, + } = request; const valPutParams = { authInfo, @@ -87,10 +184,13 @@ function objectPutCopyPart(authInfo, request, sourceBucket, // as validating for the destination bucket except additionally need // the uploadId and splitter. // Also, requestType is 'putPart or complete' - const valMPUParams = Object.assign({ - uploadId, - splitter: constants.splitter, - }, valPutParams); + const valMPUParams = Object.assign( + { + uploadId, + splitter: constants.splitter, + }, + valPutParams, + ); valMPUParams.requestType = 'putPart or complete'; const dataStoreContext = { @@ -103,131 +203,170 @@ function objectPutCopyPart(authInfo, request, sourceBucket, enableQuota: true, }; - return async.waterfall([ - function checkDestAuth(next) { - return standardMetadataValidateBucketAndObj(valPutParams, request.actionImplicitDenies, log, - (err, destBucketMD) => { - if (err) { - log.debug('error validating authorization for ' + - 'destination bucket', - { error: err }); - return next(err, destBucketMD); - } - const flag = destBucketMD.hasDeletedFlag() - || destBucketMD.hasTransientFlag(); - if (flag) { - log.trace('deleted flag or transient flag ' + - 'on destination bucket', { flag }); - return next(errors.NoSuchBucket); - } - return next(null, destBucketMD); - }); - }, - function checkSourceAuthorization(destBucketMD, next) { - return standardMetadataValidateBucketAndObj({ - ...valGetParams, - serverAccessLogOptions: { copySource: true }, - }, request.actionImplicitDenies, log, - (err, sourceBucketMD, sourceObjMD) => { - if (err) { - log.debug('error validating get part of request', - { error: err }); - // eslint-disable-next-line no-param-reassign - request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); - return next(err, destBucketMD); - } - if (!sourceObjMD) { - log.debug('no source object', { sourceObject }); - const err = reqVersionId ? errors.NoSuchVersion : - errors.NoSuchKey; - // eslint-disable-next-line no-param-reassign - request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); - return next(err, destBucketMD); - } - let sourceLocationConstraintName = - sourceObjMD.dataStoreName; - // for backwards compatibility before storing dataStoreName - // TODO: handle in objectMD class - if (!sourceLocationConstraintName && - sourceObjMD.location[0] && - sourceObjMD.location[0].dataStoreName) { - sourceLocationConstraintName = - sourceObjMD.location[0].dataStoreName; - } - // check if object data is in a cold storage - const coldErr = verifyColdObjectAvailable(sourceObjMD); - if (coldErr) { - // eslint-disable-next-line no-param-reassign - request.sourceServerAccessLog && (request.sourceServerAccessLog.error = coldErr); - return next(coldErr, null); - } - if (sourceObjMD.isDeleteMarker) { - log.debug('delete marker on source object', - { sourceObject }); - let err; - if (reqVersionId) { - err = errorInstances.InvalidRequest - .customizeDescription('The source of a copy ' + - 'request may not specifically refer to a delete' + - 'marker by version id.'); - } else { - // if user specifies a key in a versioned source bucket - // without specifying a version, and the object has a - // delete marker, return NoSuchKey - err = errors.NoSuchKey; + return async.waterfall( + [ + function checkDestAuth(next) { + return standardMetadataValidateBucketAndObj( + valPutParams, + request.actionImplicitDenies, + log, + (err, destBucketMD) => { + if (err) { + log.debug('error validating authorization for ' + 'destination bucket', { error: err }); + return next(err, destBucketMD); } - // eslint-disable-next-line no-param-reassign - request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); - return next(err, destBucketMD); - } - const headerValResult = - validateHeaders(request.headers, - sourceObjMD['last-modified'], - sourceObjMD['content-md5']); - if (headerValResult.error) { - request.sourceServerAccessLog + const flag = destBucketMD.hasDeletedFlag() || destBucketMD.hasTransientFlag(); + if (flag) { + log.trace('deleted flag or transient flag ' + 'on destination bucket', { flag }); + return next(errors.NoSuchBucket); + } + return next(null, destBucketMD); + }, + ); + }, + function checkSourceAuthorization(destBucketMD, next) { + return standardMetadataValidateBucketAndObj( + { + ...valGetParams, + serverAccessLogOptions: { copySource: true }, + }, + request.actionImplicitDenies, + log, + (err, sourceBucketMD, sourceObjMD) => { + if (err) { + log.debug('error validating get part of request', { error: err }); // eslint-disable-next-line no-param-reassign - && (request.sourceServerAccessLog.error = errors.PreconditionFailed); - return next(errors.PreconditionFailed, destBucketMD); - } - const copyLocator = setUpCopyLocator(sourceObjMD, - request.headers['x-amz-copy-source-range'], log); - if (copyLocator.error) { - // eslint-disable-next-line no-param-reassign - request.sourceServerAccessLog && (request.sourceServerAccessLog.error = copyLocator.error); - return next(copyLocator.error, destBucketMD); - } - let sourceVerId; - // If specific version requested, include copy source - // version id in response. Include in request by default - // if versioning is enabled or suspended. - if (sourceBucketMD.getVersioningConfiguration() || - reqVersionId) { - if (sourceObjMD.isNull || !sourceObjMD.versionId) { - sourceVerId = 'null'; - } else { - sourceVerId = - versionIdUtils.encode(sourceObjMD.versionId); + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); + return next(err, destBucketMD); } - } - return next(null, copyLocator.dataLocator, destBucketMD, - copyLocator.copyObjectSize, sourceVerId, - sourceLocationConstraintName, sourceObjMD); - }); - }, - function _validateQuotas(dataLocator, destBucketMD, - copyObjectSize, sourceVerId, - sourceLocationConstraintName, sourceObjMD, next) { - return validateQuotas(request, destBucketMD, request.accountQuotas, valPutParams.requestType, - request.apiMethod, sourceObjMD?.['content-length'] || 0, false, log, err => - next(err, dataLocator, destBucketMD, copyObjectSize, sourceVerId, sourceLocationConstraintName)); - }, - // get MPU shadow bucket to get splitter based on MD version - function getMpuShadowBucket(dataLocator, destBucketMD, - copyObjectSize, sourceVerId, - sourceLocationConstraintName, next) { - return metadata.getBucket(mpuBucketName, log, - (err, mpuBucket) => { + if (!sourceObjMD) { + log.debug('no source object', { sourceObject }); + const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey; + // eslint-disable-next-line no-param-reassign + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); + return next(err, destBucketMD); + } + let sourceLocationConstraintName = sourceObjMD.dataStoreName; + // for backwards compatibility before storing dataStoreName + // TODO: handle in objectMD class + if ( + !sourceLocationConstraintName && + sourceObjMD.location[0] && + sourceObjMD.location[0].dataStoreName + ) { + sourceLocationConstraintName = sourceObjMD.location[0].dataStoreName; + } + // check if object data is in a cold storage + const coldErr = verifyColdObjectAvailable(sourceObjMD); + if (coldErr) { + // eslint-disable-next-line no-param-reassign + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = coldErr); + return next(coldErr, null); + } + if (sourceObjMD.isDeleteMarker) { + log.debug('delete marker on source object', { sourceObject }); + let err; + if (reqVersionId) { + err = errorInstances.InvalidRequest.customizeDescription( + 'The source of a copy ' + + 'request may not specifically refer to a delete' + + 'marker by version id.', + ); + } else { + // if user specifies a key in a versioned source bucket + // without specifying a version, and the object has a + // delete marker, return NoSuchKey + err = errors.NoSuchKey; + } + // eslint-disable-next-line no-param-reassign + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); + return next(err, destBucketMD); + } + const headerValResult = validateHeaders( + request.headers, + sourceObjMD['last-modified'], + sourceObjMD['content-md5'], + ); + if (headerValResult.error) { + request.sourceServerAccessLog && + // eslint-disable-next-line no-param-reassign + (request.sourceServerAccessLog.error = errors.PreconditionFailed); + return next(errors.PreconditionFailed, destBucketMD); + } + const copyLocator = setUpCopyLocator( + sourceObjMD, + request.headers['x-amz-copy-source-range'], + log, + ); + if (copyLocator.error) { + // eslint-disable-next-line no-param-reassign + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = copyLocator.error); + return next(copyLocator.error, destBucketMD); + } + let sourceVerId; + // If specific version requested, include copy source + // version id in response. Include in request by default + // if versioning is enabled or suspended. + if (sourceBucketMD.getVersioningConfiguration() || reqVersionId) { + if (sourceObjMD.isNull || !sourceObjMD.versionId) { + sourceVerId = 'null'; + } else { + sourceVerId = versionIdUtils.encode(sourceObjMD.versionId); + } + } + return next( + null, + copyLocator.dataLocator, + destBucketMD, + copyLocator.copyObjectSize, + sourceVerId, + sourceLocationConstraintName, + sourceObjMD, + ); + }, + ); + }, + function _validateQuotas( + dataLocator, + destBucketMD, + copyObjectSize, + sourceVerId, + sourceLocationConstraintName, + sourceObjMD, + next, + ) { + return validateQuotas( + request, + destBucketMD, + request.accountQuotas, + valPutParams.requestType, + request.apiMethod, + sourceObjMD?.['content-length'] || 0, + false, + log, + err => + next( + err, + dataLocator, + destBucketMD, + copyObjectSize, + sourceVerId, + sourceLocationConstraintName, + sourceObjMD, + ), + ); + }, + // get MPU shadow bucket to get splitter based on MD version + function getMpuShadowBucket( + dataLocator, + destBucketMD, + copyObjectSize, + sourceVerId, + sourceLocationConstraintName, + sourceObjMD, + next, + ) { + return metadata.getBucket(mpuBucketName, log, (err, mpuBucket) => { // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver if (err && err.NoSuchBucket) { return next(errors.NoSuchUpload); @@ -243,105 +382,240 @@ function objectPutCopyPart(authInfo, request, sourceBucket, if (mpuBucket.getMdBucketModelVersion() < 2) { splitter = constants.oldSplitter; } - return next(null, dataLocator, destBucketMD, - copyObjectSize, sourceVerId, splitter, - sourceLocationConstraintName); + return next( + null, + dataLocator, + destBucketMD, + copyObjectSize, + sourceVerId, + splitter, + sourceLocationConstraintName, + sourceObjMD, + ); }); - }, - // Get MPU overview object to check authorization to put a part - // and to get any object location constraint info - function getMpuOverviewObject(dataLocator, destBucketMD, - copyObjectSize, sourceVerId, splitter, - sourceLocationConstraintName, next) { - const mpuOverviewKey = - `overview${splitter}${destObjectKey}${splitter}${uploadId}`; - return metadata.getObjectMD(mpuBucketName, mpuOverviewKey, - null, log, (err, res) => { - if (err) { - // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver - if (err.NoSuchKey) { - return next(errors.NoSuchUpload); - } - log.error('error getting overview object from ' + - 'mpu bucket', { - error: err, - method: 'objectPutCopyPart::' + - 'metadata.getObjectMD', - }); - return next(err); - } - const initiatorID = res.initiator.ID; - const requesterID = authInfo.isRequesterAnIAMUser() ? - authInfo.getArn() : authInfo.getCanonicalID(); - if (initiatorID !== requesterID) { - return next(errors.AccessDenied); - } - const destObjLocationConstraint = - res.controllingLocationConstraint; - const sseAlgo = res['x-amz-server-side-encryption']; - const sse = sseAlgo ? { - algorithm: sseAlgo, - masterKeyId: res['x-amz-server-side-encryption-aws-kms-key-id'], - } : null; - return next(null, dataLocator, destBucketMD, - destObjLocationConstraint, copyObjectSize, - sourceVerId, sourceLocationConstraintName, sse, splitter); - }); - }, - function goGetData( - dataLocator, - destBucketMD, - destObjLocationConstraint, - copyObjectSize, - sourceVerId, - sourceLocationConstraintName, - sse, - splitter, - next, - ) { - const originalIdentityAuthzResults = request.actionImplicitDenies; - // eslint-disable-next-line no-param-reassign - delete request.actionImplicitDenies; - data.uploadPartCopy( - request, - log, + }, + // Get MPU overview object to check authorization to put a part + // and to get any object location constraint info + function getMpuOverviewObject( + dataLocator, destBucketMD, + copyObjectSize, + sourceVerId, + splitter, sourceLocationConstraintName, - destObjLocationConstraint, - dataLocator, - dataStoreContext, - locationConstraintCheck, - sse, - (error, eTag, lastModified, serverSideEncryption, locations) => { - // eslint-disable-next-line no-param-reassign - request.actionImplicitDenies = originalIdentityAuthzResults; - if (error) { - if (error.message === 'skip') { - return next(skipError, destBucketMD, eTag, - lastModified, sourceVerId, - serverSideEncryption); + sourceObjMD, + next, + ) { + const mpuOverviewKey = `overview${splitter}${destObjectKey}${splitter}${uploadId}`; + return metadata.getObjectMD(mpuBucketName, mpuOverviewKey, null, log, (err, mpuOverviewMD) => { + if (err) { + // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver + if (err.NoSuchKey) { + return next(errors.NoSuchUpload); } - // eslint-disable-next-line no-param-reassign - request.sourceServerAccessLog && (request.sourceServerAccessLog.error = error); - return next(error, destBucketMD); + log.error('error getting overview object from ' + 'mpu bucket', { + error: err, + method: 'objectPutCopyPart::' + 'metadata.getObjectMD', + }); + return next(err); + } + const initiatorID = mpuOverviewMD.initiator.ID; + const requesterID = authInfo.isRequesterAnIAMUser() ? authInfo.getArn() : authInfo.getCanonicalID(); + if (initiatorID !== requesterID) { + return next(errors.AccessDenied); } - return next(null, destBucketMD, locations, eTag, - copyObjectSize, sourceVerId, serverSideEncryption, - lastModified, splitter); + const destObjLocationConstraint = mpuOverviewMD.controllingLocationConstraint; + const sseAlgo = mpuOverviewMD['x-amz-server-side-encryption']; + const sse = sseAlgo + ? { + algorithm: sseAlgo, + masterKeyId: mpuOverviewMD['x-amz-server-side-encryption-aws-kms-key-id'], + } + : null; + return next( + null, + dataLocator, + destBucketMD, + destObjLocationConstraint, + copyObjectSize, + sourceVerId, + sourceLocationConstraintName, + sse, + splitter, + sourceObjMD, + mpuOverviewMD, + ); }); - }, - function getExistingPartInfo(destBucketMD, locations, totalHash, - copyObjectSize, sourceVerId, serverSideEncryption, lastModified, - splitter, next) { - const partKey = - `${uploadId}${constants.splitter}${paddedPartNumber}`; - metadata.getObjectMD(mpuBucketName, partKey, {}, log, - (err, result) => { + }, + function goGetData( + dataLocator, + destBucketMD, + destObjLocationConstraint, + copyObjectSize, + sourceVerId, + sourceLocationConstraintName, + sse, + splitter, + sourceObjMD, + mpuOverviewMD, + next, + ) { + const originalIdentityAuthzResults = request.actionImplicitDenies; + // eslint-disable-next-line no-param-reassign + delete request.actionImplicitDenies; + + const algo = mpuOverviewMD.checksumAlgorithm; + const recompute = algo && _shouldRecomputeChecksum(request, sourceObjMD.checksum, algo); + // External backends need their own MPU API, so _copyPartStreamingWithChecksum (data.put) can't be used. + const destIsExternal = + constants.externalBackends[config.getLocationConstraintType(destObjLocationConstraint)]; + if (recompute && dataLocator.length > 0 && !destIsExternal) { + return _copyPartStreamingWithChecksum( + dataLocator, + copyObjectSize, + sse, + destObjLocationConstraint, + dataStoreContext, + algo, + log, + (err, result) => { + // eslint-disable-next-line no-param-reassign + request.actionImplicitDenies = originalIdentityAuthzResults; + if (err) { + // eslint-disable-next-line no-param-reassign + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = err); + return next(err, destBucketMD); + } + return next( + null, + destBucketMD, + result.locations, + result.totalHash, + copyObjectSize, + sourceVerId, + sse, + new Date().toJSON(), + splitter, + mpuOverviewMD, + result.checksum, + ); + }, + ); + } + + return data.uploadPartCopy( + request, + log, + destBucketMD, + sourceLocationConstraintName, + destObjLocationConstraint, + dataLocator, + dataStoreContext, + locationConstraintCheck, + sse, + (error, eTag, lastModified, serverSideEncryption, locations) => { + // eslint-disable-next-line no-param-reassign + request.actionImplicitDenies = originalIdentityAuthzResults; + const isSkip = error && error.message === 'skip'; + if (error && !isSkip) { + // eslint-disable-next-line no-param-reassign + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = error); + return next(error, destBucketMD); + } + + // Skip checksum compute for external backends matching UploadPart. + if (recompute && !destIsExternal) { + return computeChecksumFromDataLocator(dataLocator, algo, log, (cksErr, partChecksum) => { + if (cksErr) { + // eslint-disable-next-line no-param-reassign + request.sourceServerAccessLog && (request.sourceServerAccessLog.error = cksErr); + return next(cksErr, destBucketMD); + } + if (isSkip) { + return next( + skipError, + destBucketMD, + eTag, + lastModified, + sourceVerId, + serverSideEncryption, + undefined, + undefined, + mpuOverviewMD, + partChecksum, + ); + } + return next( + null, + destBucketMD, + locations, + eTag, + copyObjectSize, + sourceVerId, + serverSideEncryption, + lastModified, + splitter, + mpuOverviewMD, + partChecksum, + ); + }); + } + + // Reuse the source's stored checksum, or none for a legacy or + // external-backend MPU. + const partChecksum = + algo && !destIsExternal + ? { algorithm: algo, value: sourceObjMD.checksum.checksumValue } + : undefined; + if (isSkip) { + return next( + skipError, + destBucketMD, + eTag, + lastModified, + sourceVerId, + serverSideEncryption, + undefined, + undefined, + mpuOverviewMD, + partChecksum, + ); + } + return next( + null, + destBucketMD, + locations, + eTag, + copyObjectSize, + sourceVerId, + serverSideEncryption, + lastModified, + splitter, + mpuOverviewMD, + partChecksum, + ); + }, + ); + }, + function getExistingPartInfo( + destBucketMD, + locations, + totalHash, + copyObjectSize, + sourceVerId, + serverSideEncryption, + lastModified, + splitter, + mpuOverviewMD, + partChecksum, + next, + ) { + const partKey = `${uploadId}${constants.splitter}${paddedPartNumber}`; + metadata.getObjectMD(mpuBucketName, partKey, {}, log, (err, result) => { // If there is nothing being overwritten just move on // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver if (err && !err.NoSuchKey) { - log.debug('error getting current part (if any)', - { error: err }); + log.debug('error getting current part (if any)', { error: err }); return next(err); } let oldLocations; @@ -352,158 +626,277 @@ function objectPutCopyPart(authInfo, request, sourceBucket, // Pull locations to clean up any potential orphans // in data if object put is an overwrite of // already existing object with same key and part number - oldLocations = Array.isArray(oldLocations) ? - oldLocations : [oldLocations]; + oldLocations = Array.isArray(oldLocations) ? oldLocations : [oldLocations]; } - return next(null, destBucketMD, locations, totalHash, - prevObjectSize, copyObjectSize, sourceVerId, - serverSideEncryption, lastModified, oldLocations, splitter); + return next( + null, + destBucketMD, + locations, + totalHash, + prevObjectSize, + copyObjectSize, + sourceVerId, + serverSideEncryption, + lastModified, + oldLocations, + splitter, + mpuOverviewMD, + partChecksum, + ); }); - }, - function storeNewPartMetadata(destBucketMD, locations, totalHash, - prevObjectSize, copyObjectSize, sourceVerId, serverSideEncryption, - lastModified, oldLocations, splitter, next) { - const metaStoreParams = { - partNumber: paddedPartNumber, - contentMD5: totalHash, - size: copyObjectSize, - uploadId, - splitter: constants.splitter, + }, + function storeNewPartMetadata( + destBucketMD, + locations, + totalHash, + prevObjectSize, + copyObjectSize, + sourceVerId, + serverSideEncryption, lastModified, - overheadField: constants.overheadField, - ownerId: destBucketMD.getOwner(), - }; - return services.metadataStorePart(mpuBucketName, - locations, metaStoreParams, log, err => { + oldLocations, + splitter, + mpuOverviewMD, + partChecksum, + next, + ) { + const metaStoreParams = { + partNumber: paddedPartNumber, + contentMD5: totalHash, + size: copyObjectSize, + uploadId, + splitter: constants.splitter, + lastModified, + overheadField: constants.overheadField, + ownerId: destBucketMD.getOwner(), + }; + if (partChecksum) { + metaStoreParams.checksumValue = partChecksum.value; + metaStoreParams.checksumAlgorithm = partChecksum.algorithm; + } + return services.metadataStorePart(mpuBucketName, locations, metaStoreParams, log, err => { if (err) { - log.debug('error storing new metadata', - { error: err, method: 'storeNewPartMetadata' }); + log.debug('error storing new metadata', { error: err, method: 'storeNewPartMetadata' }); return next(err); } - return next(null, locations, oldLocations, destBucketMD, totalHash, - lastModified, sourceVerId, serverSideEncryption, - prevObjectSize, copyObjectSize, splitter); + return next( + null, + locations, + oldLocations, + destBucketMD, + totalHash, + lastModified, + sourceVerId, + serverSideEncryption, + prevObjectSize, + copyObjectSize, + splitter, + mpuOverviewMD, + partChecksum, + ); }); - }, - function checkCanDeleteOldLocations(partLocations, oldLocations, destBucketMD, - totalHash, lastModified, sourceVerId, serverSideEncryption, - prevObjectSize, copyObjectSize, splitter, next) { - if (!oldLocations) { - return next(null, oldLocations, destBucketMD, totalHash, - lastModified, sourceVerId, serverSideEncryption, - prevObjectSize, copyObjectSize); - } - return services.isCompleteMPUInProgress({ - bucketName: destBucketName, - objectKey: destObjectKey, - uploadId, + }, + function checkCanDeleteOldLocations( + partLocations, + oldLocations, + destBucketMD, + totalHash, + lastModified, + sourceVerId, + serverSideEncryption, + prevObjectSize, + copyObjectSize, splitter, - }, log, (err, completeInProgress) => { - if (err) { - return next(err, destBucketMD); + mpuOverviewMD, + partChecksum, + next, + ) { + if (!oldLocations) { + return next( + null, + oldLocations, + destBucketMD, + totalHash, + lastModified, + sourceVerId, + serverSideEncryption, + prevObjectSize, + copyObjectSize, + mpuOverviewMD, + partChecksum, + ); } - let oldLocationsToDelete = oldLocations; - // Prevent deletion of old data if a completeMPU - // is already in progress because then there is no - // guarantee that the old location will not be the - // committed one. - if (completeInProgress) { - log.warn('not deleting old locations because CompleteMPU is in progress', { - method: 'objectPutCopyPart::checkCanDeleteOldLocations', + return services.isCompleteMPUInProgress( + { bucketName: destBucketName, objectKey: destObjectKey, uploadId, - partLocations, - oldLocations, - }); - oldLocationsToDelete = null; - } - return next(null, oldLocationsToDelete, destBucketMD, totalHash, - lastModified, sourceVerId, serverSideEncryption, - prevObjectSize, copyObjectSize); - }); - }, - function cleanupExistingData(oldLocationsToDelete, destBucketMD, totalHash, - lastModified, sourceVerId, serverSideEncryption, - prevObjectSize, copyObjectSize, next) { - // Clean up the old data now that new metadata (with new - // data locations) has been stored - if (oldLocationsToDelete) { - return data.batchDelete(oldLocationsToDelete, request.method, null, - log, err => { + splitter, + }, + log, + (err, completeInProgress) => { + if (err) { + return next(err, destBucketMD); + } + let oldLocationsToDelete = oldLocations; + // Prevent deletion of old data if a completeMPU + // is already in progress because then there is no + // guarantee that the old location will not be the + // committed one. + if (completeInProgress) { + log.warn('not deleting old locations because CompleteMPU is in progress', { + method: 'objectPutCopyPart::checkCanDeleteOldLocations', + bucketName: destBucketName, + objectKey: destObjectKey, + uploadId, + partLocations, + oldLocations, + }); + oldLocationsToDelete = null; + } + return next( + null, + oldLocationsToDelete, + destBucketMD, + totalHash, + lastModified, + sourceVerId, + serverSideEncryption, + prevObjectSize, + copyObjectSize, + mpuOverviewMD, + partChecksum, + ); + }, + ); + }, + function cleanupExistingData( + oldLocationsToDelete, + destBucketMD, + totalHash, + lastModified, + sourceVerId, + serverSideEncryption, + prevObjectSize, + copyObjectSize, + mpuOverviewMD, + partChecksum, + next, + ) { + // Clean up the old data now that new metadata (with new + // data locations) has been stored + if (oldLocationsToDelete) { + return data.batchDelete(oldLocationsToDelete, request.method, null, log, err => { if (err) { // if error, log the error and move on as it is not // relevant to the client as the client's // object already succeeded putting data, metadata - log.error('error deleting existing data', - { error: err }); + log.error('error deleting existing data', { error: err }); } - return next(null, destBucketMD, totalHash, - lastModified, sourceVerId, serverSideEncryption, - prevObjectSize, copyObjectSize); + return next( + null, + destBucketMD, + totalHash, + lastModified, + sourceVerId, + serverSideEncryption, + prevObjectSize, + copyObjectSize, + mpuOverviewMD, + partChecksum, + ); }); - } - return next(null, destBucketMD, totalHash, - lastModified, sourceVerId, serverSideEncryption, - prevObjectSize, copyObjectSize); - }, - ], (err, destBucketMD, totalHash, lastModified, sourceVerId, - serverSideEncryption, prevObjectSize, copyObjectSize) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, destBucketMD); + } + return next( + null, + destBucketMD, + totalHash, + lastModified, + sourceVerId, + serverSideEncryption, + prevObjectSize, + copyObjectSize, + mpuOverviewMD, + partChecksum, + ); + }, + ], + ( + err, + destBucketMD, + totalHash, + lastModified, + sourceVerId, + serverSideEncryption, + prevObjectSize, + copyObjectSize, + mpuOverviewMD, + partChecksum, + ) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, destBucketMD); - // Store full object size for server access logs - if (request.serverAccessLog) { - // eslint-disable-next-line no-param-reassign - request.serverAccessLog.objectSize = copyObjectSize; - } + // Store full object size for server access logs + if (request.serverAccessLog) { + // eslint-disable-next-line no-param-reassign + request.serverAccessLog.objectSize = copyObjectSize; + } - // Initialize the queue for internal log request logging - initializeInternalLogRequestQueue(request); - // Queue the source-side access log (REST.COPY.PART_GET) - queueInternalLogRequest(request, { - operation: 'REST.COPY.PART_GET', - sourceBucket, - sourceObject, - objectSize: copyObjectSize || null, - }); + // Initialize the queue for internal log request logging + initializeInternalLogRequestQueue(request); + // Queue the source-side access log (REST.COPY.PART_GET) + queueInternalLogRequest(request, { + operation: 'REST.COPY.PART_GET', + sourceBucket, + sourceObject, + objectSize: copyObjectSize || null, + }); - if (err && err !== skipError) { - log.trace('error from copy part waterfall', - { error: err }); - monitoring.promMetrics('PUT', destBucketName, err.code, - 'putObjectCopyPart'); - return callback(err, null, corsHeaders); - } - const xml = [ - '', - '', - '', new Date(lastModified) - .toISOString(), '', - '"', totalHash, '"', - '', - ].join(''); + if (err && err !== skipError) { + log.trace('error from copy part waterfall', { error: err }); + monitoring.promMetrics('PUT', destBucketName, err.code, 'putObjectCopyPart'); + return callback(err, null, corsHeaders); + } + const xml = [ + '', + '', + '', + new Date(lastModified).toISOString(), + '', + '"', + totalHash, + '"', + ]; + // Surface the part checksum only for non-default MPUs like AWS. + if (partChecksum && !mpuOverviewMD.checksumIsDefault) { + const xmlTag = algorithms[partChecksum.algorithm] && algorithms[partChecksum.algorithm].xmlTag; + if (xmlTag) { + xml.push(`<${xmlTag}>`, partChecksum.value, ``); + } + } + xml.push(''); + const xmlStr = xml.join(''); - const additionalHeaders = corsHeaders || {}; - if (serverSideEncryption) { - setSSEHeaders(additionalHeaders, - serverSideEncryption.algorithm, - serverSideEncryption.masterKeyId); - } - additionalHeaders['x-amz-copy-source-version-id'] = sourceVerId; - pushMetric('uploadPartCopy', log, { - authInfo, - canonicalID: destBucketMD.getOwner(), - bucket: destBucketName, - keys: [destObjectKey], - newByteLength: copyObjectSize, - oldByteLength: prevObjectSize, - location: destBucketMD.getLocationConstraint(), - }); - monitoring.promMetrics( - 'PUT', destBucketName, '200', 'putObjectCopyPart'); - return callback(null, xml, additionalHeaders); - }); + const additionalHeaders = corsHeaders || {}; + if (serverSideEncryption) { + setSSEHeaders(additionalHeaders, serverSideEncryption.algorithm, serverSideEncryption.masterKeyId); + } + additionalHeaders['x-amz-copy-source-version-id'] = sourceVerId; + pushMetric('uploadPartCopy', log, { + authInfo, + canonicalID: destBucketMD.getOwner(), + bucket: destBucketName, + keys: [destObjectKey], + newByteLength: copyObjectSize, + oldByteLength: prevObjectSize, + location: destBucketMD.getLocationConstraint(), + }); + monitoring.promMetrics('PUT', destBucketName, '200', 'putObjectCopyPart'); + return callback(null, xmlStr, additionalHeaders); + }, + ); } module.exports = objectPutCopyPart; +// exported for unit tests +module.exports._shouldRecomputeChecksum = _shouldRecomputeChecksum; +module.exports._copyPartStreamingWithChecksum = _copyPartStreamingWithChecksum; diff --git a/lib/api/objectPutPart.js b/lib/api/objectPutPart.js index f115b8b727..d8c2d560cb 100644 --- a/lib/api/objectPutPart.js +++ b/lib/api/objectPutPart.js @@ -6,14 +6,12 @@ const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const constants = require('../../constants'); const { data } = require('../data/wrapper'); const { dataStore } = require('./apiUtils/object/storeObject'); -const { isBucketAuthorized } = - require('./apiUtils/authorization/permissionChecks'); +const { isBucketAuthorized } = require('./apiUtils/authorization/permissionChecks'); const kms = require('../kms/wrapper'); const metadata = require('../metadata/wrapper'); const { pushMetric } = require('../utapi/utilities'); const services = require('../services'); -const locationConstraintCheck - = require('./apiUtils/object/locationConstraintCheck'); +const locationConstraintCheck = require('./apiUtils/object/locationConstraintCheck'); const monitoring = require('../utilities/monitoringHandler'); const { config } = require('../Config'); const { BackendInfo } = models; @@ -21,10 +19,7 @@ const { BackendInfo } = models; const writeContinue = require('../utilities/writeContinue'); const { parseObjectEncryptionHeaders } = require('./apiUtils/bucket/bucketEncryption'); const validateChecksumHeaders = require('./apiUtils/object/validateChecksumHeaders'); -const { - getChecksumDataFromHeaders, - arsenalErrorFromChecksumError, -} = require('./apiUtils/integrity/validateChecksums'); +const { getChecksumDataFromHeaders, arsenalErrorFromChecksumError } = require('./apiUtils/integrity/validateChecksums'); const { validateQuotas } = require('./apiUtils/quotas/quotaUtils'); const { setSSEHeaders } = require('./apiUtils/object/sseHeaders'); const { storeServerAccessLogInfo } = require('../metadata/metadataUtils'); @@ -44,6 +39,12 @@ function _getPartKey(uploadId, splitter, paddedPartNumber) { return `${uploadId}${splitter}${paddedPartNumber}`; } +function checksumTypeMismatchErr(expected, actual) { + return errors.InvalidRequest.customizeDescription( + `Checksum Type mismatch occurred, expected checksum Type: ${expected}, ` + `actual checksum Type: ${actual}`, + ); +} + /** * PUT part of object during a multipart upload. Steps include: * validating metadata for authorization, bucket existence @@ -62,8 +63,7 @@ function _getPartKey(uploadId, splitter, paddedPartNumber) { * @param {function} cb - final callback to call with the result * @return {undefined} */ -function objectPutPart(authInfo, request, streamingV4Params, log, - cb) { +function objectPutPart(authInfo, request, streamingV4Params, log, cb) { log.debug('processing request', { method: 'objectPutPart' }); const size = request.parsedContentLength; @@ -72,8 +72,7 @@ function objectPutPart(authInfo, request, streamingV4Params, log, if (Number.parseInt(size, 10) > constants.maximumAllowedPartSize) { log.debug('put part size too large', { size }); - monitoring.promMetrics('PUT', request.bucketName, 400, - 'putObjectPart'); + monitoring.promMetrics('PUT', request.bucketName, 400, 'putObjectPart'); return cb(errors.EntityTooLarge); } @@ -91,13 +90,11 @@ function objectPutPart(authInfo, request, streamingV4Params, log, const partNumber = Number.parseInt(request.query.partNumber, 10); // AWS caps partNumbers at 10,000 if (partNumber > 10000) { - monitoring.promMetrics('PUT', request.bucketName, 400, - 'putObjectPart'); + monitoring.promMetrics('PUT', request.bucketName, 400, 'putObjectPart'); return cb(errors.TooManyParts); } if (!Number.isInteger(partNumber) || partNumber < 1) { - monitoring.promMetrics('PUT', request.bucketName, 400, - 'putObjectPart'); + monitoring.promMetrics('PUT', request.bucketName, 400, 'putObjectPart'); return cb(errors.InvalidArgument); } const bucketName = request.bucketName; @@ -109,7 +106,9 @@ function objectPutPart(authInfo, request, streamingV4Params, log, }); // Note that keys in the query object retain their case, so // `request.query.uploadId` must be called with that exact capitalization. - const { query: { uploadId } } = request; + const { + query: { uploadId }, + } = request; const mpuBucketName = `${constants.mpuBucketPrefix}${bucketName}`; const { objectKey } = request; const originalIdentityAuthzResults = request.actionImplicitDenies; @@ -118,76 +117,98 @@ function objectPutPart(authInfo, request, streamingV4Params, log, const requestType = request.apiMethods || 'objectPutPart'; let partChecksum; let mpuChecksumAlgo; + let mpuChecksumType; let mpuChecksumIsDefault; + let clientSuppliedChecksum; - return async.waterfall([ - // Get the destination bucket. - next => metadata.getBucket(bucketName, log, - (err, destinationBucket, raftSessionId) => { - if (err?.is?.NoSuchBucket) { - return next(errors.NoSuchBucket, destinationBucket); - } - if (err) { - log.error('error getting the destination bucket', { - error: err, - method: 'objectPutPart::metadata.getBucket', - }); - return next(err, destinationBucket); + return async.waterfall( + [ + // Get the destination bucket. + next => + metadata.getBucket(bucketName, log, (err, destinationBucket, raftSessionId) => { + if (err?.is?.NoSuchBucket) { + return next(errors.NoSuchBucket, destinationBucket); + } + if (err) { + log.error('error getting the destination bucket', { + error: err, + method: 'objectPutPart::metadata.getBucket', + }); + return next(err, destinationBucket); + } + storeServerAccessLogInfo(request, destinationBucket, raftSessionId); + return next(null, destinationBucket); + }), + // Check the bucket authorization. + (destinationBucket, next) => { + if ( + !isBucketAuthorized( + destinationBucket, + requestType, + canonicalID, + authInfo, + log, + request, + request.actionImplicitDenies, + ) + ) { + log.debug('access denied for user on bucket', { requestType }); + return next(errors.AccessDenied, destinationBucket); } - storeServerAccessLogInfo(request, destinationBucket, raftSessionId); return next(null, destinationBucket); - }), - // Check the bucket authorization. - (destinationBucket, next) => { - if (!isBucketAuthorized(destinationBucket, requestType, canonicalID, authInfo, - log, request, request.actionImplicitDenies)) { - log.debug('access denied for user on bucket', { requestType }); - return next(errors.AccessDenied, destinationBucket); - } - return next(null, destinationBucket); - }, - (destinationBucket, next) => validateQuotas(request, destinationBucket, request.accountQuotas, - requestType, request.apiMethod, size, isPutVersion, log, err => next(err, destinationBucket)), - // Validate that no object SSE is provided for part. - // Part must use SSE from initiateMPU (overview in metadata) - (destinationBucket, next) => { - const { error, objectSSE } = parseObjectEncryptionHeaders(request.headers); - if (error) { - return next(error, destinationBucket); - } - if (objectSSE.algorithm) { - return next(errors.InvalidArgument.customizeDescription( - 'x-amz-server-side-encryption header is not supported for this operation.')); - } - return next(null, destinationBucket); - }, - // Get the MPU shadow bucket. - (destinationBucket, next) => - metadata.getBucket(mpuBucketName, log, - (err, mpuBucket) => { - if (err?.is?.NoSuchBucket) { - return next(errors.NoSuchUpload, destinationBucket); - } - if (err) { - log.error('error getting the shadow mpu bucket', { - error: err, - method: 'objectPutPart::metadata.getBucket', - }); - return next(err, destinationBucket); + }, + (destinationBucket, next) => + validateQuotas( + request, + destinationBucket, + request.accountQuotas, + requestType, + request.apiMethod, + size, + isPutVersion, + log, + err => next(err, destinationBucket), + ), + // Validate that no object SSE is provided for part. + // Part must use SSE from initiateMPU (overview in metadata) + (destinationBucket, next) => { + const { error, objectSSE } = parseObjectEncryptionHeaders(request.headers); + if (error) { + return next(error, destinationBucket); } - let splitter = constants.splitter; - // BACKWARD: Remove to remove the old splitter - if (mpuBucket.getMdBucketModelVersion() < 2) { - splitter = constants.oldSplitter; + if (objectSSE.algorithm) { + return next( + errors.InvalidArgument.customizeDescription( + 'x-amz-server-side-encryption header is not supported for this operation.', + ), + ); } - return next(null, destinationBucket, splitter); - }), - // Check authorization of the MPU shadow bucket. - (destinationBucket, splitter, next) => { - const mpuOverviewKey = _getOverviewKey(splitter, objectKey, - uploadId); - return metadata.getObjectMD(mpuBucketName, mpuOverviewKey, {}, log, - (err, res) => { + return next(null, destinationBucket); + }, + // Get the MPU shadow bucket. + (destinationBucket, next) => + metadata.getBucket(mpuBucketName, log, (err, mpuBucket) => { + if (err?.is?.NoSuchBucket) { + return next(errors.NoSuchUpload, destinationBucket); + } + if (err) { + log.error('error getting the shadow mpu bucket', { + error: err, + method: 'objectPutPart::metadata.getBucket', + }); + return next(err, destinationBucket); + } + let splitter = constants.splitter; + // BACKWARD: Remove to remove the old splitter + if (mpuBucket.getMdBucketModelVersion() < 2) { + splitter = constants.oldSplitter; + } + return next(null, destinationBucket, splitter); + }), + // Check authorization of the MPU shadow bucket. + (destinationBucket, splitter, next) => { + const mpuOverviewKey = _getOverviewKey(splitter, objectKey, uploadId); + return metadata.getObjectMD(mpuBucketName, mpuOverviewKey, {}, log, (err, res) => { if (err) { log.error('error getting the object from mpu bucket', { error: err, @@ -196,88 +217,92 @@ function objectPutPart(authInfo, request, streamingV4Params, log, return next(err, destinationBucket); } const initiatorID = res.initiator.ID; - const requesterID = authInfo.isRequesterAnIAMUser() ? - authInfo.getArn() : authInfo.getCanonicalID(); + const requesterID = authInfo.isRequesterAnIAMUser() ? authInfo.getArn() : authInfo.getCanonicalID(); if (initiatorID !== requesterID) { return next(errors.AccessDenied, destinationBucket); } mpuChecksumAlgo = res.checksumAlgorithm; + mpuChecksumType = res.checksumType; mpuChecksumIsDefault = res.checksumIsDefault; - const objectLocationConstraint = - res.controllingLocationConstraint; + const objectLocationConstraint = res.controllingLocationConstraint; const sseAlgo = res['x-amz-server-side-encryption']; - const sse = sseAlgo ? { - algorithm: sseAlgo, - masterKeyId: res['x-amz-server-side-encryption-aws-kms-key-id'], - } : null; - return next(null, destinationBucket, - objectLocationConstraint, - sse, splitter); + const sse = sseAlgo + ? { + algorithm: sseAlgo, + masterKeyId: res['x-amz-server-side-encryption-aws-kms-key-id'], + } + : null; + return next(null, destinationBucket, objectLocationConstraint, sse, splitter); }); - }, - // Use MPU overview SSE config - (destinationBucket, objectLocationConstraint, encryption, splitter, next) => { - // If MPU has server-side encryption, pass the `res` value - if (encryption) { - return kms.createCipherBundle(encryption, log, (err, res) => { - if (err) { - log.error('error processing the cipher bundle for ' + - 'the destination bucket', { - error: err, - }); - return next(err, destinationBucket); - } - return next(null, destinationBucket, objectLocationConstraint, res, splitter); - // Allow KMS to use a key from previous provider (if sseMigration configured) - // Because ongoing MPU started before sseMigration is no migrated - }, { previousOk: true }); - } - // The MPU does not have server-side encryption, so pass `null` - return next(null, destinationBucket, objectLocationConstraint, null, splitter); - }, - // If data backend is backend that handles mpu (like real AWS), - // no need to store part info in metadata - (destinationBucket, objectLocationConstraint, cipherBundle, - splitter, next) => { - const mpuInfo = { - destinationBucket, - size, - objectKey, - uploadId, - partNumber, - bucketName, - }; - // eslint-disable-next-line no-param-reassign - delete request.actionImplicitDenies; - writeContinue(request, request._response); - return data.putPart(request, mpuInfo, streamingV4Params, - objectLocationConstraint, locationConstraintCheck, log, - (err, partInfo, updatedObjectLC) => { - if (err) { - return next(err, destinationBucket); - } - // if data backend handles mpu, skip to end of waterfall - // TODO CLDSRV-640 (artesca) data backend should return SSE to include in response headers - if (partInfo && partInfo.dataStoreType === 'aws_s3') { - return next(skipError, destinationBucket, - partInfo.dataStoreETag); + }, + // Use MPU overview SSE config + (destinationBucket, objectLocationConstraint, encryption, splitter, next) => { + // If MPU has server-side encryption, pass the `res` value + if (encryption) { + return kms.createCipherBundle( + encryption, + log, + (err, res) => { + if (err) { + log.error('error processing the cipher bundle for ' + 'the destination bucket', { + error: err, + }); + return next(err, destinationBucket); + } + return next(null, destinationBucket, objectLocationConstraint, res, splitter); + // Allow KMS to use a key from previous provider (if sseMigration configured) + // Because ongoing MPU started before sseMigration is no migrated + }, + { previousOk: true }, + ); } - // partInfo will be null if data backend is not external - // if the object location constraint undefined because - // mpu was initiated in legacy version, update it - return next(null, destinationBucket, updatedObjectLC, - cipherBundle, splitter, partInfo); - }); - }, - // Get any pre-existing part. - (destinationBucket, objectLocationConstraint, cipherBundle, - splitter, partInfo, next) => { - const paddedPartNumber = _getPaddedPartNumber(partNumber); - const partKey = _getPartKey(uploadId, splitter, paddedPartNumber); - return metadata.getObjectMD(mpuBucketName, partKey, {}, log, - (err, res) => { + // The MPU does not have server-side encryption, so pass `null` + return next(null, destinationBucket, objectLocationConstraint, null, splitter); + }, + // If data backend is backend that handles mpu (like real AWS), + // no need to store part info in metadata + (destinationBucket, objectLocationConstraint, cipherBundle, splitter, next) => { + const mpuInfo = { + destinationBucket, + size, + objectKey, + uploadId, + partNumber, + bucketName, + }; + // eslint-disable-next-line no-param-reassign + delete request.actionImplicitDenies; + writeContinue(request, request._response); + return data.putPart( + request, + mpuInfo, + streamingV4Params, + objectLocationConstraint, + locationConstraintCheck, + log, + (err, partInfo, updatedObjectLC) => { + if (err) { + return next(err, destinationBucket); + } + // if data backend handles mpu, skip to end of waterfall + // TODO CLDSRV-640 (artesca) data backend should return SSE to include in response headers + if (partInfo && partInfo.dataStoreType === 'aws_s3') { + return next(skipError, destinationBucket, partInfo.dataStoreETag); + } + // partInfo will be null if data backend is not external + // if the object location constraint undefined because + // mpu was initiated in legacy version, update it + return next(null, destinationBucket, updatedObjectLC, cipherBundle, splitter, partInfo); + }, + ); + }, + // Get any pre-existing part. + (destinationBucket, objectLocationConstraint, cipherBundle, splitter, partInfo, next) => { + const paddedPartNumber = _getPaddedPartNumber(partNumber); + const partKey = _getPartKey(uploadId, splitter, paddedPartNumber); + return metadata.getObjectMD(mpuBucketName, partKey, {}, log, (err, res) => { // If there is no object with the same key, continue. if (err && !err.is.NoSuchKey) { log.error('error getting current part (if any)', { @@ -295,123 +320,179 @@ function objectPutPart(authInfo, request, streamingV4Params, log, // Pull locations to clean up any potential orphans in // data if object put is an overwrite of a pre-existing // object with the same key and part number. - oldLocations = Array.isArray(res.partLocations) ? - res.partLocations : [res.partLocations]; + oldLocations = Array.isArray(res.partLocations) ? res.partLocations : [res.partLocations]; } - return next(null, destinationBucket, - objectLocationConstraint, cipherBundle, - partKey, prevObjectSize, oldLocations, partInfo, splitter); + return next( + null, + destinationBucket, + objectLocationConstraint, + cipherBundle, + partKey, + prevObjectSize, + oldLocations, + partInfo, + splitter, + ); }); - }, - // Store in data backend. - (destinationBucket, objectLocationConstraint, cipherBundle, - partKey, prevObjectSize, oldLocations, partInfo, splitter, next) => { - // NOTE: set oldLocations to null so we do not batchDelete for now - if (partInfo && - constants.skipBatchDeleteBackends[partInfo.dataStoreType]) { - // skip to storing metadata - return next(null, destinationBucket, partInfo, - partInfo.dataStoreETag, - cipherBundle, partKey, prevObjectSize, null, - objectLocationConstraint, splitter); - } - const objectContext = { - bucketName, - owner: canonicalID, - namespace: request.namespace, - objectKey, - partNumber: _getPaddedPartNumber(partNumber), - uploadId, - }; - const backendInfo = new BackendInfo(config, - objectLocationConstraint); + }, + // Store in data backend. + ( + destinationBucket, + objectLocationConstraint, + cipherBundle, + partKey, + prevObjectSize, + oldLocations, + partInfo, + splitter, + next, + ) => { + // NOTE: set oldLocations to null so we do not batchDelete for now + if (partInfo && constants.skipBatchDeleteBackends[partInfo.dataStoreType]) { + // skip to storing metadata + return next( + null, + destinationBucket, + partInfo, + partInfo.dataStoreETag, + cipherBundle, + partKey, + prevObjectSize, + null, + objectLocationConstraint, + splitter, + ); + } + const objectContext = { + bucketName, + owner: canonicalID, + namespace: request.namespace, + objectKey, + partNumber: _getPaddedPartNumber(partNumber), + uploadId, + }; + const backendInfo = new BackendInfo(config, objectLocationConstraint); - const headerChecksum = getChecksumDataFromHeaders(request.headers); - if (headerChecksum && headerChecksum.error) { - return next(arsenalErrorFromChecksumError(headerChecksum), destinationBucket); - } + const headerChecksum = getChecksumDataFromHeaders(request.headers); + if (headerChecksum && headerChecksum.error) { + return next(arsenalErrorFromChecksumError(headerChecksum), destinationBucket); + } + // Whether the client sent a per-part checksum header (vs. one the + // server computes implicitly). Drives whether we echo it back. + clientSuppliedChecksum = !!headerChecksum; - // If the MPU specifies a non-default checksum algo and the - // client sends a different algo, reject the request. - if (headerChecksum && mpuChecksumAlgo && !mpuChecksumIsDefault - && headerChecksum.algorithm !== mpuChecksumAlgo) { - return next(errors.InvalidRequest.customizeDescription( - `Checksum algorithm '${headerChecksum.algorithm}' is not the same ` + - `as the checksum algorithm '${mpuChecksumAlgo}' specified during ` + - 'CreateMultipartUpload.' - ), destinationBucket); - } + // If the MPU specifies a non-default checksum algo and the + // client sends a different algo, reject the request. + if ( + headerChecksum && + mpuChecksumAlgo && + !mpuChecksumIsDefault && + headerChecksum.algorithm !== mpuChecksumAlgo + ) { + return next(checksumTypeMismatchErr(mpuChecksumAlgo, headerChecksum.algorithm), destinationBucket); + } - const primaryAlgo = mpuChecksumAlgo || 'crc64nvme'; - let checksums; - if (headerChecksum && headerChecksum.algorithm === mpuChecksumAlgo) { - checksums = { - primary: headerChecksum, // MPU and Header match only need to calculate one. - secondary: null, - }; - } else if (headerChecksum) { - checksums = { - primary: { algorithm: primaryAlgo, isTrailer: false, expected: undefined }, - secondary: headerChecksum, // MPU and Header mismatch, need to verify the header checksum. - }; - } else { - checksums = { - primary: { algorithm: primaryAlgo, isTrailer: false, expected: undefined }, - secondary: null, // No Header checksum, we only calculate the MPU one. + // A COMPOSITE MPU's final checksum is composed from the per-part + // checksums, so every part must carry one. + if (!headerChecksum && mpuChecksumType === 'COMPOSITE') { + return next(checksumTypeMismatchErr(mpuChecksumAlgo, 'null'), destinationBucket); + } + + const primaryAlgo = mpuChecksumAlgo || 'crc64nvme'; + let checksums; + if (headerChecksum && headerChecksum.algorithm === mpuChecksumAlgo) { + checksums = { + primary: headerChecksum, // MPU and Header match only need to calculate one. + secondary: null, + }; + } else if (headerChecksum) { + checksums = { + primary: { algorithm: primaryAlgo, isTrailer: false, expected: undefined }, + secondary: headerChecksum, // MPU and Header mismatch, need to verify the header checksum. + }; + } else { + checksums = { + primary: { algorithm: primaryAlgo, isTrailer: false, expected: undefined }, + secondary: null, // No Header checksum, we only calculate the MPU one. + }; + } + return dataStore( + objectContext, + cipherBundle, + request, + size, + streamingV4Params, + backendInfo, + checksums, + log, + (err, dataGetInfo, hexDigest, checksum) => { + if (err) { + return next(err, destinationBucket); + } + partChecksum = checksum; + return next( + null, + destinationBucket, + dataGetInfo, + hexDigest, + cipherBundle, + partKey, + prevObjectSize, + oldLocations, + objectLocationConstraint, + splitter, + ); + }, + ); + }, + // Store data locations in metadata and delete any overwritten + // data if completeMPU hasn't been initiated yet. + ( + destinationBucket, + dataGetInfo, + hexDigest, + cipherBundle, + partKey, + prevObjectSize, + oldLocations, + objectLocationConstraint, + splitter, + next, + ) => { + // Use an array to be consistent with objectPutCopyPart where there + // could be multiple locations. + const partLocations = [dataGetInfo]; + const sseHeaders = {}; + if (cipherBundle) { + const { algorithm, masterKeyId, cryptoScheme, cipheredDataKey } = cipherBundle; + partLocations[0].sseAlgorithm = algorithm; + partLocations[0].sseMasterKeyId = masterKeyId; + partLocations[0].sseCryptoScheme = cryptoScheme; + partLocations[0].sseCipheredDataKey = cipheredDataKey; + sseHeaders.algo = algorithm; + sseHeaders.kmsKey = masterKeyId; + } + const omVal = { + // back to Version 3 since number-subparts is not needed + 'md-model-version': 3, + partLocations, + key: partKey, + 'last-modified': new Date().toJSON(), + 'content-md5': hexDigest, + 'content-length': size, + 'owner-id': destinationBucket.getOwner(), }; - } - return dataStore(objectContext, cipherBundle, request, - size, streamingV4Params, backendInfo, checksums, log, - (err, dataGetInfo, hexDigest, checksum) => { - if (err) { - return next(err, destinationBucket); + if (partChecksum) { + if (partChecksum.storageChecksum) { + omVal.checksumValue = partChecksum.storageChecksum.value; + omVal.checksumAlgorithm = partChecksum.storageChecksum.algorithm; + } else { + omVal.checksumValue = partChecksum.value; + omVal.checksumAlgorithm = partChecksum.algorithm; } - partChecksum = checksum; - return next(null, destinationBucket, dataGetInfo, hexDigest, - cipherBundle, partKey, prevObjectSize, oldLocations, - objectLocationConstraint, splitter); - }); - }, - // Store data locations in metadata and delete any overwritten - // data if completeMPU hasn't been initiated yet. - (destinationBucket, dataGetInfo, hexDigest, cipherBundle, partKey, - prevObjectSize, oldLocations, objectLocationConstraint, splitter, next) => { - // Use an array to be consistent with objectPutCopyPart where there - // could be multiple locations. - const partLocations = [dataGetInfo]; - const sseHeaders = {}; - if (cipherBundle) { - const { algorithm, masterKeyId, cryptoScheme, - cipheredDataKey } = cipherBundle; - partLocations[0].sseAlgorithm = algorithm; - partLocations[0].sseMasterKeyId = masterKeyId; - partLocations[0].sseCryptoScheme = cryptoScheme; - partLocations[0].sseCipheredDataKey = cipheredDataKey; - sseHeaders.algo = algorithm; - sseHeaders.kmsKey = masterKeyId; - } - const omVal = { - // back to Version 3 since number-subparts is not needed - 'md-model-version': 3, - partLocations, - 'key': partKey, - 'last-modified': new Date().toJSON(), - 'content-md5': hexDigest, - 'content-length': size, - 'owner-id': destinationBucket.getOwner(), - }; - if (partChecksum) { - if (partChecksum.storageChecksum) { - omVal.checksumValue = partChecksum.storageChecksum.value; - omVal.checksumAlgorithm = partChecksum.storageChecksum.algorithm; - } else { - omVal.checksumValue = partChecksum.value; - omVal.checksumAlgorithm = partChecksum.algorithm; } - } - const mdParams = { overheadField: constants.overheadField }; - return metadata.putObjectMD(mpuBucketName, partKey, omVal, mdParams, log, - err => { + const mdParams = { overheadField: constants.overheadField }; + return metadata.putObjectMD(mpuBucketName, partKey, omVal, mdParams, log, err => { if (err) { log.error('error putting object in mpu bucket', { error: err, @@ -419,104 +500,150 @@ function objectPutPart(authInfo, request, streamingV4Params, log, }); return next(err, destinationBucket); } - return next(null, partLocations, oldLocations, objectLocationConstraint, - destinationBucket, hexDigest, sseHeaders, prevObjectSize, splitter); + return next( + null, + partLocations, + oldLocations, + objectLocationConstraint, + destinationBucket, + hexDigest, + sseHeaders, + prevObjectSize, + splitter, + ); }); - }, - (partLocations, oldLocations, objectLocationConstraint, destinationBucket, - hexDigest, sseHeaders, prevObjectSize, splitter, next) => { - if (!oldLocations) { - return next(null, oldLocations, objectLocationConstraint, - destinationBucket, hexDigest, sseHeaders, prevObjectSize); - } - return services.isCompleteMPUInProgress({ - bucketName, - objectKey, - uploadId, + }, + ( + partLocations, + oldLocations, + objectLocationConstraint, + destinationBucket, + hexDigest, + sseHeaders, + prevObjectSize, splitter, - }, log, (err, completeInProgress) => { - if (err) { - return next(err, destinationBucket); + next, + ) => { + if (!oldLocations) { + return next( + null, + oldLocations, + objectLocationConstraint, + destinationBucket, + hexDigest, + sseHeaders, + prevObjectSize, + ); } - let oldLocationsToDelete = oldLocations; - // Prevent deletion of old data if a completeMPU - // is already in progress because then there is no - // guarantee that the old location will not be the - // committed one. - if (completeInProgress) { - log.warn('not deleting old locations because CompleteMPU is in progress', { - method: 'objectPutPart::metadata.getObjectMD', + return services.isCompleteMPUInProgress( + { bucketName, objectKey, uploadId, - partLocations, - oldLocations, - }); - oldLocationsToDelete = null; - } - return next(null, oldLocationsToDelete, objectLocationConstraint, - destinationBucket, hexDigest, sseHeaders, prevObjectSize); - }); - }, - // Clean up any old data now that new metadata (with new - // data locations) has been stored. - (oldLocationsToDelete, objectLocationConstraint, destinationBucket, hexDigest, - sseHeaders, prevObjectSize, next) => { - if (oldLocationsToDelete) { - log.trace('overwriting mpu part, deleting data'); - return data.batchDelete(oldLocationsToDelete, request.method, - objectLocationConstraint, log, err => { + splitter, + }, + log, + (err, completeInProgress) => { if (err) { - // if error, log the error and move on as it is not - // relevant to the client as the client's - // object already succeeded putting data, metadata - log.error('error deleting existing data', - { error: err }); + return next(err, destinationBucket); } - return next(null, destinationBucket, hexDigest, - sseHeaders, prevObjectSize); - }); + let oldLocationsToDelete = oldLocations; + // Prevent deletion of old data if a completeMPU + // is already in progress because then there is no + // guarantee that the old location will not be the + // committed one. + if (completeInProgress) { + log.warn('not deleting old locations because CompleteMPU is in progress', { + method: 'objectPutPart::metadata.getObjectMD', + bucketName, + objectKey, + uploadId, + partLocations, + oldLocations, + }); + oldLocationsToDelete = null; + } + return next( + null, + oldLocationsToDelete, + objectLocationConstraint, + destinationBucket, + hexDigest, + sseHeaders, + prevObjectSize, + ); + }, + ); + }, + // Clean up any old data now that new metadata (with new + // data locations) has been stored. + ( + oldLocationsToDelete, + objectLocationConstraint, + destinationBucket, + hexDigest, + sseHeaders, + prevObjectSize, + next, + ) => { + if (oldLocationsToDelete) { + log.trace('overwriting mpu part, deleting data'); + return data.batchDelete( + oldLocationsToDelete, + request.method, + objectLocationConstraint, + log, + err => { + if (err) { + // if error, log the error and move on as it is not + // relevant to the client as the client's + // object already succeeded putting data, metadata + log.error('error deleting existing data', { error: err }); + } + return next(null, destinationBucket, hexDigest, sseHeaders, prevObjectSize); + }, + ); + } + return next(null, destinationBucket, hexDigest, sseHeaders, prevObjectSize); + }, + ], + (err, destinationBucket, hexDigest, sseHeaders, prevObjectSize) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, destinationBucket); + // eslint-disable-next-line no-param-reassign + request.actionImplicitDenies = originalIdentityAuthzResults; + if (sseHeaders) { + setSSEHeaders(corsHeaders, sseHeaders.algo, sseHeaders.kmsKey); } - return next(null, destinationBucket, hexDigest, - sseHeaders, prevObjectSize); - }, - ], (err, destinationBucket, hexDigest, sseHeaders, prevObjectSize) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, destinationBucket); - // eslint-disable-next-line no-param-reassign - request.actionImplicitDenies = originalIdentityAuthzResults; - if (sseHeaders) { - setSSEHeaders(corsHeaders, sseHeaders.algo, sseHeaders.kmsKey); - } - if (err) { - if (err === skipError) { - return cb(null, hexDigest, corsHeaders); + if (err) { + if (err === skipError) { + return cb(null, hexDigest, corsHeaders); + } + log.error('error in object put part (upload part)', { + error: err, + method: 'objectPutPart', + }); + monitoring.promMetrics('PUT', bucketName, err.code, 'putObjectPart'); + return cb(err, null, corsHeaders); } - log.error('error in object put part (upload part)', { - error: err, - method: 'objectPutPart', + // Surface the part checksum unless it is the server-computed default. + // A client-supplied checksum, and any explicit-algorithm MPU, is still echoed - matching AWS. + if (partChecksum && (!mpuChecksumIsDefault || clientSuppliedChecksum)) { + const { algorithm, value } = partChecksum; + corsHeaders[`x-amz-checksum-${algorithm}`] = value; + } + pushMetric('uploadPart', log, { + authInfo, + canonicalID: destinationBucket.getOwner(), + bucket: bucketName, + keys: [objectKey], + newByteLength: size, + oldByteLength: prevObjectSize, + location: destinationBucket.getLocationConstraint(), }); - monitoring.promMetrics('PUT', bucketName, err.code, - 'putObjectPart'); - return cb(err, null, corsHeaders); - } - if (partChecksum) { - const { algorithm, value } = partChecksum; - corsHeaders[`x-amz-checksum-${algorithm}`] = value; - } - pushMetric('uploadPart', log, { - authInfo, - canonicalID: destinationBucket.getOwner(), - bucket: bucketName, - keys: [objectKey], - newByteLength: size, - oldByteLength: prevObjectSize, - location: destinationBucket.getLocationConstraint(), - }); - monitoring.promMetrics('PUT', bucketName, - '200', 'putObjectPart', size, prevObjectSize); - return cb(null, hexDigest, corsHeaders); - }); + monitoring.promMetrics('PUT', bucketName, '200', 'putObjectPart', size, prevObjectSize); + return cb(null, hexDigest, corsHeaders); + }, + ); } module.exports = objectPutPart; diff --git a/lib/services.js b/lib/services.js index 3d515e7f8d..5d07c79753 100644 --- a/lib/services.js +++ b/lib/services.js @@ -858,8 +858,18 @@ const services = { */ metadataStorePart(mpuBucketName, partLocations, metaStoreParams, log, cb) { assert.strictEqual(typeof mpuBucketName, 'string'); - const { partNumber, contentMD5, size, uploadId, lastModified, splitter, overheadField, ownerId } = - metaStoreParams; + const { + partNumber, + contentMD5, + size, + uploadId, + lastModified, + splitter, + overheadField, + ownerId, + checksumValue, + checksumAlgorithm, + } = metaStoreParams; const dateModified = typeof lastModified === 'string' ? lastModified : new Date().toJSON(); assert.strictEqual(typeof splitter, 'string'); const partKey = `${uploadId}${splitter}${partNumber}`; @@ -874,6 +884,10 @@ const services = { 'content-length': size, 'owner-id': ownerId, }; + if (checksumValue && checksumAlgorithm) { + omVal.checksumValue = checksumValue; + omVal.checksumAlgorithm = checksumAlgorithm; + } const params = {}; if (overheadField) { diff --git a/tests/functional/aws-node-sdk/test/object/completeMpuChecksum.js b/tests/functional/aws-node-sdk/test/object/completeMpuChecksum.js index 8ed43e9a80..557b2f366e 100644 --- a/tests/functional/aws-node-sdk/test/object/completeMpuChecksum.js +++ b/tests/functional/aws-node-sdk/test/object/completeMpuChecksum.js @@ -184,6 +184,78 @@ describe('CompleteMultipartUpload final-object checksum', () => assert.strictEqual(head.ChecksumType, 'FULL_OBJECT'); }); + describe('SDK-style checksum forwarding', () => { + let fwdS3; + const checksumFields = [ + 'ChecksumCRC32', + 'ChecksumCRC32C', + 'ChecksumCRC64NVME', + 'ChecksumSHA1', + 'ChecksumSHA256', + ]; + // explicit algorithms + the no-algorithm default (null) + const configs = ['CRC32', 'CRC32C', 'CRC64NVME', 'SHA1', 'SHA256', null]; + + before(() => { + // WHEN_REQUIRED: the SDK sends a per-part checksum only when we + // explicitly provide one, and nothing for the default MPU. + fwdS3 = new BucketUtility('default', { + ...sigCfg, + requestChecksumCalculation: 'WHEN_REQUIRED', + responseChecksumValidation: 'WHEN_REQUIRED', + }).s3; + }); + + configs.forEach(algo => { + const label = algo || 'no algorithm (default)'; + it(`should forward the UploadPart checksum and complete (${label})`, async () => { + const key = `complete-forward-${(algo || 'default').toLowerCase()}-${Date.now()}`; + const create = await fwdS3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ...(algo ? { ChecksumAlgorithm: algo } : {}), + }), + ); + + const uploadParams = { + Bucket: bucket, + Key: key, + UploadId: create.UploadId, + PartNumber: 1, + Body: partBody, + }; + // Explicit-algo MPUs require the matching per-part checksum. + if (algo) { + uploadParams[tagField(algo)] = await algorithms[algo.toLowerCase()].digest(partBody); + } + const uploadPart = await fwdS3.send(new UploadPartCommand(uploadParams)); + + // Forward whatever checksum the UploadPart response surfaced. + const completedPart = { PartNumber: 1, ETag: uploadPart.ETag }; + checksumFields.forEach(f => { + if (uploadPart[f] !== undefined) { + completedPart[f] = uploadPart[f]; + } + }); + + const complete = await fwdS3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: create.UploadId, + MultipartUpload: { Parts: [completedPart] }, + }), + ); + const expectedField = algo ? tagField(algo) : 'ChecksumCRC64NVME'; + assert( + complete[expectedField], + `expected ${expectedField} on CompleteMPU response, got: ${JSON.stringify(complete)}`, + ); + }); + }); + }); + // AWS S3 rejects any per-part // Checksum field on a default MPU (one created without an // explicit ChecksumAlgorithm) with InvalidPart — even when the diff --git a/tests/functional/aws-node-sdk/test/object/copyPartChecksum.js b/tests/functional/aws-node-sdk/test/object/copyPartChecksum.js new file mode 100644 index 0000000000..d09ebaf949 --- /dev/null +++ b/tests/functional/aws-node-sdk/test/object/copyPartChecksum.js @@ -0,0 +1,423 @@ +const assert = require('assert'); +const { + CreateBucketCommand, + PutObjectCommand, + CreateMultipartUploadCommand, + UploadPartCommand, + UploadPartCopyCommand, + CompleteMultipartUploadCommand, + ListPartsCommand, + AbortMultipartUploadCommand, + DeleteBucketCommand, +} = require('@aws-sdk/client-s3'); + +const withV4 = require('../support/withV4'); +const BucketUtility = require('../../lib/utility/bucket-util'); +const { algorithms } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); + +const bucket = `copypart-checksum-${Date.now()}`; +const sourceKey = 'copypart-checksum-source'; +const sourceBody = Buffer.from('UploadPartCopy checksum source content', 'utf8'); +const bigSourceKey = 'copypart-checksum-big-source'; +const bigBody = Buffer.alloc(5 * 1024 * 1024, 0x61); + +// algo -> the SDK CopyPartResult / CompletedPart field name +const field = algo => `Checksum${algo}`; +const allFields = ['CRC32', 'CRC32C', 'CRC64NVME', 'SHA1', 'SHA256'].map(field); +const digest = (algo, body) => algorithms[algo.toLowerCase()].digest(body); + +describe('UploadPartCopy checksums', () => + withV4(sigCfg => { + let s3; + let bucketUtil; + const openUploads = []; + + before(async () => { + // WHEN_REQUIRED so the SDK does not auto-attach checksums on + // CreateMPU/CompleteMPU and muddy the assertions. UploadPartCopy + // itself never sends a body checksum. + bucketUtil = new BucketUtility('default', { + ...sigCfg, + requestChecksumCalculation: 'WHEN_REQUIRED', + responseChecksumValidation: 'WHEN_REQUIRED', + }); + s3 = bucketUtil.s3; + await s3.send(new CreateBucketCommand({ Bucket: bucket })); + // Source stored without a checksum, so the copy always recomputes. + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: sourceKey, Body: sourceBody })); + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: bigSourceKey, Body: bigBody })); + }); + + after(async () => { + await Promise.all( + openUploads.map(u => + s3 + .send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: u.key, + UploadId: u.uploadId, + }), + ) + .catch(() => undefined), + ), + ); + await bucketUtil.empty(bucket); + await s3.send(new DeleteBucketCommand({ Bucket: bucket })); + }); + + async function createMpu(key, opts = {}) { + const res = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ...opts, + }), + ); + openUploads.push({ key, uploadId: res.UploadId }); + return res.UploadId; + } + + function copyPart(key, uploadId, extra = {}) { + return s3.send( + new UploadPartCopyCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + PartNumber: 1, + CopySource: `${bucket}/${sourceKey}`, + ...extra, + }), + ); + } + + ['CRC32', 'CRC32C', 'CRC64NVME', 'SHA1', 'SHA256'].forEach(algo => { + it(`should return the recomputed ${algo} checksum in CopyPartResult`, async () => { + const key = `cpr-${algo}`; + const uploadId = await createMpu(key, { ChecksumAlgorithm: algo }); + const res = await copyPart(key, uploadId); + assert.strictEqual(res.CopyPartResult[field(algo)], await digest(algo, sourceBody)); + }); + }); + + it('should not return any checksum in CopyPartResult for a default MPU', async () => { + const key = 'cpr-default'; + const uploadId = await createMpu(key); + const res = await copyPart(key, uploadId); + assert(res.CopyPartResult.ETag); + allFields.forEach(f => + assert.strictEqual( + res.CopyPartResult[f], + undefined, + `default MPU CopyPartResult should not include ${f}`, + ), + ); + }); + + it('should recompute in the MPU algorithm when the source has a different one', async () => { + // Source stored with CRC32; destination MPU is SHA256 -> recompute. + const srcCrc32 = 'copypart-checksum-source-crc32'; + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: srcCrc32, + Body: sourceBody, + ChecksumAlgorithm: 'CRC32', + }), + ); + const key = 'cpr-mismatch'; + const uploadId = await createMpu(key, { ChecksumAlgorithm: 'SHA256' }); + const res = await copyPart(key, uploadId, { CopySource: `${bucket}/${srcCrc32}` }); + assert.strictEqual(res.CopyPartResult.ChecksumSHA256, await digest('SHA256', sourceBody)); + }); + + it('should checksum only the copied byte range', async () => { + const key = 'cpr-range'; + const uploadId = await createMpu(key, { ChecksumAlgorithm: 'CRC32' }); + const res = await copyPart(key, uploadId, { CopySourceRange: 'bytes=0-3' }); + assert.strictEqual(res.CopyPartResult.ChecksumCRC32, await digest('CRC32', sourceBody.subarray(0, 4))); + }); + + it('should checksum a 0-byte copied part', async () => { + const emptyKey = 'copypart-checksum-empty'; + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: emptyKey, Body: '' })); + const key = 'cpr-empty'; + const uploadId = await createMpu(key, { ChecksumAlgorithm: 'CRC32' }); + const res = await copyPart(key, uploadId, { CopySource: `${bucket}/${emptyKey}` }); + assert.strictEqual(res.CopyPartResult.ChecksumCRC32, await digest('CRC32', Buffer.alloc(0))); + }); + + [ + { algo: 'CRC32', type: 'COMPOSITE' }, + { algo: 'CRC32', type: 'FULL_OBJECT' }, + { algo: 'CRC32C', type: 'FULL_OBJECT' }, + { algo: 'SHA1', type: 'COMPOSITE' }, + { algo: 'SHA256', type: 'COMPOSITE' }, + { algo: 'CRC64NVME', type: 'FULL_OBJECT' }, + ].forEach(({ algo, type }) => { + it(`should complete an MPU with a copied part (${algo}/${type})`, async () => { + const key = `cmp-${algo}-${type}`; + const uploadId = await createMpu(key, { ChecksumAlgorithm: algo, ChecksumType: type }); + const copy = await copyPart(key, uploadId); + const partChecksum = copy.CopyPartResult[field(algo)]; + assert(partChecksum, `expected ${field(algo)} in CopyPartResult`); + const complete = await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + MultipartUpload: { + Parts: [ + { + PartNumber: 1, + ETag: copy.CopyPartResult.ETag, + [field(algo)]: partChecksum, + }, + ], + }, + }), + ); + assert.strictEqual(complete.ChecksumType, type); + assert(complete[field(algo)], `expected final ${field(algo)} on CompleteMPU response`); + if (type === 'COMPOSITE') { + assert( + complete[field(algo)].endsWith('-1'), + `expected -1 suffix for 1-part COMPOSITE, got ${complete[field(algo)]}`, + ); + } + }); + }); + + it('should complete a default MPU with a copied part', async () => { + const key = 'cmp-default'; + const uploadId = await createMpu(key); + const copy = await copyPart(key, uploadId); + const complete = await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + MultipartUpload: { Parts: [{ PartNumber: 1, ETag: copy.CopyPartResult.ETag }] }, + }), + ); + assert(complete.ChecksumCRC64NVME, 'expected default-MPU final ChecksumCRC64NVME'); + assert.strictEqual(complete.ChecksumType, 'FULL_OBJECT'); + }); + + it('should surface the copied part checksum in ListParts for an explicit MPU', async () => { + const key = 'lp-explicit'; + const uploadId = await createMpu(key, { ChecksumAlgorithm: 'CRC32' }); + await copyPart(key, uploadId); + const list = await s3.send(new ListPartsCommand({ Bucket: bucket, Key: key, UploadId: uploadId })); + assert.strictEqual(list.Parts[0].ChecksumCRC32, await digest('CRC32', sourceBody)); + }); + + it('should not surface a checksum in ListParts for a default MPU', async () => { + const key = 'lp-default'; + const uploadId = await createMpu(key); + await copyPart(key, uploadId); + const list = await s3.send(new ListPartsCommand({ Bucket: bucket, Key: key, UploadId: uploadId })); + allFields.forEach(f => + assert.strictEqual(list.Parts[0][f], undefined, `default MPU ListParts should not include ${f}`), + ); + }); + + it('should complete a multi-part COMPOSITE MPU of copied parts with the -N suffix', async () => { + const key = 'cmp-multi-composite'; + const uploadId = await createMpu(key, { ChecksumAlgorithm: 'CRC32', ChecksumType: 'COMPOSITE' }); + const p1 = await copyPart(key, uploadId, { PartNumber: 1, CopySource: `${bucket}/${bigSourceKey}` }); + const p2 = await copyPart(key, uploadId, { PartNumber: 2 }); + const complete = await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + MultipartUpload: { + Parts: [ + { + PartNumber: 1, + ETag: p1.CopyPartResult.ETag, + ChecksumCRC32: p1.CopyPartResult.ChecksumCRC32, + }, + { + PartNumber: 2, + ETag: p2.CopyPartResult.ETag, + ChecksumCRC32: p2.CopyPartResult.ChecksumCRC32, + }, + ], + }, + }), + ); + assert.strictEqual(complete.ChecksumType, 'COMPOSITE'); + assert( + complete.ChecksumCRC32.endsWith('-2'), + `expected -2 suffix for a 2-part COMPOSITE, got ${complete.ChecksumCRC32}`, + ); + }); + + it('should complete a multi-part FULL_OBJECT MPU of copied parts with the linear digest', async () => { + const key = 'cmp-multi-full'; + const uploadId = await createMpu(key, { ChecksumAlgorithm: 'CRC32', ChecksumType: 'FULL_OBJECT' }); + const p1 = await copyPart(key, uploadId, { PartNumber: 1, CopySource: `${bucket}/${bigSourceKey}` }); + const p2 = await copyPart(key, uploadId, { PartNumber: 2 }); + const complete = await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + MultipartUpload: { + Parts: [ + { + PartNumber: 1, + ETag: p1.CopyPartResult.ETag, + ChecksumCRC32: p1.CopyPartResult.ChecksumCRC32, + }, + { + PartNumber: 2, + ETag: p2.CopyPartResult.ETag, + ChecksumCRC32: p2.CopyPartResult.ChecksumCRC32, + }, + ], + }, + }), + ); + assert.strictEqual(complete.ChecksumType, 'FULL_OBJECT'); + assert.strictEqual(complete.ChecksumCRC32, await digest('CRC32', Buffer.concat([bigBody, sourceBody]))); + }); + + it('should reuse a matching source checksum without recomputing', async () => { + const srcKey = 'copypart-checksum-reuse-src'; + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: srcKey, + Body: sourceBody, + ChecksumAlgorithm: 'CRC32', + }), + ); + const key = 'cpr-reuse'; + const uploadId = await createMpu(key, { ChecksumAlgorithm: 'CRC32' }); + const res = await copyPart(key, uploadId, { CopySource: `${bucket}/${srcKey}` }); + assert.strictEqual(res.CopyPartResult.ChecksumCRC32, await digest('CRC32', sourceBody)); + }); + + it('should checksum a copied part whose source is a multi-part object', async () => { + const srcKey = 'copypart-checksum-mpu-source'; + const srcUploadId = await createMpu(srcKey); + const a = bigBody; // part 1 must be >= 5 MiB to complete the source MPU + const b = Buffer.from('multipart-source-part-B', 'utf8'); + const up1 = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: srcKey, + UploadId: srcUploadId, + PartNumber: 1, + Body: a, + }), + ); + const up2 = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: srcKey, + UploadId: srcUploadId, + PartNumber: 2, + Body: b, + }), + ); + await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: srcKey, + UploadId: srcUploadId, + MultipartUpload: { + Parts: [ + { PartNumber: 1, ETag: up1.ETag }, + { PartNumber: 2, ETag: up2.ETag }, + ], + }, + }), + ); + const key = 'cpr-mp-source'; + const uploadId = await createMpu(key, { ChecksumAlgorithm: 'CRC32' }); + const res = await copyPart(key, uploadId, { CopySource: `${bucket}/${srcKey}` }); + assert.strictEqual(res.CopyPartResult.ChecksumCRC32, await digest('CRC32', Buffer.concat([a, b]))); + }); + + it('should complete an MPU mixing an uploaded part and a copied part', async () => { + const key = 'cmp-mixed'; + const uploadId = await createMpu(key, { ChecksumAlgorithm: 'CRC32', ChecksumType: 'FULL_OBJECT' }); + const partBody = bigBody; // uploaded part 1 must be >= 5 MiB + const up1 = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + PartNumber: 1, + Body: partBody, + ChecksumAlgorithm: 'CRC32', + }), + ); + const cp2 = await copyPart(key, uploadId, { PartNumber: 2 }); + const complete = await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + MultipartUpload: { + Parts: [ + { PartNumber: 1, ETag: up1.ETag, ChecksumCRC32: up1.ChecksumCRC32 }, + { + PartNumber: 2, + ETag: cp2.CopyPartResult.ETag, + ChecksumCRC32: cp2.CopyPartResult.ChecksumCRC32, + }, + ], + }, + }), + ); + assert.strictEqual(complete.ChecksumType, 'FULL_OBJECT'); + assert.strictEqual(complete.ChecksumCRC32, await digest('CRC32', Buffer.concat([partBody, sourceBody]))); + }); + + it('should reject CompleteMPU on a COMPOSITE MPU when the copied part checksum is omitted', async () => { + const key = 'cmp-omit-cksum'; + const uploadId = await createMpu(key, { ChecksumAlgorithm: 'CRC32', ChecksumType: 'COMPOSITE' }); + const copy = await copyPart(key, uploadId); + await assert.rejects( + s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + MultipartUpload: { Parts: [{ PartNumber: 1, ETag: copy.CopyPartResult.ETag }] }, + }), + ), + err => { + assert.strictEqual(err.name, 'InvalidRequest'); + return true; + }, + ); + }); + + it('should reject CompleteMPU when the copied part checksum is wrong', async () => { + const key = 'cmp-wrong-cksum'; + const uploadId = await createMpu(key, { ChecksumAlgorithm: 'CRC32', ChecksumType: 'COMPOSITE' }); + const copy = await copyPart(key, uploadId); + await assert.rejects( + s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ PartNumber: 1, ETag: copy.CopyPartResult.ETag, ChecksumCRC32: 'AAAAAA==' }], + }, + }), + ), + err => { + assert.strictEqual(err.name, 'InvalidPart'); + return true; + }, + ); + }); + })); diff --git a/tests/functional/aws-node-sdk/test/object/mpuUploadPartChecksum.js b/tests/functional/aws-node-sdk/test/object/mpuUploadPartChecksum.js index 1bcf410912..fffa81bfab 100644 --- a/tests/functional/aws-node-sdk/test/object/mpuUploadPartChecksum.js +++ b/tests/functional/aws-node-sdk/test/object/mpuUploadPartChecksum.js @@ -43,13 +43,14 @@ before(async () => { } }); -async function assertPartChecksumStored(s3, uploadId, partNumber, - checksumHeader, expectedChecksum) { - const listRes = await s3.send(new ListPartsCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - })); +async function assertPartChecksumStored(s3, uploadId, partNumber, checksumHeader, expectedChecksum) { + const listRes = await s3.send( + new ListPartsCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ); const found = listRes.Parts.find(part => part.PartNumber === partNumber); assert(found, `Expected part ${partNumber} in ListParts response`); assert.strictEqual(found[checksumHeader], expectedChecksum); @@ -81,38 +82,60 @@ describe('UploadPart checksum validation', () => let uploadId; before(async () => { - const res = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, Key: key, - ChecksumAlgorithm: mpuAlgo, - })); + const res = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ChecksumAlgorithm: mpuAlgo, + }), + ); uploadId = res.UploadId; }); after(async () => { - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, Key: key, UploadId: uploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ); }); it(`should accept ${mpuAlgo} with correct digest`, async () => { const partNumber = 1; - const res = await s3.send(new UploadPartCommand({ - Bucket: bucket, Key: key, UploadId: uploadId, - PartNumber: partNumber, Body: partBody, - [checksumField[mpuAlgo]]: correctDigest[mpuAlgo], - })); + const res = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + PartNumber: partNumber, + Body: partBody, + [checksumField[mpuAlgo]]: correctDigest[mpuAlgo], + }), + ); assert.strictEqual(res[checksumField[mpuAlgo]], correctDigest[mpuAlgo]); - await assertPartChecksumStored(s3, uploadId, partNumber, - checksumField[mpuAlgo], correctDigest[mpuAlgo]); + await assertPartChecksumStored( + s3, + uploadId, + partNumber, + checksumField[mpuAlgo], + correctDigest[mpuAlgo], + ); }); it(`should reject ${mpuAlgo} with wrong digest (BadDigest)`, async () => { await assert.rejects( - s3.send(new UploadPartCommand({ - Bucket: bucket, Key: key, UploadId: uploadId, - PartNumber: 2, Body: partBody, - [checksumField[mpuAlgo]]: wrongDigest[mpuAlgo], - })), + s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + PartNumber: 2, + Body: partBody, + [checksumField[mpuAlgo]]: wrongDigest[mpuAlgo], + }), + ), { name: 'BadDigest' }, ); }); @@ -121,18 +144,40 @@ describe('UploadPart checksum validation', () => // so "no checksum header" cannot be tested via the SDK for // non-default MPUs (it would be rejected as a mismatch). - allAlgos.filter(a => a !== mpuAlgo).forEach((otherAlgo, idx) => { - it(`should reject ${otherAlgo} when MPU is ${mpuAlgo} (InvalidRequest)`, async () => { - await assert.rejects( - s3.send(new UploadPartCommand({ - Bucket: bucket, Key: key, UploadId: uploadId, - PartNumber: 3 + idx, Body: partBody, - [checksumField[otherAlgo]]: correctDigest[otherAlgo], - })), - { name: 'InvalidRequest' }, - ); + allAlgos + .filter(a => a !== mpuAlgo) + .forEach((otherAlgo, idx) => { + it(`should reject ${otherAlgo} when MPU is ${mpuAlgo} (InvalidRequest)`, async () => { + await assert.rejects( + s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + PartNumber: 3 + idx, + Body: partBody, + [checksumField[otherAlgo]]: correctDigest[otherAlgo], + }), + ), + err => { + assert.strictEqual( + err.name, + 'InvalidRequest', + `expected InvalidRequest, got ${err.name}: ${err.message}`, + ); + // AWS names the expected (MPU) and actual (sent) algorithms. + assert.match( + err.message, + new RegExp( + `expected checksum Type: ${mpuAlgo.toLowerCase()}, ` + + `actual checksum Type: ${otherAlgo.toLowerCase()}`, + ), + ); + return true; + }, + ); + }); }); - }); }); }); @@ -141,47 +186,174 @@ describe('UploadPart checksum validation', () => let uploadId; before(async () => { - const res = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, Key: key, - })); + const res = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }), + ); uploadId = res.UploadId; }); after(async () => { - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, Key: key, UploadId: uploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ); }); allAlgos.forEach((algo, idx) => { it(`should accept ${algo} with correct digest`, async () => { - const res = await s3.send(new UploadPartCommand({ - Bucket: bucket, Key: key, UploadId: uploadId, - PartNumber: 2 * idx + 1, Body: partBody, - [checksumField[algo]]: correctDigest[algo], - })); + const res = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + PartNumber: 2 * idx + 1, + Body: partBody, + [checksumField[algo]]: correctDigest[algo], + }), + ); assert.strictEqual(res[checksumField[algo]], correctDigest[algo]); }); it(`should reject ${algo} with wrong digest (BadDigest)`, async () => { await assert.rejects( - s3.send(new UploadPartCommand({ - Bucket: bucket, Key: key, UploadId: uploadId, - PartNumber: 2 * idx + 2, Body: partBody, - [checksumField[algo]]: wrongDigest[algo], - })), + s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + PartNumber: 2 * idx + 2, + Body: partBody, + [checksumField[algo]]: wrongDigest[algo], + }), + ), { name: 'BadDigest' }, ); }); }); - it('should accept part with no checksum header', async () => { - const res = await s3.send(new UploadPartCommand({ - Bucket: bucket, Key: key, UploadId: uploadId, - PartNumber: 2 * allAlgos.length + 1, Body: partBody, - })); + it('should return no per-part checksum when none is sent', async () => { + // WHEN_REQUIRED so the SDK does not auto-attach a crc32: the + // part is genuinely uploaded with no checksum. + const noCksumS3 = new BucketUtility('default', { + ...sigCfg, + requestChecksumCalculation: 'WHEN_REQUIRED', + responseChecksumValidation: 'WHEN_REQUIRED', + }).s3; + const res = await noCksumS3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + PartNumber: 2 * allAlgos.length + 1, + Body: partBody, + }), + ); assert(res.ETag); + const present = [ + 'ChecksumCRC32', + 'ChecksumCRC32C', + 'ChecksumCRC64NVME', + 'ChecksumSHA1', + 'ChecksumSHA256', + ].filter(f => res[f] !== undefined); + assert.deepStrictEqual( + present, + [], + `default MPU UploadPart should return no checksum, got: ${present.join(', ')}`, + ); }); }); - }) -); + + describe('per-part checksum requirement by checksum type', () => { + // WHEN_REQUIRED so the SDK does not auto-attach a checksum, letting + // us upload a genuinely checksum-less part. + let noCksumS3; + const openUploads = []; + + before(() => { + noCksumS3 = new BucketUtility('default', { + ...sigCfg, + requestChecksumCalculation: 'WHEN_REQUIRED', + responseChecksumValidation: 'WHEN_REQUIRED', + }).s3; + }); + + after(async () => { + await Promise.all( + openUploads.map(uploadId => + noCksumS3 + .send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ) + .catch(() => undefined), + ), + ); + }); + + async function createMpu(algo, type) { + const res = await noCksumS3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ChecksumAlgorithm: algo, + ChecksumType: type, + }), + ); + openUploads.push(res.UploadId); + return res.UploadId; + } + + ['CRC32', 'CRC32C', 'SHA1', 'SHA256'].forEach(algo => { + it(`should reject UploadPart with no checksum on a ${algo}/COMPOSITE MPU`, async () => { + const uploadId = await createMpu(algo, 'COMPOSITE'); + await assert.rejects( + noCksumS3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + PartNumber: 1, + Body: partBody, + }), + ), + err => { + assert.strictEqual( + err.name, + 'InvalidRequest', + `expected InvalidRequest, got ${err.name}: ${err.message}`, + ); + assert.match(err.message, new RegExp(`expected checksum Type: ${algo.toLowerCase()}`)); + return true; + }, + ); + }); + }); + + ['CRC32', 'CRC32C', 'CRC64NVME'].forEach(algo => { + it(`should accept UploadPart with no checksum on a ${algo}/FULL_OBJECT MPU`, async () => { + const uploadId = await createMpu(algo, 'FULL_OBJECT'); + const res = await noCksumS3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + PartNumber: 1, + Body: partBody, + }), + ); + assert(res.ETag); + assert(res[`Checksum${algo}`], `expected Checksum${algo} echoed, got: ${JSON.stringify(res)}`); + }); + }); + }); + })); diff --git a/tests/functional/aws-node-sdk/test/object/objectCopy.js b/tests/functional/aws-node-sdk/test/object/objectCopy.js index 38a73c10d2..e7b4f566a9 100644 --- a/tests/functional/aws-node-sdk/test/object/objectCopy.js +++ b/tests/functional/aws-node-sdk/test/object/objectCopy.js @@ -2135,5 +2135,97 @@ describe('Object Copy checksum behavior', () => { assert.strictEqual(err.message, expected); } }); + + // The AWS-correct way to (re)compute an object's checksum in place is a + // self-copy with MetadataDirective=REPLACE (the metadata change is what + // makes the self-copy legal); the checksum is then recomputed in place. + const selfCopyBody = 'in-place-checksum-change-body'; + checksumFixtures.forEach(({ algo, header, key }) => { + const sourceHeader = header === 'CRC32' ? 'SHA256' : 'CRC32'; + it(`should change an object's checksum to ${algo} via a self-copy (REPLACE directive)`, async () => { + await s3.send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: selfCopyBody, + ChecksumAlgorithm: sourceHeader, + }), + ); + const expectedDigest = await Promise.resolve(algorithms[algo].digest(Buffer.from(selfCopyBody))); + + // Self-copy with MetadataDirective=REPLACE, changing the checksum + // algorithm: the metadata change makes it legal and the checksum + // is recomputed in place. + const copyRes = await s3.send( + new CopyObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + MetadataDirective: 'REPLACE', + ChecksumAlgorithm: header, + }), + ); + assert.strictEqual(copyRes.CopyObjectResult[key], expectedDigest); + assert.strictEqual(copyRes.CopyObjectResult.ChecksumType, 'FULL_OBJECT'); + + const headRes = await s3.send( + new HeadObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + ChecksumMode: 'ENABLED', + }), + ); + assert.strictEqual(headRes[key], expectedDigest); + assert.strictEqual(headRes.ChecksumType, 'FULL_OBJECT'); + }); + }); + + // A COPY-directive self-copy is illegal even when a checksum algorithm is + // requested: the checksum header is not a "change" (matches AWS). + it('should reject a self-copy that only requests a checksum (COPY directive)', async () => { + await s3.send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: selfCopyBody, + }), + ); + try { + await s3.send( + new CopyObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + ChecksumAlgorithm: 'CRC32', + }), + ); + throw new Error('Expected 400 InvalidRequest'); + } catch (err) { + checkError(err, 'InvalidRequest', 400); + } + }); + + // A self-copy with no checksum algorithm and no other change is rejected. + it('should reject a no-change self-copy (COPY directive)', async () => { + await s3.send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: selfCopyBody, + }), + ); + try { + await s3.send( + new CopyObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + }), + ); + throw new Error('Expected 400 InvalidRequest'); + } catch (err) { + checkError(err, 'InvalidRequest', 400); + } + }); }); }); diff --git a/tests/functional/raw-node/test/checksumPutObjectUploadPart.js b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js index 855eb3ebe8..165d9406f1 100644 --- a/tests/functional/raw-node/test/checksumPutObjectUploadPart.js +++ b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js @@ -32,26 +32,24 @@ const algos = [ // inject a wrong value for negative tests. function buildTrailerBody(body, algoName, digest) { const hexLen = body.length.toString(16); - const actualDigest = digest !== undefined - ? digest - : crypto.createHash(algoName).update(body).digest('base64'); + const actualDigest = digest !== undefined ? digest : crypto.createHash(algoName).update(body).digest('base64'); return `${hexLen}\r\n${body.toString()}\r\n0\r\nx-amz-checksum-${algoName}:${actualDigest}\n\r\n\r\n\r\n`; } function doPutRequest(url, headers, body, callback) { - const req = new HttpRequestAuthV4( - url, - Object.assign({ method: 'PUT', headers }, authCredentials), - res => { - let data = ''; - res.on('data', chunk => { data += chunk; }); - res.on('end', () => callback(null, { + const req = new HttpRequestAuthV4(url, Object.assign({ method: 'PUT', headers }, authCredentials), res => { + let data = ''; + res.on('data', chunk => { + data += chunk; + }); + res.on('end', () => + callback(null, { statusCode: res.statusCode, body: data, headers: res.headers, - })); - } - ); + }), + ); + }); req.on('error', callback); req.write(body); req.end(); @@ -122,30 +120,31 @@ const testContent2Sha256B64 = crypto.createHash('sha256').update(testContent2).d const largeBody = Buffer.alloc(10 * 1024 * 1024, 'a'); const largeBodySha256B64 = crypto.createHash('sha256').update(largeBody).digest('base64'); - // Assert that the response has the given HTTP status code and (optionally) // that the body contains the expected error code string. // Returns a (err, res, done) callback suitable for use with doPutRequest. function assertStatus(expectedStatus, expectedCode, expectedMessage) { return (err, res, done) => { assert.ifError(err); - assert.strictEqual(res.statusCode, expectedStatus, - `expected ${expectedStatus}, got ${res.statusCode}: ${res.body}`); + assert.strictEqual( + res.statusCode, + expectedStatus, + `expected ${expectedStatus}, got ${res.statusCode}: ${res.body}`, + ); if (expectedCode) { - assert(res.body.includes(expectedCode), - `expected "${expectedCode}" in body: "${res.body}"`); + assert(res.body.includes(expectedCode), `expected "${expectedCode}" in body: "${res.body}"`); } if (expectedMessage) { - assert(res.body.includes(expectedMessage), - `expected "${expectedMessage}" in body: "${res.body}"`); + assert(res.body.includes(expectedMessage), `expected "${expectedMessage}" in body: "${res.body}"`); } done(); }; } -const msgMalformedTrailer = 'The request contained trailing data that was not well-formed' + - ' or did not conform to our published schema.'; -const msgSdkMissingTrailer = 'x-amz-sdk-checksum-algorithm specified, but no corresponding' + +const msgMalformedTrailer = + 'The request contained trailing data that was not well-formed' + ' or did not conform to our published schema.'; +const msgSdkMissingTrailer = + 'x-amz-sdk-checksum-algorithm specified, but no corresponding' + ' x-amz-checksum-* or x-amz-trailer headers were found.'; // Module-level variables for computed crc64nvme checksums (filled in before hook) @@ -154,7 +153,7 @@ let crc64nvmeOfTrailerContent; // Create the common protocol-scenario tests for a given URL factory. // urlFn() is called lazily at test runtime so that uploadId is available. -function makeScenarioTests(urlFn) { +function makeScenarioTests(urlFn, { expectsImplicitChecksum = true } = {}) { before(async () => { if (!crc64nvmeOfTestContent2) { crc64nvmeOfTestContent2 = await algorithms.crc64nvme.digest(testContent2); @@ -164,415 +163,569 @@ function makeScenarioTests(urlFn) { } }); - itSkipIfAWS( - 'should return 200 for signed sha256 in x-amz-content-sha256, no x-amz-checksum header', - done => { - doPutRequest(urlFn(), { + // When no client checksum is sent, PutObject echoes the server-computed + // default crc64nvme, but a default-MPU UploadPart does not (matching AWS). + function assertImplicitChecksum(res, expected) { + if (expectsImplicitChecksum) { + assert.strictEqual( + res.headers['x-amz-checksum-crc64nvme'], + expected, + `expected x-amz-checksum-crc64nvme: ${expected}`, + ); + } else { + assert.strictEqual( + res.headers['x-amz-checksum-crc64nvme'], + undefined, + 'default-MPU UploadPart should not echo an implicit checksum', + ); + } + } + + itSkipIfAWS('should return 200 for signed sha256 in x-amz-content-sha256, no x-amz-checksum header', done => { + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': testContent2Sha256Hex, 'content-length': testContent2.length, - }, testContent2, (err, res) => { + }, + testContent2, + (err, res) => { assertStatus(200)(err, res, () => { - assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], crc64nvmeOfTestContent2, - `expected x-amz-checksum-crc64nvme: ${crc64nvmeOfTestContent2}`); + assertImplicitChecksum(res, crc64nvmeOfTestContent2); done(); }); - }); - }); + }, + ); + }); - itSkipIfAWS( - 'should return 200 for correct sha256 checksum with x-amz-sdk-checksum-algorithm', - done => { - doPutRequest(urlFn(), { + itSkipIfAWS('should return 200 for correct sha256 checksum with x-amz-sdk-checksum-algorithm', done => { + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': testContent2Sha256Hex, 'x-amz-sdk-checksum-algorithm': 'SHA256', 'x-amz-checksum-sha256': testContent2Sha256B64, 'content-length': testContent2.length, - }, testContent2, (err, res) => { + }, + testContent2, + (err, res) => { assertStatus(200)(err, res, () => { - assert.strictEqual(res.headers['x-amz-checksum-sha256'], testContent2Sha256B64, - `expected x-amz-checksum-sha256: ${testContent2Sha256B64}`); + assert.strictEqual( + res.headers['x-amz-checksum-sha256'], + testContent2Sha256B64, + `expected x-amz-checksum-sha256: ${testContent2Sha256B64}`, + ); done(); }); - }); - }); + }, + ); + }); - itSkipIfAWS( - 'should return 400 BadDigest for wrong sha256 checksum with x-amz-sdk-checksum-algorithm', - done => { - doPutRequest(urlFn(), { + itSkipIfAWS('should return 400 BadDigest for wrong sha256 checksum with x-amz-sdk-checksum-algorithm', done => { + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': testContent2Sha256Hex, 'x-amz-sdk-checksum-algorithm': 'SHA256', 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', 'content-length': testContent2.length, - }, testContent2, (err, res) => assertStatus(400, 'BadDigest', - 'The SHA256 you specified did not match the calculated checksum.')(err, res, done)); - }); + }, + testContent2, + (err, res) => + assertStatus(400, 'BadDigest', 'The SHA256 you specified did not match the calculated checksum.')( + err, + res, + done, + ), + ); + }); - itSkipIfAWS( - 'should return 200 for UNSIGNED-PAYLOAD with correct sha256 checksum', - done => { - doPutRequest(urlFn(), { + itSkipIfAWS('should return 200 for UNSIGNED-PAYLOAD with correct sha256 checksum', done => { + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', 'x-amz-sdk-checksum-algorithm': 'SHA256', 'x-amz-checksum-sha256': testContent2Sha256B64, 'content-length': testContent2.length, - }, testContent2, (err, res) => { + }, + testContent2, + (err, res) => { assertStatus(200)(err, res, () => { - assert.strictEqual(res.headers['x-amz-checksum-sha256'], testContent2Sha256B64, - `expected x-amz-checksum-sha256: ${testContent2Sha256B64}`); + assert.strictEqual( + res.headers['x-amz-checksum-sha256'], + testContent2Sha256B64, + `expected x-amz-checksum-sha256: ${testContent2Sha256B64}`, + ); done(); }); - }); - }); + }, + ); + }); - itSkipIfAWS( - 'should return 400 IncompleteBody for TRAILER with empty body', - done => { - doPutRequest(urlFn(), { + itSkipIfAWS('should return 400 IncompleteBody for TRAILER with empty body', done => { + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', 'x-amz-trailer': 'x-amz-checksum-sha256', 'x-amz-decoded-content-length': trailerContent.length, 'content-length': 0, - }, Buffer.alloc(0), (err, res) => assertStatus(400, 'IncompleteBody', - 'The request body terminated unexpectedly')(err, res, done)); - }); + }, + Buffer.alloc(0), + (err, res) => + assertStatus(400, 'IncompleteBody', 'The request body terminated unexpectedly')(err, res, done), + ); + }); - itSkipIfAWS( - 'should return 200 for TRAILER with correct sha256 checksum', - done => { - const body = buildTrailerBody(trailerContent, 'sha256'); - doPutRequest(urlFn(), { + itSkipIfAWS('should return 200 for TRAILER with correct sha256 checksum', done => { + const body = buildTrailerBody(trailerContent, 'sha256'); + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', 'x-amz-trailer': 'x-amz-checksum-sha256', 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => { + }, + body, + (err, res) => { assertStatus(200)(err, res, () => { - assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256, - `expected x-amz-checksum-sha256: ${trailerContentSha256}`); + assert.strictEqual( + res.headers['x-amz-checksum-sha256'], + trailerContentSha256, + `expected x-amz-checksum-sha256: ${trailerContentSha256}`, + ); done(); }); - }); - }); + }, + ); + }); - itSkipIfAWS( - 'should return 500 InternalError for wrong x-amz-decoded-content-length', - done => { - // Two chunks of 16 bytes each with a valid crc64nvme trailer. - const body = - '10\r\n0123456789abcdef\r\n' + - '10\r\n0123456789abcdef\r\n' + - '0\r\nx-amz-checksum-crc64nvme:skQv82y5rgE=\r\n\r\n\r\n'; - doPutRequest(urlFn(), { + itSkipIfAWS('should return 500 InternalError for wrong x-amz-decoded-content-length', done => { + // Two chunks of 16 bytes each with a valid crc64nvme trailer. + const body = + '10\r\n0123456789abcdef\r\n' + + '10\r\n0123456789abcdef\r\n' + + '0\r\nx-amz-checksum-crc64nvme:skQv82y5rgE=\r\n\r\n\r\n'; + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', 'x-amz-trailer': 'x-amz-checksum-crc64nvme', 'x-amz-decoded-content-length': 7, // wrong: actual content is 32 bytes 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(500, 'InternalError', - 'We encountered an internal error. Please try again.')(err, res, done)); - }); + }, + body, + (err, res) => + assertStatus(500, 'InternalError', 'We encountered an internal error. Please try again.')( + err, + res, + done, + ), + ); + }); itSkipIfAWS( 'should return 400 MalformedTrailerError when x-amz-trailer says sha1 but body trailer has sha256', done => { // Header announces sha1 but the actual trailer line carries sha256. - const body = - `f\r\ntrailer content\r\n0\r\nx-amz-checksum-sha256:${trailerContentSha256}\n\r\n\r\n\r\n`; - doPutRequest(urlFn(), { - 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', - 'x-amz-trailer': 'x-amz-checksum-sha1', - 'x-amz-decoded-content-length': trailerContent.length, - 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(400, 'MalformedTrailerError', - msgMalformedTrailer)(err, res, done)); - }); + const body = `f\r\ntrailer content\r\n0\r\nx-amz-checksum-sha256:${trailerContentSha256}\n\r\n\r\n\r\n`; + doPutRequest( + urlFn(), + { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha1', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, + body, + (err, res) => assertStatus(400, 'MalformedTrailerError', msgMalformedTrailer)(err, res, done), + ); + }, + ); - itSkipIfAWS( - 'should return 400 BadDigest for TRAILER with wrong sha256 checksum', - done => { - const wrongSha256 = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='; - const body = - `f\r\ntrailer content\r\n0\r\nx-amz-checksum-sha256:${wrongSha256}\n\r\n\r\n\r\n`; - doPutRequest(urlFn(), { + itSkipIfAWS('should return 400 BadDigest for TRAILER with wrong sha256 checksum', done => { + const wrongSha256 = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='; + const body = `f\r\ntrailer content\r\n0\r\nx-amz-checksum-sha256:${wrongSha256}\n\r\n\r\n\r\n`; + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', 'x-amz-trailer': 'x-amz-checksum-sha256', 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(400, 'BadDigest', - 'The SHA256 you specified did not match the calculated checksum.')(err, res, done)); - }); + }, + body, + (err, res) => + assertStatus(400, 'BadDigest', 'The SHA256 you specified did not match the calculated checksum.')( + err, + res, + done, + ), + ); + }); - itSkipIfAWS( - 'should return 400 InvalidRequest for x-amz-trailer + x-amz-checksum-crc32 header', - done => { - const body = buildTrailerBody(trailerContent, 'sha256'); - doPutRequest(urlFn(), { + itSkipIfAWS('should return 400 InvalidRequest for x-amz-trailer + x-amz-checksum-crc32 header', done => { + const body = buildTrailerBody(trailerContent, 'sha256'); + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', 'x-amz-trailer': 'x-amz-checksum-sha256', 'x-amz-checksum-crc32': 'H+Yzmw==', // crc32("trailer content") 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(400, 'InvalidRequest', - 'Expecting a single x-amz-checksum- header')(err, res, done)); - }); + }, + body, + (err, res) => + assertStatus(400, 'InvalidRequest', 'Expecting a single x-amz-checksum- header')(err, res, done), + ); + }); - itSkipIfAWS( - 'should return 400 MalformedTrailerError when no x-amz-trailer header but body has trailer', - done => { - const body = buildTrailerBody(trailerContent, 'sha256'); - doPutRequest(urlFn(), { + itSkipIfAWS('should return 400 MalformedTrailerError when no x-amz-trailer header but body has trailer', done => { + const body = buildTrailerBody(trailerContent, 'sha256'); + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', // no x-amz-trailer header 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(400, 'MalformedTrailerError', - msgMalformedTrailer)(err, res, done)); - }); + }, + body, + (err, res) => assertStatus(400, 'MalformedTrailerError', msgMalformedTrailer)(err, res, done), + ); + }); - itSkipIfAWS( - 'should return 200 for TRAILER with explicit Content-Length', - done => { - const body = buildTrailerBody(trailerContent, 'sha256'); - doPutRequest(urlFn(), { + itSkipIfAWS('should return 200 for TRAILER with explicit Content-Length', done => { + const body = buildTrailerBody(trailerContent, 'sha256'); + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', 'x-amz-trailer': 'x-amz-checksum-sha256', 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => { + }, + body, + (err, res) => { assertStatus(200)(err, res, () => { - assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256, - `expected x-amz-checksum-sha256: ${trailerContentSha256}`); + assert.strictEqual( + res.headers['x-amz-checksum-sha256'], + trailerContentSha256, + `expected x-amz-checksum-sha256: ${trailerContentSha256}`, + ); done(); }); - }); - }); + }, + ); + }); - itSkipIfAWS( - 'should return 200 for TRAILER with matching x-amz-sdk-checksum-algorithm:SHA256', - done => { - const body = buildTrailerBody(trailerContent, 'sha256'); - doPutRequest(urlFn(), { + itSkipIfAWS('should return 200 for TRAILER with matching x-amz-sdk-checksum-algorithm:SHA256', done => { + const body = buildTrailerBody(trailerContent, 'sha256'); + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', 'x-amz-trailer': 'x-amz-checksum-sha256', 'x-amz-sdk-checksum-algorithm': 'SHA256', 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => { + }, + body, + (err, res) => { assertStatus(200)(err, res, () => { - assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256, - `expected x-amz-checksum-sha256: ${trailerContentSha256}`); + assert.strictEqual( + res.headers['x-amz-checksum-sha256'], + trailerContentSha256, + `expected x-amz-checksum-sha256: ${trailerContentSha256}`, + ); done(); }); - }); - }); + }, + ); + }); itSkipIfAWS( 'should return 400 InvalidRequest when x-amz-sdk-checksum-algorithm:SHA1 but x-amz-trailer is sha256', done => { const body = buildTrailerBody(trailerContent, 'sha256'); - doPutRequest(urlFn(), { - 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', - 'x-amz-trailer': 'x-amz-checksum-sha256', - 'x-amz-sdk-checksum-algorithm': 'SHA1', // mismatch - 'x-amz-decoded-content-length': trailerContent.length, - 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(400, 'InvalidRequest', - 'Value for x-amz-sdk-checksum-algorithm header is invalid.')(err, res, done)); - }); + doPutRequest( + urlFn(), + { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-sdk-checksum-algorithm': 'SHA1', // mismatch + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, + body, + (err, res) => + assertStatus(400, 'InvalidRequest', 'Value for x-amz-sdk-checksum-algorithm header is invalid.')( + err, + res, + done, + ), + ); + }, + ); - itSkipIfAWS( - 'should return 400 InvalidRequest for x-amz-trailer:x-amz-checksum-sha3 (unknown algo)', - done => { - const body = buildTrailerBody(trailerContent, 'sha256'); - doPutRequest(urlFn(), { + itSkipIfAWS('should return 400 InvalidRequest for x-amz-trailer:x-amz-checksum-sha3 (unknown algo)', done => { + const body = buildTrailerBody(trailerContent, 'sha256'); + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', 'x-amz-trailer': 'x-amz-checksum-sha3', 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(400, 'InvalidRequest', - 'The value specified in the x-amz-trailer header is not supported')(err, res, done)); - }); + }, + body, + (err, res) => + assertStatus(400, 'InvalidRequest', 'The value specified in the x-amz-trailer header is not supported')( + err, + res, + done, + ), + ); + }); - itSkipIfAWS( - 'should return 400 InvalidRequest for x-amz-trailer with non-checksum value', - done => { - const body = buildTrailerBody(trailerContent, 'sha256'); - doPutRequest(urlFn(), { + itSkipIfAWS('should return 400 InvalidRequest for x-amz-trailer with non-checksum value', done => { + const body = buildTrailerBody(trailerContent, 'sha256'); + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', 'x-amz-trailer': 'AAAAAAAAAAAAAAAAAAAAA', 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(400, 'InvalidRequest', - 'The value specified in the x-amz-trailer header is not supported')(err, res, done)); - }); + }, + body, + (err, res) => + assertStatus(400, 'InvalidRequest', 'The value specified in the x-amz-trailer header is not supported')( + err, + res, + done, + ), + ); + }); - itSkipIfAWS( - 'should return 400 InvalidRequest for trailer body with invalid base64 checksum value', - done => { - const body = 'f\r\ntrailer content\r\n0\r\nx-amz-checksum-sha256:BAD\n\r\n\r\n\r\n'; - doPutRequest(urlFn(), { + itSkipIfAWS('should return 400 InvalidRequest for trailer body with invalid base64 checksum value', done => { + const body = 'f\r\ntrailer content\r\n0\r\nx-amz-checksum-sha256:BAD\n\r\n\r\n\r\n'; + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', 'x-amz-trailer': 'x-amz-checksum-sha256', 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(400, 'InvalidRequest', - 'Value for x-amz-checksum-sha256 trailing header is invalid.')(err, res, done)); - }); + }, + body, + (err, res) => + assertStatus(400, 'InvalidRequest', 'Value for x-amz-checksum-sha256 trailing header is invalid.')( + err, + res, + done, + ), + ); + }); - itSkipIfAWS( - 'should return 400 InvalidRequest for x-amz-sdk-checksum-algorithm without x-amz-trailer', - done => { - const body = buildTrailerBody(trailerContent, 'sha256'); - doPutRequest(urlFn(), { + itSkipIfAWS('should return 400 InvalidRequest for x-amz-sdk-checksum-algorithm without x-amz-trailer', done => { + const body = buildTrailerBody(trailerContent, 'sha256'); + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', // no x-amz-trailer 'x-amz-sdk-checksum-algorithm': 'SHA256', 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(400, 'InvalidRequest', - msgSdkMissingTrailer)(err, res, done)); - }); + }, + body, + (err, res) => assertStatus(400, 'InvalidRequest', msgSdkMissingTrailer)(err, res, done), + ); + }); itSkipIfAWS( 'should return 400 MalformedTrailerError when x-amz-trailer header present but body has no trailer', done => { // Body ends with "0\r\n\r\n" — empty trailer section, no checksum line. const body = 'f\r\ntrailer content\r\n0\r\n\r\n'; - doPutRequest(urlFn(), { - 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', - 'x-amz-trailer': 'x-amz-checksum-sha256', - 'x-amz-decoded-content-length': trailerContent.length, - 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(400, 'MalformedTrailerError', - msgMalformedTrailer)(err, res, done)); - }); + doPutRequest( + urlFn(), + { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': trailerContent.length, + 'content-length': Buffer.byteLength(body), + }, + body, + (err, res) => assertStatus(400, 'MalformedTrailerError', msgMalformedTrailer)(err, res, done), + ); + }, + ); - itSkipIfAWS( - 'should return 200 when no x-amz-trailer and no body trailer', - done => { - // No x-amz-trailer header; body just has chunked data with no trailer. - const body = 'f\r\ntrailer content\r\n0\r\n\r\n'; - doPutRequest(urlFn(), { + itSkipIfAWS('should return 200 when no x-amz-trailer and no body trailer', done => { + // No x-amz-trailer header; body just has chunked data with no trailer. + const body = 'f\r\ntrailer content\r\n0\r\n\r\n'; + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', // no x-amz-trailer 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => { + }, + body, + (err, res) => { assertStatus(200)(err, res, () => { - assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], crc64nvmeOfTrailerContent, - `expected x-amz-checksum-crc64nvme: ${crc64nvmeOfTrailerContent}`); + assertImplicitChecksum(res, crc64nvmeOfTrailerContent); done(); }); - }); - }); + }, + ); + }); - itSkipIfAWS( - 'should return 200 and ignore data after final CRLF', - done => { - // No x-amz-trailer; after the terminating CRLF there is extra data. - // TrailingChecksumTransform discards everything after streamClosed=true. - const body = 'f\r\ntrailer content\r\n0\r\n\r\nRANDOM DATA IGNORED'; - doPutRequest(urlFn(), { + itSkipIfAWS('should return 200 and ignore data after final CRLF', done => { + // No x-amz-trailer; after the terminating CRLF there is extra data. + // TrailingChecksumTransform discards everything after streamClosed=true. + const body = 'f\r\ntrailer content\r\n0\r\n\r\nRANDOM DATA IGNORED'; + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', // no x-amz-trailer 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => { + }, + body, + (err, res) => { assertStatus(200)(err, res, () => { - assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], crc64nvmeOfTrailerContent, - `expected x-amz-checksum-crc64nvme: ${crc64nvmeOfTrailerContent}`); + assertImplicitChecksum(res, crc64nvmeOfTrailerContent); done(); }); - }); - }); + }, + ); + }); - itSkipIfAWS( - 'should return 200 for TRAILER with correct Content-MD5 header', - done => { - const body = buildTrailerBody(trailerContent, 'sha256'); - doPutRequest(urlFn(), { + itSkipIfAWS('should return 200 for TRAILER with correct Content-MD5 header', done => { + const body = buildTrailerBody(trailerContent, 'sha256'); + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', 'x-amz-trailer': 'x-amz-checksum-sha256', 'x-amz-sdk-checksum-algorithm': 'SHA256', 'content-md5': trailerContentMd5B64, 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => { + }, + body, + (err, res) => { assertStatus(200)(err, res, () => { - assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256, - `expected x-amz-checksum-sha256: ${trailerContentSha256}`); + assert.strictEqual( + res.headers['x-amz-checksum-sha256'], + trailerContentSha256, + `expected x-amz-checksum-sha256: ${trailerContentSha256}`, + ); done(); }); - }); - }); + }, + ); + }); - itSkipIfAWS( - 'should return 200 for trailer line with whitespace around name and value', - done => { - // TrailingChecksumTransform trims both name and value, so whitespace is accepted. - const body = - `f\r\ntrailer content\r\n0\r\n x-amz-checksum-sha256 : ${trailerContentSha256} \n\r\n\r\n\r\n`; - doPutRequest(urlFn(), { + itSkipIfAWS('should return 200 for trailer line with whitespace around name and value', done => { + // TrailingChecksumTransform trims both name and value, so whitespace is accepted. + // eslint-disable-next-line max-len -- prettier keeps this fixture template on one line (121 > 120) + const body = `f\r\ntrailer content\r\n0\r\n x-amz-checksum-sha256 : ${trailerContentSha256} \n\r\n\r\n\r\n`; + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', 'x-amz-trailer': 'x-amz-checksum-sha256', 'x-amz-decoded-content-length': trailerContent.length, 'content-length': Buffer.byteLength(body), - }, body, (err, res) => { + }, + body, + (err, res) => { assertStatus(200)(err, res, () => { - assert.strictEqual(res.headers['x-amz-checksum-sha256'], trailerContentSha256, - `expected x-amz-checksum-sha256: ${trailerContentSha256}`); + assert.strictEqual( + res.headers['x-amz-checksum-sha256'], + trailerContentSha256, + `expected x-amz-checksum-sha256: ${trailerContentSha256}`, + ); done(); }); - }); - }); + }, + ); + }); } // Large-body tests: 10MB of 'a's, verifying that streaming checksumming // accumulates data across chunks correctly. function makeLargeBodyTests(urlFn) { - itSkipIfAWS( - 'should return 200 for UNSIGNED-PAYLOAD with correct sha256 checksum on 10MB body', - done => { - doPutRequest(urlFn(), { + itSkipIfAWS('should return 200 for UNSIGNED-PAYLOAD with correct sha256 checksum on 10MB body', done => { + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', 'x-amz-checksum-sha256': largeBodySha256B64, 'content-length': largeBody.length, - }, largeBody, (err, res) => assertStatus(200)(err, res, done)); - }); + }, + largeBody, + (err, res) => assertStatus(200)(err, res, done), + ); + }); - itSkipIfAWS( - 'should return 400 BadDigest for UNSIGNED-PAYLOAD with wrong sha256 checksum on 10MB body', - done => { - doPutRequest(urlFn(), { + itSkipIfAWS('should return 400 BadDigest for UNSIGNED-PAYLOAD with wrong sha256 checksum on 10MB body', done => { + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', 'x-amz-checksum-sha256': 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', 'content-length': largeBody.length, - }, largeBody, (err, res) => assertStatus(400, 'BadDigest')(err, res, done)); - }); + }, + largeBody, + (err, res) => assertStatus(400, 'BadDigest')(err, res, done), + ); + }); itSkipIfAWS( 'should return 200 for STREAMING-UNSIGNED-PAYLOAD-TRAILER with correct sha256 checksum on 10MB body', done => { const body = buildTrailerBody(largeBody, 'sha256'); - doPutRequest(urlFn(), { - 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', - 'x-amz-trailer': 'x-amz-checksum-sha256', - 'x-amz-decoded-content-length': largeBody.length, - 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(200)(err, res, done)); - }); + doPutRequest( + urlFn(), + { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': largeBody.length, + 'content-length': Buffer.byteLength(body), + }, + body, + (err, res) => assertStatus(200)(err, res, done), + ); + }, + ); itSkipIfAWS( 'should return 400 BadDigest for STREAMING-UNSIGNED-PAYLOAD-TRAILER with wrong sha256 checksum on 10MB body', done => { const body = buildTrailerBody(largeBody, 'sha256', 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='); - doPutRequest(urlFn(), { - 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', - 'x-amz-trailer': 'x-amz-checksum-sha256', - 'x-amz-decoded-content-length': largeBody.length, - 'content-length': Buffer.byteLength(body), - }, body, (err, res) => assertStatus(400, 'BadDigest')(err, res, done)); - }); + doPutRequest( + urlFn(), + { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': largeBody.length, + 'content-length': Buffer.byteLength(body), + }, + body, + (err, res) => assertStatus(400, 'BadDigest')(err, res, done), + ); + }, + ); } describe('PutObject: bad checksum is rejected', () => { @@ -599,9 +752,10 @@ describe('PutObject: bad checksum is rejected', () => { `should return 400 BadDigest for ${protocol.name} with wrong x-amz-checksum-${algo.name}`, done => { const url = `http://localhost:8000/${bucket}/${objectKey}`; - doPutRequest(url, protocol.buildHeaders(algo), protocol.buildBody(algo), - (err, res) => assertBadDigest(err, res, done)); - } + doPutRequest(url, protocol.buildHeaders(algo), protocol.buildBody(algo), (err, res) => + assertBadDigest(err, res, done), + ); + }, ); } } @@ -611,43 +765,59 @@ describe('UploadPart: bad checksum is rejected', () => { let uploadId; before(done => { - async.series([ - next => makeS3Request({ method: 'PUT', authCredentials, bucket }, next), - next => makeS3Request({ - method: 'POST', - authCredentials, - bucket, - objectKey, - queryObj: { uploads: '' }, - }, (err, res) => { - if (err) { return next(err); } - const match = res.body.match(/([^<]+)<\/UploadId>/); - assert(match, `missing UploadId in response: ${res.body}`); - uploadId = match[1]; - return next(); - }), - ], err => { - assert.ifError(err); - done(); - }); + async.series( + [ + next => makeS3Request({ method: 'PUT', authCredentials, bucket }, next), + next => + makeS3Request( + { + method: 'POST', + authCredentials, + bucket, + objectKey, + queryObj: { uploads: '' }, + }, + (err, res) => { + if (err) { + return next(err); + } + const match = res.body.match(/([^<]+)<\/UploadId>/); + assert(match, `missing UploadId in response: ${res.body}`); + uploadId = match[1]; + return next(); + }, + ), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); after(done => { - async.series([ - next => makeS3Request({ - method: 'DELETE', - authCredentials, - bucket, - objectKey, - queryObj: { uploadId }, - }, next), - // Delete the object key first (defensive: clears any state left by a previous run). - next => makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => next()), - next => makeS3Request({ method: 'DELETE', authCredentials, bucket }, next), - ], err => { - assert.ifError(err); - done(); - }); + async.series( + [ + next => + makeS3Request( + { + method: 'DELETE', + authCredentials, + bucket, + objectKey, + queryObj: { uploadId }, + }, + next, + ), + // Delete the object key first (defensive: clears any state left by a previous run). + next => makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => next()), + next => makeS3Request({ method: 'DELETE', authCredentials, bucket }, next), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); for (const protocol of protocols) { @@ -655,11 +825,11 @@ describe('UploadPart: bad checksum is rejected', () => { itSkipIfAWS( `should return 400 BadDigest for ${protocol.name} with wrong x-amz-checksum-${algo.name}`, done => { - const url = `http://localhost:8000/${bucket}/${objectKey}` + - `?partNumber=1&uploadId=${uploadId}`; - doPutRequest(url, protocol.buildHeaders(algo), protocol.buildBody(algo), - (err, res) => assertBadDigest(err, res, done)); - } + const url = `http://localhost:8000/${bucket}/${objectKey}` + `?partNumber=1&uploadId=${uploadId}`; + doPutRequest(url, protocol.buildHeaders(algo), protocol.buildBody(algo), (err, res) => + assertBadDigest(err, res, done), + ); + }, ); } } @@ -684,55 +854,70 @@ describe('PutObject: trailer and checksum protocol scenarios', () => { }); makeScenarioTests(() => `http://localhost:8000/${bucket}/${objectKey}`); - }); describe('UploadPart: trailer and checksum protocol scenarios', () => { let uploadId2; before(done => { - async.series([ - next => makeS3Request({ method: 'PUT', authCredentials, bucket }, next), - next => makeS3Request({ - method: 'POST', - authCredentials, - bucket, - objectKey, - queryObj: { uploads: '' }, - }, (err, res) => { - if (err) { return next(err); } - const match = res.body.match(/([^<]+)<\/UploadId>/); - assert(match, `missing UploadId in response: ${res.body}`); - uploadId2 = match[1]; - return next(); - }), - ], err => { - assert.ifError(err); - done(); - }); + async.series( + [ + next => makeS3Request({ method: 'PUT', authCredentials, bucket }, next), + next => + makeS3Request( + { + method: 'POST', + authCredentials, + bucket, + objectKey, + queryObj: { uploads: '' }, + }, + (err, res) => { + if (err) { + return next(err); + } + const match = res.body.match(/([^<]+)<\/UploadId>/); + assert(match, `missing UploadId in response: ${res.body}`); + uploadId2 = match[1]; + return next(); + }, + ), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); after(done => { - async.series([ - next => makeS3Request({ - method: 'DELETE', - authCredentials, - bucket, - objectKey, - queryObj: { uploadId: uploadId2 }, - }, next), - // Delete the object key first (defensive: clears any state left by a previous run). - next => makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => next()), - next => makeS3Request({ method: 'DELETE', authCredentials, bucket }, next), - ], err => { - assert.ifError(err); - done(); - }); + async.series( + [ + next => + makeS3Request( + { + method: 'DELETE', + authCredentials, + bucket, + objectKey, + queryObj: { uploadId: uploadId2 }, + }, + next, + ), + // Delete the object key first (defensive: clears any state left by a previous run). + next => makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => next()), + next => makeS3Request({ method: 'DELETE', authCredentials, bucket }, next), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); - makeScenarioTests( - () => `http://localhost:8000/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId2}` - ); + makeScenarioTests(() => `http://localhost:8000/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId2}`, { + expectsImplicitChecksum: false, + }); }); describe('PutObject: checksum response header per algorithm', () => { @@ -744,8 +929,8 @@ describe('PutObject: checksum response header per algorithm', () => { before(async () => { await new Promise((resolve, reject) => - makeS3Request({ method: 'PUT', authCredentials, bucket }, - err => (err ? reject(err) : resolve()))); + makeS3Request({ method: 'PUT', authCredentials, bucket }, err => (err ? reject(err) : resolve())), + ); expectedCrc64nvme = await algorithms.crc64nvme.digest(body); }); @@ -767,43 +952,51 @@ describe('PutObject: checksum response header per algorithm', () => { ]; for (const algo of checksumAlgos) { - itSkipIfAWS( - `should return x-amz-checksum-${algo.name} response header with correct value`, - done => { - const expectedValue = algo.computeExpected(); - const headerName = `x-amz-checksum-${algo.name}`; - doPutRequest(url, { + itSkipIfAWS(`should return x-amz-checksum-${algo.name} response header with correct value`, done => { + const expectedValue = algo.computeExpected(); + const headerName = `x-amz-checksum-${algo.name}`; + doPutRequest( + url, + { 'x-amz-content-sha256': sha256Hex, [headerName]: expectedValue, 'content-length': body.length, - }, body, (err, res) => { + }, + body, + (err, res) => { assert.ifError(err); - assert.strictEqual(res.statusCode, 200, - `expected 200, got ${res.statusCode}: ${res.body}`); - assert.strictEqual(res.headers[headerName], expectedValue, - `expected ${headerName}: ${expectedValue}`); + assert.strictEqual(res.statusCode, 200, `expected 200, got ${res.statusCode}: ${res.body}`); + assert.strictEqual( + res.headers[headerName], + expectedValue, + `expected ${headerName}: ${expectedValue}`, + ); done(); - }); - } - ); + }, + ); + }); } - itSkipIfAWS( - 'should return x-amz-checksum-crc64nvme response header when no checksum header is sent', - done => { - doPutRequest(url, { + itSkipIfAWS('should return x-amz-checksum-crc64nvme response header when no checksum header is sent', done => { + doPutRequest( + url, + { 'x-amz-content-sha256': sha256Hex, 'content-length': body.length, - }, body, (err, res) => { + }, + body, + (err, res) => { assert.ifError(err); - assert.strictEqual(res.statusCode, 200, - `expected 200, got ${res.statusCode}: ${res.body}`); - assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], expectedCrc64nvme, - `expected x-amz-checksum-crc64nvme: ${expectedCrc64nvme}`); + assert.strictEqual(res.statusCode, 200, `expected 200, got ${res.statusCode}: ${res.body}`); + assert.strictEqual( + res.headers['x-amz-checksum-crc64nvme'], + expectedCrc64nvme, + `expected x-amz-checksum-crc64nvme: ${expectedCrc64nvme}`, + ); done(); - }); - } - ); + }, + ); + }); }); describe('PutObject: zero-byte object checksum handling', () => { @@ -834,11 +1027,10 @@ describe('PutObject: zero-byte object checksum handling', () => { }; before(done => { - makeS3Request({ method: 'PUT', authCredentials, bucket: zeroBucket }, - err => { - assert.ifError(err); - done(); - }); + makeS3Request({ method: 'PUT', authCredentials, bucket: zeroBucket }, err => { + assert.ifError(err); + done(); + }); }); after(done => { @@ -853,58 +1045,70 @@ describe('PutObject: zero-byte object checksum handling', () => { itSkipIfAWS( 'should return 200 with x-amz-checksum-crc64nvme of empty body when no checksum header is sent', done => { - doPutRequest(zeroUrl, { - 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', - 'content-length': 0, - }, emptyBody, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 200, - `expected 200, got ${res.statusCode}: ${res.body}`); - assert.strictEqual(res.headers['x-amz-checksum-crc64nvme'], - expectedEmptyChecksums.crc64nvme, - `expected x-amz-checksum-crc64nvme: ${expectedEmptyChecksums.crc64nvme}`); - done(); - }); - }); - - for (const algoName of ['crc32', 'crc32c', 'crc64nvme', 'sha1', 'sha256']) { - itSkipIfAWS( - `should return 200 with echoed ${algoName} response header for correct empty-body checksum`, - done => { - const expected = expectedEmptyChecksums[algoName]; - const headerName = `x-amz-checksum-${algoName}`; - doPutRequest(zeroUrl, { + doPutRequest( + zeroUrl, + { 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', - [headerName]: expected, 'content-length': 0, - }, emptyBody, (err, res) => { + }, + emptyBody, + (err, res) => { assert.ifError(err); - assert.strictEqual(res.statusCode, 200, - `expected 200, got ${res.statusCode}: ${res.body}`); - assert.strictEqual(res.headers[headerName], expected, - `expected ${headerName}: ${expected}`); + assert.strictEqual(res.statusCode, 200, `expected 200, got ${res.statusCode}: ${res.body}`); + assert.strictEqual( + res.headers['x-amz-checksum-crc64nvme'], + expectedEmptyChecksums.crc64nvme, + `expected x-amz-checksum-crc64nvme: ${expectedEmptyChecksums.crc64nvme}`, + ); done(); - }); - }); + }, + ); + }, + ); + for (const algoName of ['crc32', 'crc32c', 'crc64nvme', 'sha1', 'sha256']) { itSkipIfAWS( - `should return 400 BadDigest for wrong empty-body ${algoName} checksum`, + `should return 200 with echoed ${algoName} response header for correct empty-body checksum`, done => { - const wrong = wrongEmptyDigests[algoName]; + const expected = expectedEmptyChecksums[algoName]; const headerName = `x-amz-checksum-${algoName}`; - doPutRequest(zeroUrl, { + doPutRequest( + zeroUrl, + { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + [headerName]: expected, + 'content-length': 0, + }, + emptyBody, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200, `expected 200, got ${res.statusCode}: ${res.body}`); + assert.strictEqual(res.headers[headerName], expected, `expected ${headerName}: ${expected}`); + done(); + }, + ); + }, + ); + + itSkipIfAWS(`should return 400 BadDigest for wrong empty-body ${algoName} checksum`, done => { + const wrong = wrongEmptyDigests[algoName]; + const headerName = `x-amz-checksum-${algoName}`; + doPutRequest( + zeroUrl, + { 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', [headerName]: wrong, 'content-length': 0, - }, emptyBody, (err, res) => { + }, + emptyBody, + (err, res) => { assert.ifError(err); - assert.strictEqual(res.statusCode, 400, - `expected 400, got ${res.statusCode}: ${res.body}`); - assert(res.body.includes('BadDigest'), - `expected BadDigest in: ${res.body}`); + assert.strictEqual(res.statusCode, 400, `expected 400, got ${res.statusCode}: ${res.body}`); + assert(res.body.includes('BadDigest'), `expected BadDigest in: ${res.body}`); done(); - }); - }); + }, + ); + }); } itSkipIfAWS( @@ -914,20 +1118,28 @@ describe('PutObject: zero-byte object checksum handling', () => { // The trailer body is never consumed; server computes and stores the empty-body hash itself. const emptySha256 = expectedEmptyChecksums.sha256; const trailerBody = `0\r\nx-amz-checksum-sha256:${emptySha256}\n\r\n\r\n\r\n`; - doPutRequest(zeroUrl, { - 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', - 'x-amz-trailer': 'x-amz-checksum-sha256', - 'x-amz-decoded-content-length': 0, - 'content-length': Buffer.byteLength(trailerBody), - }, trailerBody, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 200, - `expected 200, got ${res.statusCode}: ${res.body}`); - assert.strictEqual(res.headers['x-amz-checksum-sha256'], emptySha256, - `expected x-amz-checksum-sha256: ${emptySha256}`); - done(); - }); - }); + doPutRequest( + zeroUrl, + { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + 'x-amz-decoded-content-length': 0, + 'content-length': Buffer.byteLength(trailerBody), + }, + trailerBody, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200, `expected 200, got ${res.statusCode}: ${res.body}`); + assert.strictEqual( + res.headers['x-amz-checksum-sha256'], + emptySha256, + `expected x-amz-checksum-sha256: ${emptySha256}`, + ); + done(); + }, + ); + }, + ); }); describe('PutObject: large body streaming checksums', () => { @@ -954,45 +1166,59 @@ describe('UploadPart: large body streaming checksums', () => { let uploadId3; before(done => { - async.series([ - next => makeS3Request({ method: 'PUT', authCredentials, bucket }, next), - next => makeS3Request({ - method: 'POST', - authCredentials, - bucket, - objectKey, - queryObj: { uploads: '' }, - }, (err, res) => { - if (err) { return next(err); } - const match = res.body.match(/([^<]+)<\/UploadId>/); - assert(match, `missing UploadId in response: ${res.body}`); - uploadId3 = match[1]; - return next(); - }), - ], err => { - assert.ifError(err); - done(); - }); + async.series( + [ + next => makeS3Request({ method: 'PUT', authCredentials, bucket }, next), + next => + makeS3Request( + { + method: 'POST', + authCredentials, + bucket, + objectKey, + queryObj: { uploads: '' }, + }, + (err, res) => { + if (err) { + return next(err); + } + const match = res.body.match(/([^<]+)<\/UploadId>/); + assert(match, `missing UploadId in response: ${res.body}`); + uploadId3 = match[1]; + return next(); + }, + ), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); after(done => { - async.series([ - next => makeS3Request({ - method: 'DELETE', - authCredentials, - bucket, - objectKey, - queryObj: { uploadId: uploadId3 }, - }, next), - next => makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => next()), - next => makeS3Request({ method: 'DELETE', authCredentials, bucket }, next), - ], err => { - assert.ifError(err); - done(); - }); + async.series( + [ + next => + makeS3Request( + { + method: 'DELETE', + authCredentials, + bucket, + objectKey, + queryObj: { uploadId: uploadId3 }, + }, + next, + ), + next => makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => next()), + next => makeS3Request({ method: 'DELETE', authCredentials, bucket }, next), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); - makeLargeBodyTests( - () => `http://localhost:8000/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId3}` - ); + makeLargeBodyTests(() => `http://localhost:8000/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId3}`); }); diff --git a/tests/unit/api/apiUtils/object/sourceChecksum.js b/tests/unit/api/apiUtils/object/sourceChecksum.js new file mode 100644 index 0000000000..057586803f --- /dev/null +++ b/tests/unit/api/apiUtils/object/sourceChecksum.js @@ -0,0 +1,165 @@ +const assert = require('assert'); +const sinon = require('sinon'); +const { Readable } = require('stream'); + +const { + buildSourcePartsStream, + computeChecksumFromDataLocator, +} = require('../../../../../lib/api/apiUtils/object/sourceChecksum'); +const dataWrapper = require('../../../../../lib/data/wrapper'); +const { algorithms } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); +const { DummyRequestLogger } = require('../../../helpers'); + +const log = new DummyRequestLogger(); + +// Build a source-part descriptor carrying its bytes (via `value`) for the +// stubbed data.get below to serve. `getError` makes data.get fail on that part. +function part(key, value, opts = {}) { + return { + key, + value, + dataStoreName: 'mem', + dataStoreType: 'mem', + ...opts, + }; +} + +// Drain a readable and hand back the fully concatenated bytes (or the error). +function collect(stream, cb) { + const chunks = []; + stream.on('data', chunk => chunks.push(chunk)); + stream.once('error', err => cb(err)); + stream.once('end', () => cb(null, Buffer.concat(chunks))); +} + +// Promisified computeChecksumFromDataLocator for the async digest assertions. +function computeChecksum(dataLocator, algorithm) { + return new Promise((resolve, reject) => { + computeChecksumFromDataLocator(dataLocator, algorithm, log, (err, res) => (err ? reject(err) : resolve(res))); + }); +} + +describe('sourceChecksum util', () => { + beforeEach(() => { + // Emulate the two data.get shapes buildSourcePartsStream relies on: + // azure writes part bytes into the provided writable; every other + // backend returns a Readable through the callback. + sinon.stub(dataWrapper.data, 'get').callsFake((p, writable, log2, cb) => { + if (p.getError) { + return cb(p.getError); + } + const bytes = Buffer.from(p.value || ''); + if (p.dataStoreType === 'azure') { + process.nextTick(() => writable.end(bytes)); + return cb(null); + } + const rs = new Readable({ read() {} }); + // Push after the caller has wired up listeners + pipe in its callback. + process.nextTick(() => { + if (bytes.length) { + rs.push(bytes); + } + rs.push(null); + }); + return cb(null, rs); + }); + }); + + afterEach(() => sinon.restore()); + + describe('buildSourcePartsStream', () => { + it('should concatenate parts in order', done => { + const locator = [part('a', 'Hello, '), part('b', 'world'), part('c', '!')]; + collect(buildSourcePartsStream(locator, log), (err, buf) => { + assert.ifError(err); + assert.strictEqual(buf.toString(), 'Hello, world!'); + done(); + }); + }); + + it('should serve azure parts (data.get writes into the provided writable)', done => { + const locator = [part('a', 'azure-bytes', { dataStoreType: 'azure', dataStoreName: 'azurebackend' })]; + collect(buildSourcePartsStream(locator, log), (err, buf) => { + assert.ifError(err); + assert.strictEqual(buf.toString(), 'azure-bytes'); + done(); + }); + }); + + it('should concatenate mixed azure and regular parts in order', done => { + const locator = [ + part('a', 'one-'), + part('b', 'two-', { dataStoreType: 'azure', dataStoreName: 'azurebackend' }), + part('c', 'three'), + ]; + collect(buildSourcePartsStream(locator, log), (err, buf) => { + assert.ifError(err); + assert.strictEqual(buf.toString(), 'one-two-three'); + done(); + }); + }); + + it('should emit an empty stream for an empty dataLocator', done => { + collect(buildSourcePartsStream([], log), (err, buf) => { + assert.ifError(err); + assert.strictEqual(buf.length, 0); + done(); + }); + }); + + it('should propagate a mid-stream read error wrapped with copyPart metadata', done => { + const boom = new Error('read failed'); + const locator = [ + part('ok', 'good'), + part('bad', '', { dataStoreName: 'ring0', dataStoreType: 'scality', getError: boom }), + ]; + const stream = buildSourcePartsStream(locator, log); + stream.on('data', () => {}); + stream.once('error', err => { + assert.strictEqual(err, boom); + assert.deepStrictEqual(err.copyPart, { + key: 'bad', + dataStoreName: 'ring0', + dataStoreType: 'scality', + }); + done(); + }); + }); + }); + + describe('computeChecksumFromDataLocator', () => { + const locator = [part('a', 'Hello, '), part('b', 'world'), part('c', '!')]; + const fullBytes = Buffer.from('Hello, world!'); + + // Derived from the algorithms map so a newly added algorithm is covered + // automatically. + Object.keys(algorithms).forEach(algo => { + it(`should compute the ${algo} digest over the concatenated source bytes`, async () => { + const expected = await algorithms[algo].digest(fullBytes); + const result = await computeChecksum(locator, algo); + assert.strictEqual(result.algorithm, algo); + assert.strictEqual(result.value, expected); + }); + + it(`should compute the empty-input ${algo} digest for an empty dataLocator`, async () => { + const expected = await algorithms[algo].digest(Buffer.alloc(0)); + const result = await computeChecksum([], algo); + assert.strictEqual(result.value, expected); + }); + }); + + it('should surface a read error wrapped with copyPart metadata', done => { + const boom = new Error('read failed'); + const locator2 = [part('bad', '', { dataStoreName: 'ring0', dataStoreType: 'scality', getError: boom })]; + computeChecksumFromDataLocator(locator2, 'crc32', log, err => { + assert.strictEqual(err, boom); + assert.deepStrictEqual(err.copyPart, { + key: 'bad', + dataStoreName: 'ring0', + dataStoreType: 'scality', + }); + done(); + }); + }); + }); +}); diff --git a/tests/unit/api/multipartUpload.js b/tests/unit/api/multipartUpload.js index c3a24cfb7b..002e99fa30 100644 --- a/tests/unit/api/multipartUpload.js +++ b/tests/unit/api/multipartUpload.js @@ -38,8 +38,16 @@ const { LOCATION_NAME_CRR } = require('../../constants'); const { data } = require('../../../lib/data/wrapper'); const { metadata } = storage.metadata.inMemory.metadata; const metadataBackend = storage.metadata.inMemory.metastore; +const originalDeleteObject = metadataBackend.deleteObject; const { ds } = storage.data.inMemory.datastore; +// Several tests override metadataBackend.deleteObject and restore it only on +// their success path, so a thrown assertion would leak the override into every +// later test. Reset it before each test (root hook, runs across all describes). +beforeEach(() => { + metadataBackend.deleteObject = originalDeleteObject; +}); + const log = new DummyRequestLogger(); const splitter = constants.splitter; @@ -3431,17 +3439,6 @@ describe('objectPutPart checksum response headers', () => { done(); }); }); - - it('should return x-amz-checksum-crc64nvme response header when no checksum header is provided', done => { - const expectedCrc64nvme = '5evlCr2wyO4='; - const partRequest = _createPutPartRequest(testUploadId, '1', postBody); - - objectPutPart(authInfo, partRequest, undefined, log, (err, _hexDigest, resHeaders) => { - assert.ifError(err); - assert.strictEqual(resHeaders['x-amz-checksum-crc64nvme'], expectedCrc64nvme); - done(); - }); - }); }); describe('initiateMultipartUpload checksum headers', () => { @@ -3898,6 +3895,30 @@ describe('validatePerPartChecksums', () => { assert.strictEqual(err.message, 'InvalidPart'); }); }); + + describe('external backend MPU (isExternal=true)', () => { + // External parts store no per-part checksum, so the COMPOSITE requirement + // is relaxed - but a checksum the client submits is still rejected, since + // there is no stored value to verify it against. + it('should not require a per-part checksum (external parts store none)', () => { + const mpuChecksum = { algorithm: 'crc32', type: 'COMPOSITE', isDefault: false }; + const stored = [makeStoredPart(1, null)]; + const jsonList = { Part: [makeJsonPart(1, 'etag1')] }; + const err = validatePerPartChecksums(jsonList, stored, splitter, mpuChecksum, true); + assert.ifError(err); + }); + + it('should still return InvalidPart for a client-submitted checksum (nothing to verify against)', () => { + const mpuChecksum = { algorithm: 'crc32', type: 'FULL_OBJECT', isDefault: false }; + const stored = [makeStoredPart(1, null)]; + const jsonList = { + Part: [makeJsonPart(1, 'etag1', { ChecksumCRC32: SAMPLE_DIGESTS.crc32[0] })], + }; + const err = validatePerPartChecksums(jsonList, stored, splitter, mpuChecksum, true); + assert(err); + assert.strictEqual(err.message, 'InvalidPart'); + }); + }); }); describe('CompleteMultipartUpload x-amz-checksum-type header', () => { @@ -4406,3 +4427,116 @@ describe('CompleteMultipartUpload final-object checksum response', () => { assert.strictEqual(headers['x-amz-checksum-type'], undefined); }); }); + +describe('CompleteMultipartUpload per-part validation on external backends', () => { + // External backend parts store no per-part checksum, so CompleteMPU relaxes + // the COMPOSITE per-part requirement for them - but still rejects any checksum + // the client submits, since it can't be verified. The location is flipped via + // a getLocationConstraintType stub so only that gate differs. + const dataClient = data.client; + const prevDataImplName = data.implName; + const prevConfigBackendsData = data.config.backends.data; + const versionID = versioning.VersionID.encode(versioning.VersionID.generateVersionId('0', '')); + + before(() => { + // Simulate a backend that handles the MPU itself, so uploaded parts get + // no local shadow checksum and CompleteMPU is backend-driven. + data.switch( + new storage.data.MultipleBackendGateway( + { + 'us-east-1': dataClient, + 'us-east-2': dataClient, + }, + metadata, + data.locStorageCheckFn, + ), + ); + data.implName = 'multipleBackends'; + data.config.backends.data = 'multiple'; + dataClient.clientType = 'aws_s3'; + }); + + after(() => { + data.switch(dataClient); + data.implName = prevDataImplName; + data.config.backends.data = prevConfigBackendsData; + delete dataClient.clientType; + }); + + beforeEach(() => { + cleanup(); + dataClient.createMPU = sinon.stub().yields(undefined, { uploadId: 'mock-uploadId' }); + dataClient.uploadPart = sinon.stub().yields(undefined, { + dataStoreType: dataClient.clientType, + dataStoreETag: 'mock-part-eTag', + }); + dataClient.completeMPU = sinon.stub().yields(undefined, { + key: objectKey, + eTag: 'mock-eTag', + dataStoreVersionId: versionID, + contentLength: 12, + }); + }); + + afterEach(() => { + sinon.restore(); + }); + + const newPutIngestBucketRequest = location => + new DummyRequest({ + bucketName, + namespace, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + post: + '' + + '' + + `${location}` + + '', + }); + + // Create an MPU on the (external) ingest backend and upload one part. The + // part is stored without a per-part checksum, as external backends do. + async function _initiateExternalMpu({ algo = 'CRC32', type = 'COMPOSITE' } = {}) { + await _bucketPut(authInfo, newPutIngestBucketRequest('us-east-1:ingest'), log); + const initiate = new DummyRequest({ + bucketName, + namespace, + objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-checksum-algorithm': algo, + 'x-amz-checksum-type': type, + }, + url: `/${objectKey}?uploads`, + }); + const initRes = await util.promisify(initiateMultipartUpload)(authInfo, initiate, log); + const uploadId = (await parseStringPromise(initRes)).InitiateMultipartUploadResult.UploadId[0]; + const partReq = _createPutPartRequest(uploadId, 1, Buffer.from('part body', 'utf8')); + const eTag = await util.promisify(objectPutPart)(authInfo, partReq, undefined, log); + return { uploadId, eTag }; + } + + const _complete = completeReq => util.promisify(cb => completeMultipartUpload(authInfo, completeReq, log, cb))(); + + describe('COMPOSITE MPU (no per-part checksum)', () => { + it('should reject on a local location', async () => { + const { uploadId, eTag } = await _initiateExternalMpu({ type: 'COMPOSITE' }); + const completeReq = _createCompleteMpuRequest(uploadId, [{ partNumber: 1, eTag }]); + sinon.stub(config, 'getLocationConstraintType').returns('scality'); + await assert.rejects(_complete(completeReq), err => { + assert.match(err.message, /InvalidRequest/); + return true; + }); + }); + + it('should complete on an external location', async () => { + const { uploadId, eTag } = await _initiateExternalMpu({ type: 'COMPOSITE' }); + const completeReq = _createCompleteMpuRequest(uploadId, [{ partNumber: 1, eTag }]); + sinon.stub(config, 'getLocationConstraintType').returns('aws_s3'); + const result = await _complete(completeReq); + assert(result, 'external COMPOSITE MPU should complete without per-part checksums'); + }); + }); +}); diff --git a/tests/unit/api/objectCopyPart.js b/tests/unit/api/objectCopyPart.js index 46c95a0452..cbcf713d40 100644 --- a/tests/unit/api/objectCopyPart.js +++ b/tests/unit/api/objectCopyPart.js @@ -6,13 +6,18 @@ const { storage } = require('arsenal'); const { bucketPut } = require('../../../lib/api/bucketPut'); const objectPut = require('../../../lib/api/objectPut'); const objectPutCopyPart = require('../../../lib/api/objectPutCopyPart'); -const initiateMultipartUpload -= require('../../../lib/api/initiateMultipartUpload'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); const { metadata } = storage.metadata.inMemory.metadata; const metadataswitch = require('../metadataswitch'); const DummyRequest = require('../DummyRequest'); -const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } = require('../helpers'); +const { Readable } = require('stream'); +const { algorithms } = require('../../../lib/api/apiUtils/integrity/validateChecksums'); +const { data } = require('../../../lib/data/wrapper'); +const { config } = require('../../../lib/Config'); +const kms = require('../../../lib/kms/wrapper'); + +const checksumAlgos = Object.keys(algorithms); const log = new DummyRequestLogger(); const canonicalID = 'accessKey1'; @@ -31,12 +36,12 @@ function _createBucketPutRequest(bucketName) { }); } -function _createInitiateRequest(bucketName) { +function _createInitiateRequest(bucketName, extraHeaders) { const params = { bucketName, namespace, objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, + headers: { host: `${bucketName}.s3.amazonaws.com`, ...extraHeaders }, url: `/${objectKey}?uploads`, }; return new DummyRequest(params); @@ -64,30 +69,27 @@ const initiateRequest = _createInitiateRequest(destBucketName); describe('objectCopyPart', () => { let uploadId; const objData = Buffer.from('foo', 'utf8'); - const testPutObjectRequest = - versioningTestUtils.createPutObjectRequest(sourceBucketName, objectKey, - objData); + const testPutObjectRequest = versioningTestUtils.createPutObjectRequest(sourceBucketName, objectKey, objData); before(done => { cleanup(); sinon.spy(metadataswitch, 'putObjectMD'); - async.waterfall([ - callback => bucketPut(authInfo, putDestBucketRequest, log, - err => callback(err)), - callback => bucketPut(authInfo, putSourceBucketRequest, log, - err => callback(err)), - callback => objectPut(authInfo, testPutObjectRequest, - undefined, log, err => callback(err)), - callback => initiateMultipartUpload(authInfo, initiateRequest, - log, (err, res) => callback(err, res)), - ], (err, res) => { - if (err) { - return done(err); - } - return parseString(res, (err, json) => { - uploadId = json.InitiateMultipartUploadResult.UploadId[0]; - return done(); - }); - }); + async.waterfall( + [ + callback => bucketPut(authInfo, putDestBucketRequest, log, err => callback(err)), + callback => bucketPut(authInfo, putSourceBucketRequest, log, err => callback(err)), + callback => objectPut(authInfo, testPutObjectRequest, undefined, log, err => callback(err)), + callback => initiateMultipartUpload(authInfo, initiateRequest, log, (err, res) => callback(err, res)), + ], + (err, res) => { + if (err) { + return done(err); + } + return parseString(res, (err, json) => { + uploadId = json.InitiateMultipartUploadResult.UploadId[0]; + return done(); + }); + }, + ); }); after(() => { @@ -95,8 +97,7 @@ describe('objectCopyPart', () => { cleanup(); }); - it('should copy part even if legacy metadata without dataStoreName', - done => { + it('should copy part even if legacy metadata without dataStoreName', done => { // force metadata for dataStoreName to be undefined metadata.keyMaps.get(sourceBucketName).get(objectKey).dataStoreName = undefined; const testObjectCopyRequest = _createObjectCopyPartRequest(destBucketName, uploadId); @@ -108,17 +109,17 @@ describe('objectCopyPart', () => { it('should return InvalidArgument error given invalid range', done => { const headers = { 'x-amz-copy-source-range': 'bad-range-parameter' }; - const req = - _createObjectCopyPartRequest(destBucketName, uploadId, headers); - objectPutCopyPart( - authInfo, req, sourceBucketName, objectKey, undefined, log, err => { - assert(err.is.InvalidArgument); - assert.strictEqual(err.description, - 'The x-amz-copy-source-range value must be of the form ' + + const req = _createObjectCopyPartRequest(destBucketName, uploadId, headers); + objectPutCopyPart(authInfo, req, sourceBucketName, objectKey, undefined, log, err => { + assert(err.is.InvalidArgument); + assert.strictEqual( + err.description, + 'The x-amz-copy-source-range value must be of the form ' + 'bytes=first-last where first and last are the ' + - 'zero-based offsets of the first and last bytes to copy'); - done(); - }); + 'zero-based offsets of the first and last bytes to copy', + ); + done(); + }); }); it('should pass overheadField', done => { @@ -132,7 +133,7 @@ describe('objectCopyPart', () => { sinon.match.any, sinon.match({ overheadField: sinon.match.array }), sinon.match.any, - sinon.match.any + sinon.match.any, ); done(); }); @@ -149,8 +150,309 @@ describe('objectCopyPart', () => { sinon.match({ 'owner-id': authInfo.canonicalID }), sinon.match.any, sinon.match.any, - sinon.match.any + sinon.match.any, + ); + done(); + }); + }); +}); + +describe('objectPutCopyPart._shouldRecomputeChecksum', () => { + const { _shouldRecomputeChecksum } = objectPutCopyPart; + const noRange = { headers: {} }; + const withRange = { headers: { 'x-amz-copy-source-range': 'bytes=0-1' } }; + + it('should return true when a copy-source-range is requested', () => { + assert.strictEqual( + _shouldRecomputeChecksum(withRange, { checksumType: 'FULL_OBJECT', checksumAlgorithm: 'crc32' }, 'crc32'), + true, + ); + }); + + it('should return true when the source has no checksum', () => { + assert.strictEqual(_shouldRecomputeChecksum(noRange, undefined, 'crc32'), true); + }); + + checksumAlgos.forEach(algo => { + const otherAlgo = algo === 'sha256' ? 'crc32' : 'sha256'; + + it(`should return false when the source is FULL_OBJECT ${algo} matching the MPU`, () => { + assert.strictEqual( + _shouldRecomputeChecksum(noRange, { checksumType: 'FULL_OBJECT', checksumAlgorithm: algo }, algo), + false, ); + }); + + it(`should return true when the source ${otherAlgo} differs from the MPU ${algo}`, () => { + assert.strictEqual( + _shouldRecomputeChecksum(noRange, { checksumType: 'FULL_OBJECT', checksumAlgorithm: otherAlgo }, algo), + true, + ); + }); + + it(`should return true when the source ${algo} checksum is COMPOSITE`, () => { + assert.strictEqual( + _shouldRecomputeChecksum(noRange, { checksumType: 'COMPOSITE', checksumAlgorithm: algo }, algo), + true, + ); + }); + }); +}); + +describe('objectPutCopyPart checksum storage', () => { + const objData = Buffer.from('foo', 'utf8'); + + function _initiateWithHeaders(headers, cb) { + const req = _createInitiateRequest(destBucketName, headers); + return initiateMultipartUpload(authInfo, req, log, (err, res) => { + if (err) { + return cb(err); + } + return parseString(res, (parseErr, json) => cb(parseErr, json.InitiateMultipartUploadResult.UploadId[0])); + }); + } + + // The part metadata is the last object written; pull its checksum fields. + function _storedPartChecksum() { + const omVal = metadataswitch.putObjectMD.lastCall.args[2]; + return { algorithm: omVal.checksumAlgorithm, value: omVal.checksumValue }; + } + + // Initiate an MPU for `algo`, optionally inject the source object's stored + // checksum (to drive the reuse-vs-recompute decision), then copy a part and + // resolve with the checksum persisted in the part metadata. + function _copyPart(algo, { sourceKey = objectKey, sourceChecksum, headers } = {}) { + return new Promise((resolve, reject) => { + _initiateWithHeaders({ 'x-amz-checksum-algorithm': algo.toUpperCase() }, (err, uploadId) => { + if (err) { + return reject(err); + } + if (sourceChecksum) { + metadata.keyMaps.get(sourceBucketName).get(sourceKey).checksum = sourceChecksum; + } + const req = _createObjectCopyPartRequest(destBucketName, uploadId, headers); + return objectPutCopyPart(authInfo, req, sourceBucketName, sourceKey, undefined, log, copyErr => + copyErr ? reject(copyErr) : resolve(_storedPartChecksum()), + ); + }); + }); + } + + beforeEach(done => { + cleanup(); + sinon.spy(metadataswitch, 'putObjectMD'); + async.waterfall( + [ + cb => bucketPut(authInfo, putDestBucketRequest, log, e => cb(e)), + cb => bucketPut(authInfo, putSourceBucketRequest, log, e => cb(e)), + cb => + objectPut( + authInfo, + versioningTestUtils.createPutObjectRequest(sourceBucketName, objectKey, objData), + undefined, + log, + e => cb(e), + ), + ], + done, + ); + }); + + afterEach(() => { + sinon.restore(); + cleanup(); + }); + + checksumAlgos.forEach(algo => { + const otherAlgo = algo === 'sha256' ? 'crc32' : 'sha256'; + const mismatch = { checksumType: 'FULL_OBJECT', checksumAlgorithm: otherAlgo, checksumValue: 'unused' }; + + it(`should recompute the part checksum (${algo}) when the source algorithm differs`, async () => { + const expected = await algorithms[algo].digest(objData); + assert.deepStrictEqual(await _copyPart(algo, { sourceChecksum: mismatch }), { + algorithm: algo, + value: expected, + }); + }); + + it(`should reuse the source checksum (${algo}) when the algorithm matches`, async () => { + const sourceValue = await algorithms[algo].digest(objData); + assert.deepStrictEqual( + await _copyPart(algo, { + sourceChecksum: { + checksumType: 'FULL_OBJECT', + checksumAlgorithm: algo, + checksumValue: sourceValue, + }, + }), + { algorithm: algo, value: sourceValue }, + ); + }); + + it(`should recompute over the ranged bytes (${algo}) when a copy-source-range is set`, async () => { + const expected = await algorithms[algo].digest(Buffer.from('fo')); + assert.deepStrictEqual( + await _copyPart(algo, { + sourceChecksum: { + checksumType: 'FULL_OBJECT', + checksumAlgorithm: algo, + checksumValue: await algorithms[algo].digest(objData), + }, + headers: { 'x-amz-copy-source-range': 'bytes=0-1' }, + }), + { algorithm: algo, value: expected }, + ); + }); + + it(`should store the empty-bytes digest (${algo}) for a 0-byte source`, async () => { + const expected = await algorithms[algo].digest(Buffer.alloc(0)); + const emptyKey = `empty-source-${algo}`; + await new Promise((resolve, reject) => + objectPut( + authInfo, + versioningTestUtils.createPutObjectRequest(sourceBucketName, emptyKey, Buffer.alloc(0)), + undefined, + log, + e => (e ? reject(e) : resolve()), + ), + ); + assert.deepStrictEqual(await _copyPart(algo, { sourceKey: emptyKey, sourceChecksum: mismatch }), { + algorithm: algo, + value: expected, + }); + }); + }); + + it('should use the one-pass stream (not data.uploadPartCopy) for a local destination', done => { + _initiateWithHeaders({ 'x-amz-checksum-algorithm': 'CRC32' }, (err, uploadId) => { + assert.ifError(err); + metadata.keyMaps.get(sourceBucketName).get(objectKey).checksum = { + checksumType: 'FULL_OBJECT', + checksumAlgorithm: 'sha256', + checksumValue: 'unused', + }; + const uploadPartCopySpy = sinon.spy(data, 'uploadPartCopy'); + const req = _createObjectCopyPartRequest(destBucketName, uploadId); + objectPutCopyPart(authInfo, req, sourceBucketName, objectKey, undefined, log, copyErr => { + assert.ifError(copyErr); + sinon.assert.notCalled(uploadPartCopySpy); + done(); + }); + }); + }); + + it('should route an external-backend destination through data.uploadPartCopy and return no checksum', done => { + _initiateWithHeaders({ 'x-amz-checksum-algorithm': 'CRC32' }, (err, uploadId) => { + assert.ifError(err); + metadata.keyMaps.get(sourceBucketName).get(objectKey).checksum = { + checksumType: 'FULL_OBJECT', + checksumAlgorithm: 'sha256', + checksumValue: 'unused', + }; + // Make the destination look like an external backend (data.put can't store its parts)... + sinon.stub(config, 'getLocationConstraintType').returns('aws_s3'); + // ...and simulate the backend's native part copy returning the skip sentinel. + const uploadPartCopyStub = sinon.stub(data, 'uploadPartCopy').callsFake((...args) => { + const cb = args[args.length - 1]; + return cb(new Error('skip'), 'etag', '2020-01-01T00:00:00.000Z', null, [{ dataStoreETag: 'etag' }]); + }); + const req = _createObjectCopyPartRequest(destBucketName, uploadId); + objectPutCopyPart(authInfo, req, sourceBucketName, objectKey, undefined, log, (copyErr, xml) => { + assert.ifError(copyErr); + sinon.assert.calledOnce(uploadPartCopyStub); + // external backends get no cloudserver checksum (matches UploadPart) + assert.doesNotMatch(xml, /Checksum/); + done(); + }); + }); + }); +}); + +describe('objectPutCopyPart._copyPartStreamingWithChecksum', () => { + const { _copyPartStreamingWithChecksum } = objectPutCopyPart; + const srcBytes = Buffer.from('hello-copy-part', 'utf8'); + const dataLocator = [{ key: 'srckey', dataStoreType: 'mem', dataStoreName: 'mem' }]; + + beforeEach(() => { + // Serve the source bytes for buildSourcePartsStream. + sinon.stub(data, 'get').callsFake((part, writable, log2, cb) => { + const rs = new Readable({ read() {} }); + process.nextTick(() => { + rs.push(srcBytes); + rs.push(null); + }); + return cb(null, rs); + }); + // Drain the checksum stream (so it flushes its digest) and report an md5. + sinon.stub(data, 'put').callsFake((cipherBundle, stream, size, ctx, backendInfo, log2, cb) => { + stream.on('data', () => {}); + stream.once('end', () => cb(null, { key: 'destkey', dataStoreName: 'mem' }, { completedHash: 'fakemd5' })); + }); + }); + + afterEach(() => sinon.restore()); + + function _run(sse, algo) { + return new Promise((resolve, reject) => + _copyPartStreamingWithChecksum( + dataLocator, + srcBytes.length, + sse, + 'us-east-1', + {}, + algo, + log, + (err, result) => (err ? reject(err) : resolve(result)), + ), + ); + } + + checksumAlgos.forEach(algo => { + it(`should return the part location, eTag and ${algo} checksum for an unencrypted copy`, async () => { + const result = await _run(null, algo); + assert.deepStrictEqual(result.locations, [ + { + key: 'destkey', + dataStoreName: 'mem', + dataStoreETag: 'fakemd5', + size: srcBytes.length, + }, + ]); + assert.strictEqual(result.totalHash, 'fakemd5'); + assert.deepStrictEqual(result.checksum, { + algorithm: algo, + value: await algorithms[algo].digest(srcBytes), + }); + }); + + it(`should add the SSE cipher fields with a ${algo} checksum when the MPU is encrypted`, async () => { + const cipherBundle = { cryptoScheme: 1, cipheredDataKey: 'dk', algorithm: 'AES256', masterKeyId: 'mk' }; + sinon.stub(kms, 'createCipherBundle').callsFake((sse, log2, cb) => cb(null, cipherBundle)); + const result = await _run({ algorithm: 'AES256' }, algo); + assert.deepStrictEqual(result.locations[0], { + key: 'destkey', + dataStoreName: 'mem', + dataStoreETag: 'fakemd5', + size: srcBytes.length, + sseCryptoScheme: 1, + sseCipheredDataKey: 'dk', + sseAlgorithm: 'AES256', + sseMasterKeyId: 'mk', + }); + assert.deepStrictEqual(result.checksum, { + algorithm: algo, + value: await algorithms[algo].digest(srcBytes), + }); + }); + }); + + it('should surface a source read error wrapped with copyPart metadata', done => { + data.get.restore(); + const boom = new Error('read failed'); + sinon.stub(data, 'get').callsFake((part, writable, log2, cb) => cb(boom)); + _copyPartStreamingWithChecksum(dataLocator, srcBytes.length, null, 'us-east-1', {}, 'crc32', log, err => { + assert.strictEqual(err, boom); + assert.strictEqual(err.copyPart.key, 'srckey'); done(); }); }); diff --git a/tests/unit/api/objectPutPartChecksum.js b/tests/unit/api/objectPutPartChecksum.js index f518ca4885..f933e56dc1 100644 --- a/tests/unit/api/objectPutPartChecksum.js +++ b/tests/unit/api/objectPutPartChecksum.js @@ -48,38 +48,48 @@ function makeInitiateRequest(extraHeaders = {}) { function makePutPartRequest(uploadId, partNumber, body, extraHeaders = {}) { const md5Hash = crypto.createHash('md5').update(body); - return new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { - host: `${bucketName}.s3.amazonaws.com`, - ...extraHeaders, + return new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + ...extraHeaders, + }, + url: `/${objectKey}?partNumber=${partNumber}&uploadId=${uploadId}`, + query: { partNumber, uploadId }, + partHash: md5Hash.digest('hex'), + actionImplicitDenies: false, }, - url: `/${objectKey}?partNumber=${partNumber}&uploadId=${uploadId}`, - query: { partNumber, uploadId }, - partHash: md5Hash.digest('hex'), - actionImplicitDenies: false, - }, body); + body, + ); } function initiateMPU(initiateHeaders, cb) { - async.waterfall([ - next => bucketPut(authInfo, bucketPutRequest, log, next), - (corsHeaders, next) => { - const req = makeInitiateRequest(initiateHeaders); - initiateMultipartUpload(authInfo, req, log, next); + async.waterfall( + [ + next => bucketPut(authInfo, bucketPutRequest, log, next), + (corsHeaders, next) => { + const req = makeInitiateRequest(initiateHeaders); + initiateMultipartUpload(authInfo, req, log, next); + }, + (result, corsHeaders, next) => parseString(result, next), + ], + (err, json) => { + if (err) { + return cb(err); + } + return cb(null, json.InitiateMultipartUploadResult.UploadId[0]); }, - (result, corsHeaders, next) => parseString(result, next), - ], (err, json) => { - if (err) {return cb(err);} - return cb(null, json.InitiateMultipartUploadResult.UploadId[0]); - }); + ); } function getPartMetadata(uploadId) { const mpuKeys = metadata.keyMaps.get(mpuBucket); - if (!mpuKeys) {return null;} + if (!mpuKeys) { + return null; + } for (const [key, val] of mpuKeys) { if (key.startsWith(uploadId) && !key.startsWith('overview')) { return val; @@ -123,11 +133,28 @@ describe('objectPutPart checksum validation', () => { }); }); - it('should accept part with no checksum on non-default MPU', done => { + it('should reject part with no checksum on a COMPOSITE MPU', done => { + // sha256 is COMPOSITE-only; a COMPOSITE MPU's final checksum is + // composed from the per-part checksums, so every part must carry + // one and AWS rejects a part sent without it. initiateMPU({ 'x-amz-checksum-algorithm': 'sha256' }, (err, uploadId) => { assert.ifError(err); // No checksum header sent const request = makePutPartRequest(uploadId, 1, partBody); + objectPutPart(authInfo, request, undefined, log, err => { + assert(err, 'Expected an error'); + assert.strictEqual(err.message, 'InvalidRequest'); + done(); + }); + }); + }); + + it('should accept part with no checksum on a FULL_OBJECT MPU', done => { + // crc64nvme is FULL_OBJECT-only; the server computes the + // full-object checksum, so a missing per-part checksum is allowed. + initiateMPU({ 'x-amz-checksum-algorithm': 'crc64nvme' }, (err, uploadId) => { + assert.ifError(err); + const request = makePutPartRequest(uploadId, 1, partBody); objectPutPart(authInfo, request, undefined, log, err => { assert.ifError(err); done(); @@ -225,22 +252,24 @@ describe('objectPutPart checksum validation', () => { Promise.all([ algorithms.crc32.digestFromHash(crc32Hash), algorithms.crc64nvme.digestFromHash(crc64Hash), - ]).then(([crc32Digest, crc64Digest]) => { - const request = makePutPartRequest(uploadId, 1, partBody, { - 'x-amz-checksum-crc32': crc32Digest, - }); - objectPutPart(authInfo, request, undefined, log, (err, hexDigest, corsHeaders) => { - assert.ifError(err); - // Response header should be the client's algo (crc32) - assert.strictEqual(corsHeaders['x-amz-checksum-crc32'], crc32Digest); - // Stored metadata should be crc64nvme with correct value - const partMD = getPartMetadata(uploadId); - assert(partMD); - assert.strictEqual(partMD.checksumAlgorithm, 'crc64nvme'); - assert.strictEqual(partMD.checksumValue, crc64Digest); - done(); - }); - }).catch(done); + ]) + .then(([crc32Digest, crc64Digest]) => { + const request = makePutPartRequest(uploadId, 1, partBody, { + 'x-amz-checksum-crc32': crc32Digest, + }); + objectPutPart(authInfo, request, undefined, log, (err, hexDigest, corsHeaders) => { + assert.ifError(err); + // Response header should be the client's algo (crc32) + assert.strictEqual(corsHeaders['x-amz-checksum-crc32'], crc32Digest); + // Stored metadata should be crc64nvme with correct value + const partMD = getPartMetadata(uploadId); + assert(partMD); + assert.strictEqual(partMD.checksumAlgorithm, 'crc64nvme'); + assert.strictEqual(partMD.checksumValue, crc64Digest); + done(); + }); + }) + .catch(done); }); }); @@ -251,31 +280,31 @@ describe('objectPutPart checksum validation', () => { hash.update(partBody); const crc64Hash = algorithms.crc64nvme.createHash(); crc64Hash.update(partBody); - Promise.all([ - algorithms.sha256.digestFromHash(hash), - algorithms.crc64nvme.digestFromHash(crc64Hash), - ]).then(([sha256Digest, crc64Digest]) => { - // Build chunked body with trailing checksum - const hexLen = partBody.length.toString(16); - const chunkedBody = `${hexLen}\r\n${partBody.toString()}\r\n` + - `0\r\nx-amz-checksum-sha256:${sha256Digest}\r\n`; - const request = makePutPartRequest(uploadId, 1, Buffer.from(chunkedBody), { - 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', - 'x-amz-trailer': 'x-amz-checksum-sha256', - }); - request.parsedContentLength = partBody.length; - objectPutPart(authInfo, request, undefined, log, (err, hexDigest, corsHeaders) => { - assert.ifError(err); - // Response should echo the client's sha256 - assert.strictEqual(corsHeaders['x-amz-checksum-sha256'], sha256Digest); - // Stored metadata should be crc64nvme with correct value - const partMD = getPartMetadata(uploadId); - assert(partMD); - assert.strictEqual(partMD.checksumAlgorithm, 'crc64nvme'); - assert.strictEqual(partMD.checksumValue, crc64Digest); - done(); - }); - }).catch(done); + Promise.all([algorithms.sha256.digestFromHash(hash), algorithms.crc64nvme.digestFromHash(crc64Hash)]) + .then(([sha256Digest, crc64Digest]) => { + // Build chunked body with trailing checksum + const hexLen = partBody.length.toString(16); + const chunkedBody = + `${hexLen}\r\n${partBody.toString()}\r\n` + + `0\r\nx-amz-checksum-sha256:${sha256Digest}\r\n`; + const request = makePutPartRequest(uploadId, 1, Buffer.from(chunkedBody), { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + }); + request.parsedContentLength = partBody.length; + objectPutPart(authInfo, request, undefined, log, (err, hexDigest, corsHeaders) => { + assert.ifError(err); + // Response should echo the client's sha256 + assert.strictEqual(corsHeaders['x-amz-checksum-sha256'], sha256Digest); + // Stored metadata should be crc64nvme with correct value + const partMD = getPartMetadata(uploadId); + assert(partMD); + assert.strictEqual(partMD.checksumAlgorithm, 'crc64nvme'); + assert.strictEqual(partMD.checksumValue, crc64Digest); + done(); + }); + }) + .catch(done); }); }); @@ -284,17 +313,81 @@ describe('objectPutPart checksum validation', () => { assert.ifError(err); const hash = algorithms.sha256.createHash(); hash.update(partBody); - Promise.resolve(algorithms.sha256.digestFromHash(hash)).then(digest => { - const request = makePutPartRequest(uploadId, 1, partBody, { - 'x-amz-checksum-sha256': digest, - }); - objectPutPart(authInfo, request, undefined, log, (err, hexDigest, corsHeaders) => { - assert.ifError(err); - assert.strictEqual(corsHeaders['x-amz-checksum-sha256'], digest); - assert.strictEqual(corsHeaders['x-amz-checksum-crc64nvme'], undefined); - done(); + Promise.resolve(algorithms.sha256.digestFromHash(hash)) + .then(digest => { + const request = makePutPartRequest(uploadId, 1, partBody, { + 'x-amz-checksum-sha256': digest, + }); + objectPutPart(authInfo, request, undefined, log, (err, hexDigest, corsHeaders) => { + assert.ifError(err); + assert.strictEqual(corsHeaders['x-amz-checksum-sha256'], digest); + assert.strictEqual(corsHeaders['x-amz-checksum-crc64nvme'], undefined); + done(); + }); + }) + .catch(done); + }); + }); + }); + + describe('response checksum header', () => { + const algos = Object.keys(algorithms); + + it('should not return a checksum header on a default MPU when none is sent', done => { + initiateMPU({}, (err, uploadId) => { + assert.ifError(err); + const request = makePutPartRequest(uploadId, 1, partBody); + objectPutPart(authInfo, request, undefined, log, (err, hexDigest, corsHeaders) => { + assert.ifError(err); + algos.forEach(algo => { + assert.strictEqual(corsHeaders[`x-amz-checksum-${algo}`], undefined); }); - }).catch(done); + // The part checksum is still stored so CompleteMPU can + // compute the final object checksum. + const partMD = getPartMetadata(uploadId); + assert(partMD); + assert.strictEqual(partMD.checksumAlgorithm, 'crc64nvme'); + assert(partMD.checksumValue); + done(); + }); + }); + }); + + algos.forEach(algo => { + it(`should echo a client-supplied ${algo} checksum on a default MPU`, done => { + initiateMPU({}, (err, uploadId) => { + assert.ifError(err); + Promise.resolve(algorithms[algo].digest(partBody)) + .then(digest => { + const request = makePutPartRequest(uploadId, 1, partBody, { + [`x-amz-checksum-${algo}`]: digest, + }); + objectPutPart(authInfo, request, undefined, log, (err, hexDigest, corsHeaders) => { + assert.ifError(err); + assert.strictEqual(corsHeaders[`x-amz-checksum-${algo}`], digest); + done(); + }); + }) + .catch(done); + }); + }); + + it(`should echo the ${algo} checksum on an explicit ${algo} MPU`, done => { + initiateMPU({ 'x-amz-checksum-algorithm': algo }, (err, uploadId) => { + assert.ifError(err); + Promise.resolve(algorithms[algo].digest(partBody)) + .then(digest => { + const request = makePutPartRequest(uploadId, 1, partBody, { + [`x-amz-checksum-${algo}`]: digest, + }); + objectPutPart(authInfo, request, undefined, log, (err, hexDigest, corsHeaders) => { + assert.ifError(err); + assert.strictEqual(corsHeaders[`x-amz-checksum-${algo}`], digest); + done(); + }); + }) + .catch(done); + }); }); }); }); diff --git a/tests/unit/lib/services.spec.js b/tests/unit/lib/services.spec.js index ba8b4499b2..3249be1c62 100644 --- a/tests/unit/lib/services.spec.js +++ b/tests/unit/lib/services.spec.js @@ -241,6 +241,65 @@ describe('services', () => { }); }); + describe('metadataStorePart checksum fields', () => { + const mpuBucketName = 'mpu-test-bucket'; + const baseParams = { + partNumber: 1, + contentMD5: 'd41d8cd98f00b204e9800998ecf8427e', + size: 5, + uploadId: 'test-upload-id', + splitter: '|', + ownerId: 'ownerCanonicalId', + }; + + let putObjectMDStub; + + beforeEach(() => { + putObjectMDStub = sinon + .stub(metadata, 'putObjectMD') + .callsFake((bucket, key, md, opts, reqLog, cb) => cb(null)); + }); + + it('should persist checksumValue and checksumAlgorithm when both are provided', done => { + const params = { + ...baseParams, + checksumValue: 'NSRBwg==', + checksumAlgorithm: 'crc32', + }; + + services.metadataStorePart(mpuBucketName, [], params, log, err => { + assert.ifError(err); + sinon.assert.calledOnce(putObjectMDStub); + const storedMD = putObjectMDStub.getCall(0).args[2]; + assert.strictEqual(storedMD.checksumValue, 'NSRBwg=='); + assert.strictEqual(storedMD.checksumAlgorithm, 'crc32'); + done(); + }); + }); + + it('should not persist checksum fields when none are provided', done => { + services.metadataStorePart(mpuBucketName, [], { ...baseParams }, log, err => { + assert.ifError(err); + const storedMD = putObjectMDStub.getCall(0).args[2]; + assert.strictEqual(storedMD.checksumValue, undefined); + assert.strictEqual(storedMD.checksumAlgorithm, undefined); + done(); + }); + }); + + it('should not persist checksum fields when only the value is provided', done => { + const params = { ...baseParams, checksumValue: 'NSRBwg==' }; + + services.metadataStorePart(mpuBucketName, [], params, log, err => { + assert.ifError(err); + const storedMD = putObjectMDStub.getCall(0).args[2]; + assert.strictEqual(storedMD.checksumValue, undefined); + assert.strictEqual(storedMD.checksumAlgorithm, undefined); + done(); + }); + }); + }); + describe('metadataValidateMultipart checksum fields', () => { const uploadId = 'test-upload-id'; const authInfo = makeAuthInfo('accessKey1');