diff --git a/.cspell.json b/.cspell.json index 0697ccc4edd..b72ea55deb2 100644 --- a/.cspell.json +++ b/.cspell.json @@ -261,6 +261,7 @@ "ossl", "ccrng", "KEYWRAP", + "HKDF", "NVME", // EC2 "IMDS", diff --git a/src/aws-cpp-sdk-transfer/include/aws/transfer/TransferHandle.h b/src/aws-cpp-sdk-transfer/include/aws/transfer/TransferHandle.h index 0d062be1e00..cb852bb0e25 100644 --- a/src/aws-cpp-sdk-transfer/include/aws/transfer/TransferHandle.h +++ b/src/aws-cpp-sdk-transfer/include/aws/transfer/TransferHandle.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -389,6 +390,9 @@ namespace Aws Aws::String GetChecksum() const { return m_checksum; } void SetChecksum(const Aws::String& checksum) { this->m_checksum = checksum; } + Aws::S3::Model::ChecksumAlgorithm GetChecksumAlgorithm() const { std::lock_guard locker(m_getterSetterLock); return m_checksumAlgorithm; } + void SetChecksumAlgorithm (const Aws::S3::Model::ChecksumAlgorithm& checksumAlgorithm) { std::lock_guard locker(m_getterSetterLock); m_checksumAlgorithm = checksumAlgorithm; } + private: void CleanupDownloadStream(); @@ -430,6 +434,7 @@ namespace Aws mutable std::condition_variable m_waitUntilFinishedSignal; mutable std::mutex m_getterSetterLock; Aws::String m_checksum; + Aws::S3::Model::ChecksumAlgorithm m_checksumAlgorithm; }; AWS_TRANSFER_API Aws::OStream& operator << (Aws::OStream& s, TransferStatus status); diff --git a/src/aws-cpp-sdk-transfer/include/aws/transfer/TransferManager.h b/src/aws-cpp-sdk-transfer/include/aws/transfer/TransferManager.h index a4b5580fd6e..725f14c1219 100644 --- a/src/aws-cpp-sdk-transfer/include/aws/transfer/TransferManager.h +++ b/src/aws-cpp-sdk-transfer/include/aws/transfer/TransferManager.h @@ -144,6 +144,13 @@ namespace Aws * upload. Defaults to CRC64-NVME. */ Aws::S3::Model::ChecksumAlgorithm checksumAlgorithm = S3::Model::ChecksumAlgorithm::CRC64NVME; + + /** + * Enable checksum validation for downloads. When enabled, checksums will be + * calculated during download and validated against S3 response headers. + * Defaults to true. + */ + bool validateChecksums = true; }; /** diff --git a/src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp b/src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp index 996e427e114..412117f7375 100644 --- a/src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp +++ b/src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -938,6 +939,7 @@ namespace Aws handle->SetContentType(getObjectOutcome.GetResult().GetContentType()); handle->ChangePartToCompleted(partState, getObjectOutcome.GetResult().GetETag()); getObjectOutcome.GetResult().GetBody().flush(); + handle->UpdateStatus(TransferStatus::COMPLETED); } else @@ -965,6 +967,7 @@ namespace Aws headObjectRequest.SetCustomizedAccessLogTag(m_transferConfig.customizedAccessLogTag); headObjectRequest.WithBucket(handle->GetBucketName()) .WithKey(handle->GetKey()); + headObjectRequest.SetChecksumMode(Aws::S3::Model::ChecksumMode::ENABLED); if(!handle->GetVersionId().empty()) { @@ -1000,6 +1003,18 @@ namespace Aws handle->SetContentType(headObjectOutcome.GetResult().GetContentType()); handle->SetMetadata(headObjectOutcome.GetResult().GetMetadata()); handle->SetEtag(headObjectOutcome.GetResult().GetETag()); + if (headObjectOutcome.GetResult().GetChecksumType() == Aws::S3::Model::ChecksumType::FULL_OBJECT) { + if (!headObjectOutcome.GetResult().GetChecksumCRC32C().empty()) { + handle->SetChecksum(headObjectOutcome.GetResult().GetChecksumCRC32C()); + handle->SetChecksumAlgorithm(S3::Model::ChecksumAlgorithm::CRC32C); + } else if (!headObjectOutcome.GetResult().GetChecksumCRC32().empty()) { + handle->SetChecksum(headObjectOutcome.GetResult().GetChecksumCRC32()); + handle->SetChecksumAlgorithm(S3::Model::ChecksumAlgorithm::CRC32); + } else if (!headObjectOutcome.GetResult().GetChecksumCRC64NVME().empty()) { + handle->SetChecksum(headObjectOutcome.GetResult().GetChecksumCRC64NVME()); + handle->SetChecksumAlgorithm(S3::Model::ChecksumAlgorithm::CRC64NVME); + } + } /* When bucket versioning is suspended, head object will return "null" for unversioned object. * Send following GetObject with "null" as versionId will result in 403 access denied error if your IAM role or policy * doesn't have GetObjectVersion permission. @@ -1147,6 +1162,51 @@ namespace Aws handle->UpdateStatus(DetermineIfFailedOrCanceled(*handle)); TriggerTransferStatusUpdatedCallback(handle); } + + if (handle->GetPendingParts().empty() && m_transferConfig.validateChecksums && !handle->GetChecksum().empty()) { + uint64_t combinedChecksum = 0; + for (const auto& part: handle->GetCompletedParts()) { + Aws::String checksumStr = part.second->GetChecksum(); + uint64_t partSize = part.second->GetSizeInBytes(); + if (checksumStr.empty()) { continue; } + auto decoded = Aws::Utils::HashingUtils::Base64Decode(checksumStr); + + if (combinedChecksum == 0) { + combinedChecksum = (handle->GetChecksumAlgorithm() == S3::Model::ChecksumAlgorithm::CRC64NVME)?*reinterpret_cast(decoded.GetUnderlyingData()): *reinterpret_cast(decoded.GetUnderlyingData()); + } else { + if (handle->GetChecksumAlgorithm() == S3::Model::ChecksumAlgorithm::CRC32) { + auto partCrc = *reinterpret_cast(decoded.GetUnderlyingData()); + combinedChecksum = Aws::Crt::Checksum::CombineCRC32(static_cast(combinedChecksum), partCrc, partSize); + } else if (handle->GetChecksumAlgorithm() == S3::Model::ChecksumAlgorithm::CRC32C){ + auto partCrc = *reinterpret_cast(decoded.GetUnderlyingData()); + combinedChecksum = Aws::Crt::Checksum::CombineCRC32C(static_cast(combinedChecksum), partCrc, partSize); + } else if (handle->GetChecksumAlgorithm() == S3::Model::ChecksumAlgorithm::CRC64NVME) { + auto partCrc = *reinterpret_cast(decoded.GetUnderlyingData()); + combinedChecksum = Aws::Crt::Checksum::CombineCRC64NVME(combinedChecksum, partCrc, partSize); + } + } + } + Aws::Utils::ByteBuffer checksumBuffer(handle->GetChecksumAlgorithm()== S3::Model::ChecksumAlgorithm::CRC64NVME ? 8 : 4); + if (handle->GetChecksumAlgorithm() == S3::Model::ChecksumAlgorithm::CRC64NVME) { + *reinterpret_cast(checksumBuffer.GetUnderlyingData()) = combinedChecksum; + } else { + *reinterpret_cast(checksumBuffer.GetUnderlyingData()) = static_cast(combinedChecksum); + } + Aws::String combinedChecksumStr = Aws::Utils::HashingUtils::Base64Encode(checksumBuffer); + + if (combinedChecksumStr != handle->GetChecksum()) { + AWS_LOGSTREAM_ERROR(CLASS_TAG, "Transfer handle [" << handle->GetId() + << "] Full-object checksum mismatch. Expected: " << handle->GetChecksum() + << ", Calculated: " << combinedChecksumStr); + Aws::Client::AWSError error(Aws::S3::S3Errors::INTERNAL_FAILURE, + "ChecksumMismatch", + "Full-object checksum validation failed", + false); + handle->SetError(error); + handle->UpdateStatus(TransferStatus::FAILED); + TriggerErrorCallback(handle, error); + } + } } void TransferManager::HandleGetObjectResponse(const Aws::S3::S3Client* client, @@ -1204,6 +1264,24 @@ namespace Aws Aws::String errMsg{handle->WritePartToDownloadStream(bufferStream, partState->GetRangeBegin())}; if (errMsg.empty()) { + partState->SetChecksum([&]() -> Aws::String { + if (!outcome.GetResult().GetChecksumCRC32().empty()) { + return outcome.GetResult().GetChecksumCRC32(); + } + if (!outcome.GetResult().GetChecksumCRC32C().empty()) { + return outcome.GetResult().GetChecksumCRC32C(); + } + if (!outcome.GetResult().GetChecksumCRC64NVME().empty()) { + return outcome.GetResult().GetChecksumCRC64NVME(); + } + if (!outcome.GetResult().GetChecksumSHA1().empty()) { + return outcome.GetResult().GetChecksumSHA1(); + } + if (!outcome.GetResult().GetChecksumSHA256().empty()) { + return outcome.GetResult().GetChecksumSHA256(); + } + return ""; + }()); handle->ChangePartToCompleted(partState, outcome.GetResult().GetETag()); } else { Aws::Client::AWSError error(Aws::S3::S3Errors::INTERNAL_FAILURE,