From d93d32d73a8438fdcf6d1cee1baa26480bf429e6 Mon Sep 17 00:00:00 2001 From: Pablo Arteaga Date: Wed, 20 Aug 2025 22:45:59 +0100 Subject: [PATCH] Add support for aws-chunked requests with STREAMING-UNSIGNED-PAYLOAD --- .../server/rest/RequestHeadersBuilder.java | 13 ++++- .../aws/proxy/server/TestHttpChunked.java | 49 +++++++++++++++++++ .../rest/TestRequestHeadersBuilder.java | 49 +++++++++++++++++++ 3 files changed, 110 insertions(+), 1 deletion(-) diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/RequestHeadersBuilder.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/RequestHeadersBuilder.java index bf907921..479cac15 100644 --- a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/RequestHeadersBuilder.java +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/RequestHeadersBuilder.java @@ -193,13 +193,18 @@ private void addPassthroughHeader(String headerName, List headerValues) passthroughHeadersBuilder.addAll(headerName, headerValues); } + private String requiredContentSha256() + { + return contentSha256.orElseThrow(() -> new WebApplicationException(BAD_REQUEST)); + } + private void assertContentTypeValid(ContentType actualContentType) { if (actualContentType == ContentType.AWS_CHUNKED || actualContentType == ContentType.AWS_CHUNKED_IN_W3C_CHUNKED || actualContentType == ContentType.W3C_CHUNKED) { if (decodedContentLength.isEmpty()) { throw new WebApplicationException(LENGTH_REQUIRED); } - String sha256 = contentSha256.orElseThrow(() -> new WebApplicationException(BAD_REQUEST)); + String sha256 = requiredContentSha256(); if (actualContentType != ContentType.W3C_CHUNKED && !sha256.startsWith("STREAMING-")) { throw new WebApplicationException(BAD_REQUEST); } @@ -214,6 +219,12 @@ private InternalRequestHeaders build(MultiMap allHeaders) if (!seenRequestPayloadContentTypes.containsAll(ImmutableSet.of(ContentType.AWS_CHUNKED, ContentType.W3C_CHUNKED))) { throw new WebApplicationException(BAD_REQUEST); } + if (requiredContentSha256().startsWith("STREAMING-UNSIGNED-PAYLOAD")) { + // Some SDKs send requests with aws-chunked content encoding, in a W3C transfer encoding + // but with an unsigned payload - meaning no chunks are signed. + // This means those requests behave more like a standard W3C Chunked request than an aws-chunked one. + yield Optional.of(ContentType.W3C_CHUNKED); + } yield Optional.of(ContentType.AWS_CHUNKED_IN_W3C_CHUNKED); } default -> throw new WebApplicationException(BAD_REQUEST); diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestHttpChunked.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestHttpChunked.java index f295c440..a54b3df5 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestHttpChunked.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestHttpChunked.java @@ -164,6 +164,8 @@ public void testHttpChunked() { String bucket = "test-http-chunked"; String bucketTwo = "test-http-chunked-two"; + String bucketThree = "test-http-chunked-three"; + storageClient.createBucket(r -> r.bucket(bucket).build()); testHttpChunked(bucket, LOREM_IPSUM, "UNSIGNED-PAYLOAD", 1); testHttpChunked(bucket, LOREM_IPSUM, "UNSIGNED-PAYLOAD", 3); @@ -173,6 +175,53 @@ public void testHttpChunked() testHttpChunked(bucketTwo, LOREM_IPSUM, sha256(LOREM_IPSUM), 1); testHttpChunked(bucketTwo, LOREM_IPSUM, sha256(LOREM_IPSUM), 3); testHttpChunked(bucketTwo, LOREM_IPSUM, sha256(LOREM_IPSUM), 5); + + storageClient.createBucket(r -> r.bucket(bucketThree).build()); + testHttpChunked(bucketThree, LOREM_IPSUM, "STREAMING-UNSIGNED-PAYLOAD-TRAILER", 1); + testHttpChunked(bucketThree, LOREM_IPSUM, "STREAMING-UNSIGNED-PAYLOAD-TRAILER", 3); + testHttpChunked(bucketThree, LOREM_IPSUM, "STREAMING-UNSIGNED-PAYLOAD-TRAILER", 5); + } + + @Test + public void testHttpChunkedWithAwsChunkedEncodingUnsignedPayload() + throws IOException + { + /* + Requests may be received with chunked Transfer-Encoding, and aws-chunked Content-Encoding. + If the content hash header denotes a streaming HMAC signature is required, each chunk in the body + should be signed as per the aws-chunked spec. That is covered in other tests. + + However, if the content hash header denotes this is a streaming unsigned payload (STREAMING-UNSIGNED-PAYLOAD-TRAILER), + we should not expect any signature to be provided - thus making request handling for those cases + behave exactly like W3C chunked, despite there being an aws-chunked indicator. + */ + + String bucket = "test-http-chunked-aws-chunked-header"; + storageClient.createBucket(r -> r.bucket(bucket).build()); + ImmutableMultiMap.Builder headersBuilder = ImmutableMultiMap.builder(false) + .add("X-Amz-Content-Sha256", "STREAMING-UNSIGNED-PAYLOAD-TRAILER"); + + assertThat(doHttpChunkedUpload( + bucket, + "basic-upload", + LOREM_IPSUM, + 3, + headersBuilder.build())).isEqualTo(200); + assertThat(getFileFromStorage(storageClient, bucket, "basic-upload")).isEqualTo(LOREM_IPSUM); + assertThat(headObjectInStorage(storageClient, bucket, "basic-upload").contentEncoding()).isNullOrEmpty(); + + assertThat(doHttpChunkedUpload( + bucket, + "basic-upload-with-encoding", + LOREM_IPSUM, + 3, + headersBuilder + .add("Content-Encoding", "aws-chunked") + .add("Content-Encoding", "gzip,compress") + .build())).isEqualTo(200); + assertThat(getFileFromStorage(storageClient, bucket, "basic-upload-with-encoding")).isEqualTo(LOREM_IPSUM); + HeadObjectResponse basicUpload = headObjectInStorage(storageClient, bucket, "basic-upload-with-encoding"); + assertThat(headObjectInStorage(storageClient, bucket, "basic-upload-with-encoding").contentEncoding()).contains("gzip,compress"); } private void testHttpChunked(String bucket, String content, String sha256, int partitionCount) diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/rest/TestRequestHeadersBuilder.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/rest/TestRequestHeadersBuilder.java index 51a65eaa..19f22061 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/rest/TestRequestHeadersBuilder.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/rest/TestRequestHeadersBuilder.java @@ -121,6 +121,55 @@ private void testBuildHeadersAwsChunkedPayload(MultiMap baseHeaders, ContentType Optional.of(expectedContentType)); } + @Test + public void testBuildHeadersHttpAndAwsChunkedUnsignedPayloadBehavesLikeW3CChunked() + { + // Testing our corner case where we want to handle aws-chunked requests with a STREAMING-UNSIGNED-PAYLOAD hash + // as if they were W3C chunked, and not aws-chunked. + ImmutableMultiMap baseHeaders = ImmutableMultiMap.builder(false) + .add("Transfer-Encoding", "chunked") + .add("X-Amz-Decoded-Content-Length", "1000") + .build(); + + testBuildHeaders( + mergeMaps( + baseHeaders, + ImmutableMultiMap.builder(false).add("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD").build()), + ImmutableMultiMap.empty(), Optional.of(W3C_CHUNKED)); + + testBuildHeaders( + mergeMaps( + baseHeaders, + ImmutableMultiMap.builder(false) + .add("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD") + .add("Content-Encoding", "gzip,compress") + .build()), + ImmutableMultiMap.builder(false) + .add("Content-Encoding", "gzip,compress").build(), + Optional.of(W3C_CHUNKED)); + + testBuildHeaders( + mergeMaps( + baseHeaders, + ImmutableMultiMap.builder(false) + .add("X-Amz-Content-Sha256", "STREAMING-UNSIGNED-PAYLOAD") + .add("Content-Encoding", "aws-chunked") + .build()), + ImmutableMultiMap.empty(), + Optional.of(W3C_CHUNKED)); + + testBuildHeaders( + mergeMaps( + baseHeaders, + ImmutableMultiMap.builder(false) + .add("X-Amz-Content-Sha256", "STREAMING-UNSIGNED-PAYLOAD-TRAILER") + .add("Content-Encoding", "aws-chunked") + .add("Content-Encoding", "aws-chunked,gzip") + .build()), + ImmutableMultiMap.builder(false).add("Content-Encoding", "gzip").build(), + Optional.of(W3C_CHUNKED)); + } + @Test public void testBuildHeadersHttpChunked() {