Skip to content

Add support for aws-chunked requests with STREAMING-UNSIGNED-PAYLOAD #200

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add support for aws-chunked requests with STREAMING-UNSIGNED-PAYLOAD #200

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/RequestHeadersBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,13 +193,18 @@ private void addPassthroughHeader(String headerName, List<String> headerValues)
passthroughHeadersBuilder.addAll(headerName, headerValues);
}

private String requiredContentSha256()
{
return contentSha256.orElseThrow(() -> new WebApplicationException(BAD_REQUEST));
}

private void assertContentTypeValid(ContentType actualContentType)
{
if (actualContentType == ContentType.AWS_CHUNKED || actualContentType == ContentType.AWS_CHUNKED_IN_W3C_CHUNKED || actualContentType == ContentType.W3C_CHUNKED) {
if (decodedContentLength.isEmpty()) {
throw new WebApplicationException(LENGTH_REQUIRED);
}
String sha256 = contentSha256.orElseThrow(() -> new WebApplicationException(BAD_REQUEST));
String sha256 = requiredContentSha256();
if (actualContentType != ContentType.W3C_CHUNKED && !sha256.startsWith("STREAMING-")) {
throw new WebApplicationException(BAD_REQUEST);
}
Expand All @@ -214,6 +219,12 @@ private InternalRequestHeaders build(MultiMap allHeaders)
if (!seenRequestPayloadContentTypes.containsAll(ImmutableSet.of(ContentType.AWS_CHUNKED, ContentType.W3C_CHUNKED))) {
throw new WebApplicationException(BAD_REQUEST);
}
if (requiredContentSha256().startsWith("STREAMING-UNSIGNED-PAYLOAD")) {
// Some SDKs send requests with aws-chunked content encoding, in a W3C transfer encoding
// but with an unsigned payload - meaning no chunks are signed.
// This means those requests behave more like a standard W3C Chunked request than an aws-chunked one.
yield Optional.of(ContentType.W3C_CHUNKED);
}
yield Optional.of(ContentType.AWS_CHUNKED_IN_W3C_CHUNKED);
}
default -> throw new WebApplicationException(BAD_REQUEST);
Expand Down
49 changes: 49 additions & 0 deletions trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestHttpChunked.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,8 @@ public void testHttpChunked()
{
String bucket = "test-http-chunked";
String bucketTwo = "test-http-chunked-two";
String bucketThree = "test-http-chunked-three";

storageClient.createBucket(r -> r.bucket(bucket).build());
testHttpChunked(bucket, LOREM_IPSUM, "UNSIGNED-PAYLOAD", 1);
testHttpChunked(bucket, LOREM_IPSUM, "UNSIGNED-PAYLOAD", 3);
Expand All @@ -173,6 +175,53 @@ public void testHttpChunked()
testHttpChunked(bucketTwo, LOREM_IPSUM, sha256(LOREM_IPSUM), 1);
testHttpChunked(bucketTwo, LOREM_IPSUM, sha256(LOREM_IPSUM), 3);
testHttpChunked(bucketTwo, LOREM_IPSUM, sha256(LOREM_IPSUM), 5);

storageClient.createBucket(r -> r.bucket(bucketThree).build());
testHttpChunked(bucketThree, LOREM_IPSUM, "STREAMING-UNSIGNED-PAYLOAD-TRAILER", 1);
testHttpChunked(bucketThree, LOREM_IPSUM, "STREAMING-UNSIGNED-PAYLOAD-TRAILER", 3);
testHttpChunked(bucketThree, LOREM_IPSUM, "STREAMING-UNSIGNED-PAYLOAD-TRAILER", 5);
}

@Test
public void testHttpChunkedWithAwsChunkedEncodingUnsignedPayload()
throws IOException
{
/*
Requests may be received with chunked Transfer-Encoding, and aws-chunked Content-Encoding.
If the content hash header denotes a streaming HMAC signature is required, each chunk in the body
should be signed as per the aws-chunked spec. That is covered in other tests.

However, if the content hash header denotes this is a streaming unsigned payload (STREAMING-UNSIGNED-PAYLOAD-TRAILER),
we should not expect any signature to be provided - thus making request handling for those cases
behave exactly like W3C chunked, despite there being an aws-chunked indicator.
*/

String bucket = "test-http-chunked-aws-chunked-header";
storageClient.createBucket(r -> r.bucket(bucket).build());
ImmutableMultiMap.Builder headersBuilder = ImmutableMultiMap.builder(false)
.add("X-Amz-Content-Sha256", "STREAMING-UNSIGNED-PAYLOAD-TRAILER");

assertThat(doHttpChunkedUpload(
bucket,
"basic-upload",
LOREM_IPSUM,
3,
headersBuilder.build())).isEqualTo(200);
assertThat(getFileFromStorage(storageClient, bucket, "basic-upload")).isEqualTo(LOREM_IPSUM);
assertThat(headObjectInStorage(storageClient, bucket, "basic-upload").contentEncoding()).isNullOrEmpty();

assertThat(doHttpChunkedUpload(
bucket,
"basic-upload-with-encoding",
LOREM_IPSUM,
3,
headersBuilder
.add("Content-Encoding", "aws-chunked")
.add("Content-Encoding", "gzip,compress")
.build())).isEqualTo(200);
assertThat(getFileFromStorage(storageClient, bucket, "basic-upload-with-encoding")).isEqualTo(LOREM_IPSUM);
HeadObjectResponse basicUpload = headObjectInStorage(storageClient, bucket, "basic-upload-with-encoding");
assertThat(headObjectInStorage(storageClient, bucket, "basic-upload-with-encoding").contentEncoding()).contains("gzip,compress");
}

private void testHttpChunked(String bucket, String content, String sha256, int partitionCount)
Expand Down
49 changes: 49 additions & 0 deletions trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/rest/TestRequestHeadersBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,55 @@ private void testBuildHeadersAwsChunkedPayload(MultiMap baseHeaders, ContentType
Optional.of(expectedContentType));
}

@Test
public void testBuildHeadersHttpAndAwsChunkedUnsignedPayloadBehavesLikeW3CChunked()
{
// Testing our corner case where we want to handle aws-chunked requests with a STREAMING-UNSIGNED-PAYLOAD hash
// as if they were W3C chunked, and not aws-chunked.
ImmutableMultiMap baseHeaders = ImmutableMultiMap.builder(false)
.add("Transfer-Encoding", "chunked")
.add("X-Amz-Decoded-Content-Length", "1000")
.build();

testBuildHeaders(
mergeMaps(
baseHeaders,
ImmutableMultiMap.builder(false).add("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD").build()),
ImmutableMultiMap.empty(), Optional.of(W3C_CHUNKED));

testBuildHeaders(
mergeMaps(
baseHeaders,
ImmutableMultiMap.builder(false)
.add("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
.add("Content-Encoding", "gzip,compress")
.build()),
ImmutableMultiMap.builder(false)
.add("Content-Encoding", "gzip,compress").build(),
Optional.of(W3C_CHUNKED));

testBuildHeaders(
mergeMaps(
baseHeaders,
ImmutableMultiMap.builder(false)
.add("X-Amz-Content-Sha256", "STREAMING-UNSIGNED-PAYLOAD")
.add("Content-Encoding", "aws-chunked")
.build()),
ImmutableMultiMap.empty(),
Optional.of(W3C_CHUNKED));

testBuildHeaders(
mergeMaps(
baseHeaders,
ImmutableMultiMap.builder(false)
.add("X-Amz-Content-Sha256", "STREAMING-UNSIGNED-PAYLOAD-TRAILER")
.add("Content-Encoding", "aws-chunked")
.add("Content-Encoding", "aws-chunked,gzip")
.build()),
ImmutableMultiMap.builder(false).add("Content-Encoding", "gzip").build(),
Optional.of(W3C_CHUNKED));
}

@Test
public void testBuildHeadersHttpChunked()
{
Expand Down