From 7d2f59bd9f0f163d1df728d689483544621fb9be Mon Sep 17 00:00:00 2001 From: tryangul Date: Thu, 9 Jan 2025 12:49:18 -0800 Subject: [PATCH 1/2] Use CRT http engine. --- airbyte-cdk/bulk/toolkits/load-s3/build.gradle | 3 ++- .../src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/bulk/toolkits/load-s3/build.gradle b/airbyte-cdk/bulk/toolkits/load-s3/build.gradle index e19bc337b342..c495dc5e9d0b 100644 --- a/airbyte-cdk/bulk/toolkits/load-s3/build.gradle +++ b/airbyte-cdk/bulk/toolkits/load-s3/build.gradle @@ -5,5 +5,6 @@ dependencies { api project(':airbyte-cdk:bulk:toolkits:bulk-cdk-toolkit-load-object-storage') testFixturesApi(testFixtures(project(":airbyte-cdk:bulk:toolkits:bulk-cdk-toolkit-load-object-storage"))) - implementation("aws.sdk.kotlin:s3:1.3.94") + implementation("aws.sdk.kotlin:s3:1.3.98") + implementation("aws.smithy.kotlin:http-client-engine-crt:1.3.31") } diff --git a/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt index e3d36350b9b3..689fdcb2caf3 100644 --- a/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt +++ b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt @@ -18,6 +18,7 @@ import aws.sdk.kotlin.services.s3.model.PutObjectRequest import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider import aws.smithy.kotlin.runtime.content.ByteStream import aws.smithy.kotlin.runtime.content.toInputStream +import aws.smithy.kotlin.runtime.http.engine.crt.CrtHttpEngine import aws.smithy.kotlin.runtime.net.url.Url import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import io.airbyte.cdk.load.command.aws.AWSAccessKeyConfigurationProvider @@ -244,8 +245,13 @@ class S3ClientFactory( Url.parse(it) } else null } + + // Fix for connection reset issue: + // https://github.com/awslabs/aws-sdk-kotlin/issues/1214#issuecomment-2464831817 + httpClient(CrtHttpEngine) } + return S3Client( s3SdkClient, bucketConfig.s3BucketConfiguration, From 2e2f0e73ecf8c2648b06ee81cc897ee99a2e0f61 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Thu, 9 Jan 2025 21:01:48 +0000 Subject: [PATCH 2/2] chore: auto-fix lint and format issues --- .../src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt index 689fdcb2caf3..810d671967b4 100644 --- a/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt +++ b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt @@ -251,7 +251,6 @@ class S3ClientFactory( httpClient(CrtHttpEngine) } - return S3Client( s3SdkClient, bucketConfig.s3BucketConfiguration,