From aa2e8c58e0a17d98f625802afa1fc4b5e48ff0bf Mon Sep 17 00:00:00 2001 From: tryangul Date: Wed, 8 Jan 2025 15:11:06 -0800 Subject: [PATCH 1/4] Add config to prevent connection closed errors. Bump versions. --- airbyte-cdk/bulk/toolkits/load-s3/build.gradle | 3 ++- .../src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/bulk/toolkits/load-s3/build.gradle b/airbyte-cdk/bulk/toolkits/load-s3/build.gradle index e19bc337b342..ec1667e9b2ca 100644 --- a/airbyte-cdk/bulk/toolkits/load-s3/build.gradle +++ b/airbyte-cdk/bulk/toolkits/load-s3/build.gradle @@ -5,5 +5,6 @@ dependencies { api project(':airbyte-cdk:bulk:toolkits:bulk-cdk-toolkit-load-object-storage') testFixturesApi(testFixtures(project(":airbyte-cdk:bulk:toolkits:bulk-cdk-toolkit-load-object-storage"))) - implementation("aws.sdk.kotlin:s3:1.3.94") + implementation("aws.sdk.kotlin:s3:1.3.98") + implementation("aws.smithy.kotlin:http-client-engine-okhttp:1.3.31") } diff --git a/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt index e3d36350b9b3..f9831faabcf1 100644 --- a/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt +++ b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt @@ -18,6 +18,7 @@ import aws.sdk.kotlin.services.s3.model.PutObjectRequest import aws.smithy.kotlin.runtime.auth.awscredentials.CredentialsProvider import aws.smithy.kotlin.runtime.content.ByteStream import aws.smithy.kotlin.runtime.content.toInputStream +import aws.smithy.kotlin.runtime.http.engine.okhttp.OkHttpEngine import aws.smithy.kotlin.runtime.net.url.Url import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import io.airbyte.cdk.load.command.aws.AWSAccessKeyConfigurationProvider @@ -38,6 +39,7 @@ import jakarta.inject.Singleton import java.io.ByteArrayOutputStream import java.io.InputStream import java.io.OutputStream +import kotlin.time.Duration.Companion.milliseconds import kotlinx.coroutines.flow.flow data class S3Object(override val key: String, override val storageConfig: S3BucketConfiguration) : @@ -244,6 +246,10 @@ class S3ClientFactory( Url.parse(it) } else null } + // Should fix connection reset issue: https://github.com/awslabs/aws-sdk-kotlin/issues/1214#issuecomment-2464831817 + httpClient(OkHttpEngine) { + connectionIdlePollingInterval = 200.milliseconds + } } return S3Client( From c6f381d094b6fdab21535bdfcb0b125da463db03 Mon Sep 17 00:00:00 2001 From: tryangul Date: Wed, 8 Jan 2025 15:11:18 -0800 Subject: [PATCH 2/4] Rev s3 connector RC. --- airbyte-integrations/connectors/destination-s3/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-s3/metadata.yaml b/airbyte-integrations/connectors/destination-s3/metadata.yaml index 04640037c2df..78e246bb1cf0 100644 --- a/airbyte-integrations/connectors/destination-s3/metadata.yaml +++ b/airbyte-integrations/connectors/destination-s3/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: file connectorType: destination definitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362 - dockerImageTag: 1.5.0-rc.4 + dockerImageTag: 1.5.0-rc.5 dockerRepository: airbyte/destination-s3 githubIssueLabel: destination-s3 icon: s3.svg From ec28cfd1707ead7c55d658d59a12cb0de595623f Mon Sep 17 00:00:00 2001 From: tryangul Date: Wed, 8 Jan 2025 15:28:05 -0800 Subject: [PATCH 3/4] Revert "Rev s3 connector RC." This reverts commit c6f381d094b6fdab21535bdfcb0b125da463db03. --- airbyte-integrations/connectors/destination-s3/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-s3/metadata.yaml b/airbyte-integrations/connectors/destination-s3/metadata.yaml index 78e246bb1cf0..04640037c2df 100644 --- a/airbyte-integrations/connectors/destination-s3/metadata.yaml +++ b/airbyte-integrations/connectors/destination-s3/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: file connectorType: destination definitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362 - dockerImageTag: 1.5.0-rc.5 + dockerImageTag: 1.5.0-rc.4 dockerRepository: airbyte/destination-s3 githubIssueLabel: destination-s3 icon: s3.svg From fb42d7032a1bb490f5cb41a5c639d29aa0bd311d Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Wed, 8 Jan 2025 23:36:59 +0000 Subject: [PATCH 4/4] chore: auto-fix lint and format issues --- .../main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt index f9831faabcf1..b25d45746292 100644 --- a/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt +++ b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt @@ -246,10 +246,9 @@ class S3ClientFactory( Url.parse(it) } else null } - // Should fix connection reset issue: https://github.com/awslabs/aws-sdk-kotlin/issues/1214#issuecomment-2464831817 - httpClient(OkHttpEngine) { - connectionIdlePollingInterval = 200.milliseconds - } + // Should fix connection reset issue: + // https://github.com/awslabs/aws-sdk-kotlin/issues/1214#issuecomment-2464831817 + httpClient(OkHttpEngine) { connectionIdlePollingInterval = 200.milliseconds } } return S3Client(