From 5c7a79e5e86205103081810aaaf280ee6222dcb8 Mon Sep 17 00:00:00 2001 From: Francis Genet Date: Thu, 9 Jan 2025 09:17:25 -0800 Subject: [PATCH 1/9] Remove/Rename all occurences of iceberg.v2 --- .../v2 => s3_data_lake}/io/BaseDeltaTaskWriter.java | 2 +- .../v2 => s3_data_lake}/GlueCredentialsProvider.kt | 2 +- .../{iceberg/v2 => s3_data_lake}/IcebergDestination.kt | 2 +- .../v2 => s3_data_lake}/IcebergStreamLoader.kt | 2 +- .../{iceberg/v2 => s3_data_lake}/IcebergV2Checker.kt | 2 +- .../v2 => s3_data_lake}/IcebergV2Configuration.kt | 2 +- .../v2 => s3_data_lake}/IcebergV2Specification.kt | 2 +- .../{iceberg/v2 => s3_data_lake}/IcebergV2Writer.kt | 2 +- .../{iceberg/v2 => s3_data_lake}/TableIdGenerator.kt | 2 +- .../v2 => s3_data_lake}/io/IcebergTableCleaner.kt | 2 +- .../io/IcebergTableWriterFactory.kt | 6 +++--- .../{iceberg/v2 => s3_data_lake}/io/IcebergUtil.kt | 2 +- .../{iceberg/v2 => s3_data_lake}/io/Operation.kt | 2 +- .../v2 => s3_data_lake}/io/PartitionedWriters.kt | 4 ++-- .../{iceberg/v2 => s3_data_lake}/io/RecordWrapper.kt | 2 +- .../v2 => s3_data_lake}/io/UnpartitionedWriters.kt | 6 +++--- .../v2 => s3_data_lake}/IcebergDestinationCleaner.kt | 4 ++-- .../v2 => s3_data_lake}/IcebergExpectedRecordMapper.kt | 2 +- .../{iceberg/v2 => s3_data_lake}/IcebergV2CheckTest.kt | 2 +- .../v2 => s3_data_lake}/IcebergV2DataDumper.kt | 2 +- .../{iceberg/v2 => s3_data_lake}/IcebergV2SpecTest.kt | 2 +- .../{iceberg/v2 => s3_data_lake}/IcebergV2TestUtil.kt | 2 +- .../{iceberg/v2 => s3_data_lake}/IcebergV2WriteTest.kt | 6 +++--- .../v2 => s3_data_lake}/NessieTestContainers.kt | 2 +- .../v2 => s3_data_lake}/IcebergV2WriterTest.kt | 2 +- .../v2 => s3_data_lake}/io/BaseDeltaTaskWriterTest.kt | 10 +++++----- .../v2 => s3_data_lake}/io/IcebergTableCleanerTest.kt | 2 +- .../io/IcebergTableWriterFactoryTest.kt | 6 +++--- .../{iceberg/v2 => s3_data_lake}/io/IcebergUtilTest.kt | 2 +- 29 files changed, 43 insertions(+), 43 deletions(-) rename airbyte-integrations/connectors/destination-s3-data-lake/src/main/java/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/io/BaseDeltaTaskWriter.java (98%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/GlueCredentialsProvider.kt (95%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/IcebergDestination.kt (81%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/IcebergStreamLoader.kt (98%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/IcebergV2Checker.kt (96%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/IcebergV2Configuration.kt (97%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/IcebergV2Specification.kt (98%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/IcebergV2Writer.kt (98%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/TableIdGenerator.kt (96%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/io/IcebergTableCleaner.kt (97%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/io/IcebergTableWriterFactory.kt (96%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/io/IcebergUtil.kt (99%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/io/Operation.kt (72%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/io/PartitionedWriters.kt (95%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/io/RecordWrapper.kt (86%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/io/UnpartitionedWriters.kt (86%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/IcebergDestinationCleaner.kt (87%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/IcebergExpectedRecordMapper.kt (95%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/IcebergV2CheckTest.kt (91%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/IcebergV2DataDumper.kt (98%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/IcebergV2SpecTest.kt (71%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/IcebergV2TestUtil.kt (96%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/IcebergV2WriteTest.kt (96%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/NessieTestContainers.kt (96%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/IcebergV2WriterTest.kt (99%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/io/BaseDeltaTaskWriterTest.kt (92%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/io/IcebergTableCleanerTest.kt (98%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/io/IcebergTableWriterFactoryTest.kt (97%) rename airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/{iceberg/v2 => s3_data_lake}/io/IcebergUtilTest.kt (99%) diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/java/io/airbyte/integrations/destination/iceberg/v2/io/BaseDeltaTaskWriter.java b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/java/io/airbyte/integrations/destination/s3_data_lake/io/BaseDeltaTaskWriter.java similarity index 98% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/main/java/io/airbyte/integrations/destination/iceberg/v2/io/BaseDeltaTaskWriter.java rename to airbyte-integrations/connectors/destination-s3-data-lake/src/main/java/io/airbyte/integrations/destination/s3_data_lake/io/BaseDeltaTaskWriter.java index 6534e9d62051..258dd6e58e2c 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/java/io/airbyte/integrations/destination/iceberg/v2/io/BaseDeltaTaskWriter.java +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/java/io/airbyte/integrations/destination/s3_data_lake/io/BaseDeltaTaskWriter.java @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2.io; +package io.airbyte.integrations.destination.s3_data_lake.io; import java.io.IOException; import java.util.Set; diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/GlueCredentialsProvider.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/GlueCredentialsProvider.kt similarity index 95% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/GlueCredentialsProvider.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/GlueCredentialsProvider.kt index a3a993b01e71..1e57ffced0f5 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/GlueCredentialsProvider.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/GlueCredentialsProvider.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2 +package io.airbyte.integrations.destination.s3_data_lake import software.amazon.awssdk.auth.credentials.AwsBasicCredentials import software.amazon.awssdk.auth.credentials.AwsCredentials diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergDestination.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergDestination.kt similarity index 81% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergDestination.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergDestination.kt index b4256e621cfb..7f81a3854e6d 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergDestination.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergDestination.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2 +package io.airbyte.integrations.destination.s3_data_lake import io.airbyte.cdk.AirbyteDestinationRunner diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergStreamLoader.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergStreamLoader.kt similarity index 98% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergStreamLoader.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergStreamLoader.kt index 7f20f5b9f55d..d400c304f482 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergStreamLoader.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergStreamLoader.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2 +package io.airbyte.integrations.destination.s3_data_lake import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import io.airbyte.cdk.load.command.DestinationStream diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2Checker.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Checker.kt similarity index 96% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2Checker.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Checker.kt index d790c27560cd..9443e90eabb9 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2Checker.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Checker.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2 +package io.airbyte.integrations.destination.s3_data_lake import io.airbyte.cdk.load.check.DestinationChecker import io.airbyte.cdk.load.command.DestinationStream diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2Configuration.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Configuration.kt similarity index 97% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2Configuration.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Configuration.kt index 7ef6e32491a8..1da1303a93c7 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2Configuration.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Configuration.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2 +package io.airbyte.integrations.destination.s3_data_lake import io.airbyte.cdk.load.command.DestinationConfiguration import io.airbyte.cdk.load.command.DestinationConfigurationFactory diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2Specification.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Specification.kt similarity index 98% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2Specification.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Specification.kt index 80d6e7e13fc0..701107af4634 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2Specification.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Specification.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2 +package io.airbyte.integrations.destination.s3_data_lake import com.fasterxml.jackson.annotation.JsonPropertyDescription import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2Writer.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Writer.kt similarity index 98% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2Writer.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Writer.kt index 20a83f29937b..3285277c0446 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2Writer.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Writer.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2 +package io.airbyte.integrations.destination.s3_data_lake import io.airbyte.cdk.load.command.DestinationStream import io.airbyte.cdk.load.data.iceberg.parquet.IcebergParquetPipelineFactory diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/TableIdGenerator.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/TableIdGenerator.kt similarity index 96% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/TableIdGenerator.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/TableIdGenerator.kt index c52dfd06f44a..30e27df54f30 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/TableIdGenerator.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/TableIdGenerator.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2 +package io.airbyte.integrations.destination.s3_data_lake import io.airbyte.cdk.load.command.DestinationStream import io.airbyte.cdk.load.command.iceberg.parquet.GlueCatalogConfiguration diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableCleaner.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableCleaner.kt similarity index 97% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableCleaner.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableCleaner.kt index e14ffcfaffeb..4507f280f763 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableCleaner.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableCleaner.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2.io +package io.airbyte.integrations.destination.s3_data_lake.io import jakarta.inject.Singleton import org.apache.iceberg.Table diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableWriterFactory.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableWriterFactory.kt similarity index 96% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableWriterFactory.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableWriterFactory.kt index 3aa7d110187b..77735b219ee8 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableWriterFactory.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableWriterFactory.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2.io +package io.airbyte.integrations.destination.s3_data_lake.io import io.airbyte.cdk.load.command.Append import io.airbyte.cdk.load.command.Dedupe @@ -117,7 +117,7 @@ class IcebergTableWriterFactory(private val icebergUtil: IcebergUtil) { targetFileSize: Long ): BaseTaskWriter { return if (table.spec().isUnpartitioned) { - UnpartitionedAppendWriter( + io.airbyte.integrations.destination.s3_data_lake.io.UnpartitionedAppendWriter( spec = table.spec(), format = format, appenderFactory = appenderFactory, @@ -147,7 +147,7 @@ class IcebergTableWriterFactory(private val icebergUtil: IcebergUtil) { identifierFieldIds: Set ): BaseTaskWriter { return if (table.spec().isUnpartitioned) { - UnpartitionedDeltaWriter( + io.airbyte.integrations.destination.s3_data_lake.io.UnpartitionedDeltaWriter( spec = table.spec(), format = format, appenderFactory = appenderFactory, diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergUtil.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergUtil.kt similarity index 99% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergUtil.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergUtil.kt index 7269614f1855..bbc97e340fb3 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergUtil.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergUtil.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2.io +package io.airbyte.integrations.destination.s3_data_lake.io import com.fasterxml.jackson.annotation.JsonProperty import io.airbyte.cdk.load.command.Dedupe diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/Operation.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/Operation.kt similarity index 72% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/Operation.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/Operation.kt index 46056c02d40d..76d466213c6d 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/Operation.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/Operation.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2.io +package io.airbyte.integrations.destination.s3_data_lake.io /** Delta operations for data. */ enum class Operation { diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/PartitionedWriters.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/PartitionedWriters.kt similarity index 95% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/PartitionedWriters.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/PartitionedWriters.kt index 1e0738489bf6..6b3c07ce135f 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/PartitionedWriters.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/PartitionedWriters.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2.io +package io.airbyte.integrations.destination.s3_data_lake.io import java.io.IOException import java.io.UncheckedIOException @@ -32,7 +32,7 @@ class PartitionedDeltaWriter( schema: Schema, identifierFieldIds: Set, ) : - BaseDeltaTaskWriter( + io.airbyte.integrations.destination.s3_data_lake.io.BaseDeltaTaskWriter( spec, format, appenderFactory, diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/RecordWrapper.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/RecordWrapper.kt similarity index 86% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/RecordWrapper.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/RecordWrapper.kt index 8bae7e714844..44a72a0c2de0 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/RecordWrapper.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/RecordWrapper.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2.io +package io.airbyte.integrations.destination.s3_data_lake.io import org.apache.iceberg.data.Record diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/UnpartitionedWriters.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/UnpartitionedWriters.kt similarity index 86% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/UnpartitionedWriters.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/UnpartitionedWriters.kt index 527d8ee53740..095420bf9b8b 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/UnpartitionedWriters.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/UnpartitionedWriters.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2.io +package io.airbyte.integrations.destination.s3_data_lake.io import org.apache.iceberg.FileFormat import org.apache.iceberg.PartitionSpec @@ -27,7 +27,7 @@ class UnpartitionedDeltaWriter( schema: Schema, identifierFieldIds: Set, ) : - BaseDeltaTaskWriter( + io.airbyte.integrations.destination.s3_data_lake.io.BaseDeltaTaskWriter( spec, format, appenderFactory, @@ -40,7 +40,7 @@ class UnpartitionedDeltaWriter( private val writer = RowDataDeltaWriter(null) - override fun route(row: Record): RowDataDeltaWriter { + override fun route(row: Record): io.airbyte.integrations.destination.s3_data_lake.io.BaseDeltaTaskWriter.RowDataDeltaWriter { return writer } diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergDestinationCleaner.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergDestinationCleaner.kt similarity index 87% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergDestinationCleaner.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergDestinationCleaner.kt index 447abd82fcba..6e5004768fa9 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergDestinationCleaner.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergDestinationCleaner.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2 +package io.airbyte.integrations.destination.s3_data_lake import io.airbyte.cdk.load.test.util.DestinationCleaner import io.airbyte.cdk.load.test.util.IntegrationTest.Companion.isNamespaceOld @@ -22,7 +22,7 @@ class IcebergDestinationCleaner(private val catalog: Catalog) : DestinationClean } // we're passing explicit TableIdentifier to clearTable, so just use SimpleTableIdGenerator - val tableCleaner = IcebergTableCleaner(IcebergUtil(SimpleTableIdGenerator())) + val tableCleaner = IcebergTableCleaner(IcebergUtil(io.airbyte.integrations.destination.s3_data_lake.SimpleTableIdGenerator())) namespaces.forEach { namespace -> catalog.listTables(namespace).forEach { tableId -> diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergExpectedRecordMapper.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergExpectedRecordMapper.kt similarity index 95% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergExpectedRecordMapper.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergExpectedRecordMapper.kt index 418c1e9d55fc..833d6adac735 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergExpectedRecordMapper.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergExpectedRecordMapper.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2 +package io.airbyte.integrations.destination.s3_data_lake import io.airbyte.cdk.load.data.AirbyteType import io.airbyte.cdk.load.data.AirbyteValue diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2CheckTest.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2CheckTest.kt similarity index 91% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2CheckTest.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2CheckTest.kt index 420bab9ef5a4..6056a1b6f0cb 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2CheckTest.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2CheckTest.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2 +package io.airbyte.integrations.destination.s3_data_lake import io.airbyte.cdk.load.check.CheckIntegrationTest import io.airbyte.cdk.load.check.CheckTestConfig diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2DataDumper.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2DataDumper.kt similarity index 98% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2DataDumper.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2DataDumper.kt index bdedea419fa5..ffe5df693df6 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2DataDumper.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2DataDumper.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2 +package io.airbyte.integrations.destination.s3_data_lake import io.airbyte.cdk.command.ConfigurationSpecification import io.airbyte.cdk.load.command.DestinationStream diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2SpecTest.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2SpecTest.kt similarity index 71% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2SpecTest.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2SpecTest.kt index 6e1af0089c81..91b1c641ce1f 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2SpecTest.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2SpecTest.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2 +package io.airbyte.integrations.destination.s3_data_lake import io.airbyte.cdk.load.spec.SpecTest diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2TestUtil.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2TestUtil.kt similarity index 96% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2TestUtil.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2TestUtil.kt index ad0fbe4b2157..601c89c7887d 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2TestUtil.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2TestUtil.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2 +package io.airbyte.integrations.destination.s3_data_lake import com.fasterxml.jackson.core.type.TypeReference import io.airbyte.cdk.command.ConfigurationSpecification diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2WriteTest.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2WriteTest.kt similarity index 96% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2WriteTest.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2WriteTest.kt index 2fdfd0609574..dd813f16f188 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2WriteTest.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2WriteTest.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2 +package io.airbyte.integrations.destination.s3_data_lake import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper @@ -86,7 +86,7 @@ abstract class IcebergV2WriteTest( class IcebergGlueWriteTest : IcebergV2WriteTest( Files.readString(IcebergV2TestUtil.GLUE_CONFIG_PATH), - IcebergDestinationCleaner( + io.airbyte.integrations.destination.s3_data_lake.IcebergDestinationCleaner( IcebergV2TestUtil.getCatalog( IcebergV2TestUtil.parseConfig(IcebergV2TestUtil.GLUE_CONFIG_PATH), IcebergV2TestUtil.getAWSSystemCredentials() @@ -97,7 +97,7 @@ class IcebergGlueWriteTest : class IcebergGlueAssumeRoleWriteTest : IcebergV2WriteTest( Files.readString(IcebergV2TestUtil.GLUE_ASSUME_ROLE_CONFIG_PATH), - IcebergDestinationCleaner( + io.airbyte.integrations.destination.s3_data_lake.IcebergDestinationCleaner( IcebergV2TestUtil.getCatalog( IcebergV2TestUtil.parseConfig(IcebergV2TestUtil.GLUE_ASSUME_ROLE_CONFIG_PATH), IcebergV2TestUtil.getAWSSystemCredentials() diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/NessieTestContainers.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/NessieTestContainers.kt similarity index 96% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/NessieTestContainers.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/NessieTestContainers.kt index 70528944948a..ec83f09c16ac 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/iceberg/v2/NessieTestContainers.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/NessieTestContainers.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2 +package io.airbyte.integrations.destination.s3_data_lake import io.airbyte.cdk.load.util.setOnce import java.io.File diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2WriterTest.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2WriterTest.kt similarity index 99% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2WriterTest.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2WriterTest.kt index 8ae1dacaf18f..01a9c0779865 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/iceberg/v2/IcebergV2WriterTest.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2WriterTest.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2 +package io.airbyte.integrations.destination.s3_data_lake import io.airbyte.cdk.load.command.Append import io.airbyte.cdk.load.command.Dedupe diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/BaseDeltaTaskWriterTest.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/BaseDeltaTaskWriterTest.kt similarity index 92% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/BaseDeltaTaskWriterTest.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/BaseDeltaTaskWriterTest.kt index 1e8228d048d9..bf1e4c981b30 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/BaseDeltaTaskWriterTest.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/BaseDeltaTaskWriterTest.kt @@ -2,9 +2,9 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2.io +package io.airbyte.integrations.destination.s3_data_lake.io -import io.airbyte.integrations.destination.iceberg.v2.io.BaseDeltaTaskWriter.RowDataDeltaWriter +import io.airbyte.integrations.destination.s3_data_lake.io.BaseDeltaTaskWriter.RowDataDeltaWriter import io.mockk.every import io.mockk.mockk import io.mockk.verify @@ -48,7 +48,7 @@ internal class BaseDeltaTaskWriterTest { } val record = RecordWrapper(delegate = mockk(), operation = operation) every { record.delegate.getField(any()) } returns mockk() - val writer: BaseDeltaTaskWriter = + val writer: io.airbyte.integrations.destination.s3_data_lake.io.BaseDeltaTaskWriter = TestDeltaWriter( spec = spec, format = format, @@ -97,7 +97,7 @@ internal class BaseDeltaTaskWriterTest { every { write(any()) } returns Unit } val record: Record = mockk() - val writer: BaseDeltaTaskWriter = + val writer: io.airbyte.integrations.destination.s3_data_lake.io.BaseDeltaTaskWriter = TestDeltaWriter( spec = spec, format = format, @@ -125,7 +125,7 @@ internal class BaseDeltaTaskWriterTest { primaryKeyIds: Set, val deltaWriter: RowDataDeltaWriter, ) : - BaseDeltaTaskWriter( + io.airbyte.integrations.destination.s3_data_lake.io.BaseDeltaTaskWriter( spec, format, appenderFactory, diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableCleanerTest.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableCleanerTest.kt similarity index 98% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableCleanerTest.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableCleanerTest.kt index b7e8b1e6b864..6db980c0b3c0 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableCleanerTest.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableCleanerTest.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2.io +package io.airbyte.integrations.destination.s3_data_lake.io import io.mockk.Runs import io.mockk.called diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableWriterFactoryTest.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableWriterFactoryTest.kt similarity index 97% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableWriterFactoryTest.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableWriterFactoryTest.kt index 8266c627d5a3..7551847b37d5 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergTableWriterFactoryTest.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableWriterFactoryTest.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2.io +package io.airbyte.integrations.destination.s3_data_lake.io import io.airbyte.cdk.load.command.Append import io.airbyte.cdk.load.command.Dedupe @@ -141,7 +141,7 @@ internal class IcebergTableWriterFactoryTest { ) ) assertNotNull(writer) - assertEquals(UnpartitionedDeltaWriter::class.java, writer.javaClass) + assertEquals(io.airbyte.integrations.destination.s3_data_lake.io.UnpartitionedDeltaWriter::class.java, writer.javaClass) } @Test @@ -247,6 +247,6 @@ internal class IcebergTableWriterFactoryTest { importType = Append ) assertNotNull(writer) - assertEquals(UnpartitionedAppendWriter::class.java, writer.javaClass) + assertEquals(io.airbyte.integrations.destination.s3_data_lake.io.UnpartitionedAppendWriter::class.java, writer.javaClass) } } diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergUtilTest.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergUtilTest.kt similarity index 99% rename from airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergUtilTest.kt rename to airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergUtilTest.kt index ffda15b3911e..3ee3d8751669 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/iceberg/v2/io/IcebergUtilTest.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergUtilTest.kt @@ -2,7 +2,7 @@ * Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.iceberg.v2.io +package io.airbyte.integrations.destination.s3_data_lake.io import io.airbyte.cdk.load.command.Append import io.airbyte.cdk.load.command.Dedupe From 62c1beae604b3524b3b2bce5ed2af2b0a929c087 Mon Sep 17 00:00:00 2001 From: Francis Genet Date: Thu, 9 Jan 2025 09:27:17 -0800 Subject: [PATCH 2/9] Some weird changes? --- .../destination/s3_data_lake/io/UnpartitionedWriters.kt | 4 ++-- .../destination/s3_data_lake/IcebergDestinationCleaner.kt | 6 +++--- .../destination/s3_data_lake/IcebergV2WriteTest.kt | 4 ++-- .../destination/s3_data_lake/io/BaseDeltaTaskWriterTest.kt | 6 +++--- .../s3_data_lake/io/IcebergTableWriterFactoryTest.kt | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/UnpartitionedWriters.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/UnpartitionedWriters.kt index 095420bf9b8b..cfaf28f8817d 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/UnpartitionedWriters.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/UnpartitionedWriters.kt @@ -27,7 +27,7 @@ class UnpartitionedDeltaWriter( schema: Schema, identifierFieldIds: Set, ) : - io.airbyte.integrations.destination.s3_data_lake.io.BaseDeltaTaskWriter( + BaseDeltaTaskWriter( spec, format, appenderFactory, @@ -40,7 +40,7 @@ class UnpartitionedDeltaWriter( private val writer = RowDataDeltaWriter(null) - override fun route(row: Record): io.airbyte.integrations.destination.s3_data_lake.io.BaseDeltaTaskWriter.RowDataDeltaWriter { + override fun route(row: Record): BaseDeltaTaskWriter.RowDataDeltaWriter { return writer } diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergDestinationCleaner.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergDestinationCleaner.kt index 6e5004768fa9..ab4c4f1c989f 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergDestinationCleaner.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergDestinationCleaner.kt @@ -7,8 +7,8 @@ package io.airbyte.integrations.destination.s3_data_lake import io.airbyte.cdk.load.test.util.DestinationCleaner import io.airbyte.cdk.load.test.util.IntegrationTest.Companion.isNamespaceOld import io.airbyte.cdk.load.test.util.IntegrationTest.Companion.randomizedNamespaceRegex -import io.airbyte.integrations.destination.iceberg.v2.io.IcebergTableCleaner -import io.airbyte.integrations.destination.iceberg.v2.io.IcebergUtil +import io.airbyte.integrations.destination.s3_data_lake.io.IcebergTableCleaner +import io.airbyte.integrations.destination.s3_data_lake.io.IcebergUtil import org.apache.iceberg.catalog.Catalog import org.apache.iceberg.catalog.Namespace import org.apache.iceberg.catalog.SupportsNamespaces @@ -22,7 +22,7 @@ class IcebergDestinationCleaner(private val catalog: Catalog) : DestinationClean } // we're passing explicit TableIdentifier to clearTable, so just use SimpleTableIdGenerator - val tableCleaner = IcebergTableCleaner(IcebergUtil(io.airbyte.integrations.destination.s3_data_lake.SimpleTableIdGenerator())) + val tableCleaner = IcebergTableCleaner(IcebergUtil(SimpleTableIdGenerator())) namespaces.forEach { namespace -> catalog.listTables(namespace).forEach { tableId -> diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2WriteTest.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2WriteTest.kt index dd813f16f188..46f6acd121c1 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2WriteTest.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2WriteTest.kt @@ -86,7 +86,7 @@ abstract class IcebergV2WriteTest( class IcebergGlueWriteTest : IcebergV2WriteTest( Files.readString(IcebergV2TestUtil.GLUE_CONFIG_PATH), - io.airbyte.integrations.destination.s3_data_lake.IcebergDestinationCleaner( + IcebergDestinationCleaner( IcebergV2TestUtil.getCatalog( IcebergV2TestUtil.parseConfig(IcebergV2TestUtil.GLUE_CONFIG_PATH), IcebergV2TestUtil.getAWSSystemCredentials() @@ -97,7 +97,7 @@ class IcebergGlueWriteTest : class IcebergGlueAssumeRoleWriteTest : IcebergV2WriteTest( Files.readString(IcebergV2TestUtil.GLUE_ASSUME_ROLE_CONFIG_PATH), - io.airbyte.integrations.destination.s3_data_lake.IcebergDestinationCleaner( + IcebergDestinationCleaner( IcebergV2TestUtil.getCatalog( IcebergV2TestUtil.parseConfig(IcebergV2TestUtil.GLUE_ASSUME_ROLE_CONFIG_PATH), IcebergV2TestUtil.getAWSSystemCredentials() diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/BaseDeltaTaskWriterTest.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/BaseDeltaTaskWriterTest.kt index bf1e4c981b30..1b57f511a7fb 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/BaseDeltaTaskWriterTest.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/BaseDeltaTaskWriterTest.kt @@ -48,7 +48,7 @@ internal class BaseDeltaTaskWriterTest { } val record = RecordWrapper(delegate = mockk(), operation = operation) every { record.delegate.getField(any()) } returns mockk() - val writer: io.airbyte.integrations.destination.s3_data_lake.io.BaseDeltaTaskWriter = + val writer: BaseDeltaTaskWriter = TestDeltaWriter( spec = spec, format = format, @@ -97,7 +97,7 @@ internal class BaseDeltaTaskWriterTest { every { write(any()) } returns Unit } val record: Record = mockk() - val writer: io.airbyte.integrations.destination.s3_data_lake.io.BaseDeltaTaskWriter = + val writer: BaseDeltaTaskWriter = TestDeltaWriter( spec = spec, format = format, @@ -125,7 +125,7 @@ internal class BaseDeltaTaskWriterTest { primaryKeyIds: Set, val deltaWriter: RowDataDeltaWriter, ) : - io.airbyte.integrations.destination.s3_data_lake.io.BaseDeltaTaskWriter( + BaseDeltaTaskWriter( spec, format, appenderFactory, diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableWriterFactoryTest.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableWriterFactoryTest.kt index 7551847b37d5..0917a0274e93 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableWriterFactoryTest.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableWriterFactoryTest.kt @@ -141,7 +141,7 @@ internal class IcebergTableWriterFactoryTest { ) ) assertNotNull(writer) - assertEquals(io.airbyte.integrations.destination.s3_data_lake.io.UnpartitionedDeltaWriter::class.java, writer.javaClass) + assertEquals(UnpartitionedDeltaWriter::class.java, writer.javaClass) } @Test @@ -247,6 +247,6 @@ internal class IcebergTableWriterFactoryTest { importType = Append ) assertNotNull(writer) - assertEquals(io.airbyte.integrations.destination.s3_data_lake.io.UnpartitionedAppendWriter::class.java, writer.javaClass) + assertEquals(UnpartitionedAppendWriter::class.java, writer.javaClass) } } From 3135b6d8c763438a5305d9895585a379fa2158ca Mon Sep 17 00:00:00 2001 From: Francis Genet Date: Thu, 9 Jan 2025 09:28:41 -0800 Subject: [PATCH 3/9] Forgot some I guess --- .../destination/s3_data_lake/io/IcebergTableWriterFactory.kt | 4 ++-- .../destination/s3_data_lake/io/PartitionedWriters.kt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableWriterFactory.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableWriterFactory.kt index 77735b219ee8..8bfe0a333197 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableWriterFactory.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergTableWriterFactory.kt @@ -117,7 +117,7 @@ class IcebergTableWriterFactory(private val icebergUtil: IcebergUtil) { targetFileSize: Long ): BaseTaskWriter { return if (table.spec().isUnpartitioned) { - io.airbyte.integrations.destination.s3_data_lake.io.UnpartitionedAppendWriter( + UnpartitionedAppendWriter( spec = table.spec(), format = format, appenderFactory = appenderFactory, @@ -147,7 +147,7 @@ class IcebergTableWriterFactory(private val icebergUtil: IcebergUtil) { identifierFieldIds: Set ): BaseTaskWriter { return if (table.spec().isUnpartitioned) { - io.airbyte.integrations.destination.s3_data_lake.io.UnpartitionedDeltaWriter( + UnpartitionedDeltaWriter( spec = table.spec(), format = format, appenderFactory = appenderFactory, diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/PartitionedWriters.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/PartitionedWriters.kt index 6b3c07ce135f..558e91820972 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/PartitionedWriters.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/PartitionedWriters.kt @@ -32,7 +32,7 @@ class PartitionedDeltaWriter( schema: Schema, identifierFieldIds: Set, ) : - io.airbyte.integrations.destination.s3_data_lake.io.BaseDeltaTaskWriter( + BaseDeltaTaskWriter( spec, format, appenderFactory, From 513d4d7ed7a70f3ee335baf2c71d3bb1f897ae19 Mon Sep 17 00:00:00 2001 From: Francis Genet Date: Thu, 9 Jan 2025 09:34:02 -0800 Subject: [PATCH 4/9] And update the ones that were not caught by the rename action --- .../destination/s3_data_lake/IcebergStreamLoader.kt | 6 +++--- .../destination/s3_data_lake/IcebergV2Checker.kt | 4 ++-- .../destination/s3_data_lake/IcebergV2Writer.kt | 4 ++-- .../destination/s3_data_lake/io/IcebergUtil.kt | 10 +++++----- .../destination/s3_data_lake/IcebergV2CheckTest.kt | 2 +- .../destination/s3_data_lake/IcebergV2TestUtil.kt | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergStreamLoader.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergStreamLoader.kt index d400c304f482..38840d29c1c6 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergStreamLoader.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergStreamLoader.kt @@ -12,9 +12,9 @@ import io.airbyte.cdk.load.message.DestinationRecordAirbyteValue import io.airbyte.cdk.load.message.SimpleBatch import io.airbyte.cdk.load.state.StreamProcessingFailed import io.airbyte.cdk.load.write.StreamLoader -import io.airbyte.integrations.destination.iceberg.v2.io.IcebergTableCleaner -import io.airbyte.integrations.destination.iceberg.v2.io.IcebergTableWriterFactory -import io.airbyte.integrations.destination.iceberg.v2.io.IcebergUtil +import io.airbyte.integrations.destination.s3_data_lake.io.IcebergTableCleaner +import io.airbyte.integrations.destination.s3_data_lake.io.IcebergTableWriterFactory +import io.airbyte.integrations.destination.s3_data_lake.io.IcebergUtil import io.github.oshai.kotlinlogging.KotlinLogging import org.apache.iceberg.Table diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Checker.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Checker.kt index 9443e90eabb9..aed868463982 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Checker.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Checker.kt @@ -6,8 +6,8 @@ package io.airbyte.integrations.destination.s3_data_lake import io.airbyte.cdk.load.check.DestinationChecker import io.airbyte.cdk.load.command.DestinationStream -import io.airbyte.integrations.destination.iceberg.v2.io.IcebergTableCleaner -import io.airbyte.integrations.destination.iceberg.v2.io.IcebergUtil +import io.airbyte.integrations.destination.s3_data_lake.io.IcebergTableCleaner +import io.airbyte.integrations.destination.s3_data_lake.io.IcebergUtil import javax.inject.Singleton import org.apache.iceberg.Schema import org.apache.iceberg.types.Types diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Writer.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Writer.kt index 3285277c0446..e4f82a0f5664 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Writer.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2Writer.kt @@ -8,8 +8,8 @@ import io.airbyte.cdk.load.command.DestinationStream import io.airbyte.cdk.load.data.iceberg.parquet.IcebergParquetPipelineFactory import io.airbyte.cdk.load.write.DestinationWriter import io.airbyte.cdk.load.write.StreamLoader -import io.airbyte.integrations.destination.iceberg.v2.io.IcebergTableWriterFactory -import io.airbyte.integrations.destination.iceberg.v2.io.IcebergUtil +import io.airbyte.integrations.destination.s3_data_lake.io.IcebergTableWriterFactory +import io.airbyte.integrations.destination.s3_data_lake.io.IcebergUtil import javax.inject.Singleton import org.apache.iceberg.Schema diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergUtil.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergUtil.kt index bbc97e340fb3..e8901207844e 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergUtil.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergUtil.kt @@ -18,11 +18,11 @@ import io.airbyte.cdk.load.data.iceberg.parquet.toIcebergRecord import io.airbyte.cdk.load.data.iceberg.parquet.toIcebergSchema import io.airbyte.cdk.load.data.withAirbyteMeta import io.airbyte.cdk.load.message.DestinationRecordAirbyteValue -import io.airbyte.integrations.destination.iceberg.v2.ACCESS_KEY_ID -import io.airbyte.integrations.destination.iceberg.v2.GlueCredentialsProvider -import io.airbyte.integrations.destination.iceberg.v2.IcebergV2Configuration -import io.airbyte.integrations.destination.iceberg.v2.SECRET_ACCESS_KEY -import io.airbyte.integrations.destination.iceberg.v2.TableIdGenerator +import io.airbyte.integrations.destination.s3_data_lake.ACCESS_KEY_ID +import io.airbyte.integrations.destination.s3_data_lake.GlueCredentialsProvider +import io.airbyte.integrations.destination.s3_data_lake.IcebergV2Configuration +import io.airbyte.integrations.destination.s3_data_lake.SECRET_ACCESS_KEY +import io.airbyte.integrations.destination.s3_data_lake.TableIdGenerator import io.github.oshai.kotlinlogging.KotlinLogging import jakarta.inject.Singleton import java.time.Duration diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2CheckTest.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2CheckTest.kt index 6056a1b6f0cb..080ad38bdb0d 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2CheckTest.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2CheckTest.kt @@ -6,7 +6,7 @@ package io.airbyte.integrations.destination.s3_data_lake import io.airbyte.cdk.load.check.CheckIntegrationTest import io.airbyte.cdk.load.check.CheckTestConfig -import io.airbyte.integrations.destination.iceberg.v2.IcebergV2TestUtil.GLUE_CONFIG_PATH +import io.airbyte.integrations.destination.s3_data_lake.IcebergV2TestUtil.GLUE_CONFIG_PATH class IcebergV2CheckTest : CheckIntegrationTest( diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2TestUtil.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2TestUtil.kt index 601c89c7887d..588a10d231c3 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2TestUtil.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2TestUtil.kt @@ -8,8 +8,8 @@ import com.fasterxml.jackson.core.type.TypeReference import io.airbyte.cdk.command.ConfigurationSpecification import io.airbyte.cdk.command.ValidatedJsonUtils import io.airbyte.cdk.load.util.Jsons -import io.airbyte.integrations.destination.iceberg.v2.io.AWSSystemCredentials -import io.airbyte.integrations.destination.iceberg.v2.io.IcebergUtil +import io.airbyte.integrations.destination.s3_data_lake.io.AWSSystemCredentials +import io.airbyte.integrations.destination.s3_data_lake.io.IcebergUtil import java.nio.file.Files import java.nio.file.Path From 8672207916cdb6678f8daa7d954d08dd0adbd231 Mon Sep 17 00:00:00 2001 From: Francis Genet Date: Thu, 9 Jan 2025 09:59:33 -0800 Subject: [PATCH 5/9] And more --- .../destination/s3_data_lake/IcebergV2WriterTest.kt | 4 ++-- .../destination/s3_data_lake/io/IcebergUtilTest.kt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2WriterTest.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2WriterTest.kt index 01a9c0779865..ba88a4526ed9 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2WriterTest.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2WriterTest.kt @@ -23,8 +23,8 @@ import io.airbyte.cdk.load.message.Meta.Companion.COLUMN_NAME_AB_EXTRACTED_AT import io.airbyte.cdk.load.message.Meta.Companion.COLUMN_NAME_AB_GENERATION_ID import io.airbyte.cdk.load.message.Meta.Companion.COLUMN_NAME_AB_META import io.airbyte.cdk.load.message.Meta.Companion.COLUMN_NAME_AB_RAW_ID -import io.airbyte.integrations.destination.iceberg.v2.io.IcebergTableWriterFactory -import io.airbyte.integrations.destination.iceberg.v2.io.IcebergUtil +import io.airbyte.integrations.destination.s3_data_lake.io.IcebergTableWriterFactory +import io.airbyte.integrations.destination.s3_data_lake.io.IcebergUtil import io.mockk.every import io.mockk.mockk import org.apache.iceberg.Schema diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergUtilTest.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergUtilTest.kt index 3ee3d8751669..504c7f7d07e0 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergUtilTest.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergUtilTest.kt @@ -27,8 +27,7 @@ import io.airbyte.cdk.load.message.Meta.Companion.COLUMN_NAME_AB_EXTRACTED_AT import io.airbyte.cdk.load.message.Meta.Companion.COLUMN_NAME_AB_GENERATION_ID import io.airbyte.cdk.load.message.Meta.Companion.COLUMN_NAME_AB_META import io.airbyte.cdk.load.message.Meta.Companion.COLUMN_NAME_AB_RAW_ID -import io.airbyte.integrations.destination.iceberg.v2.IcebergV2Configuration -import io.airbyte.integrations.destination.iceberg.v2.SimpleTableIdGenerator +import io.airbyte.integrations.destination.s3_data_lake.IcebergV2Configuration import io.mockk.every import io.mockk.mockk import io.mockk.verify @@ -37,6 +36,7 @@ import org.apache.iceberg.CatalogProperties.URI import org.apache.iceberg.CatalogProperties.WAREHOUSE_LOCATION import org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE import org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_NESSIE +import io.airbyte.integrations.destination.s3_data_lake.SimpleTableIdGenerator import org.apache.iceberg.FileFormat import org.apache.iceberg.Schema import org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT From 3d3011e41040fd68425eab0fb73612dfe2694449 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Thu, 9 Jan 2025 18:02:16 +0000 Subject: [PATCH 6/9] chore: auto-fix lint and format issues --- .../integrations/destination/s3_data_lake/io/IcebergUtilTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergUtilTest.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergUtilTest.kt index 504c7f7d07e0..3abe0e76ade7 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergUtilTest.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test/kotlin/io/airbyte/integrations/destination/s3_data_lake/io/IcebergUtilTest.kt @@ -28,6 +28,7 @@ import io.airbyte.cdk.load.message.Meta.Companion.COLUMN_NAME_AB_GENERATION_ID import io.airbyte.cdk.load.message.Meta.Companion.COLUMN_NAME_AB_META import io.airbyte.cdk.load.message.Meta.Companion.COLUMN_NAME_AB_RAW_ID import io.airbyte.integrations.destination.s3_data_lake.IcebergV2Configuration +import io.airbyte.integrations.destination.s3_data_lake.SimpleTableIdGenerator import io.mockk.every import io.mockk.mockk import io.mockk.verify @@ -36,7 +37,6 @@ import org.apache.iceberg.CatalogProperties.URI import org.apache.iceberg.CatalogProperties.WAREHOUSE_LOCATION import org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE import org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_NESSIE -import io.airbyte.integrations.destination.s3_data_lake.SimpleTableIdGenerator import org.apache.iceberg.FileFormat import org.apache.iceberg.Schema import org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT From e9176900e46e2093529a396f23ad2075c500de2b Mon Sep 17 00:00:00 2001 From: Francis Genet Date: Thu, 9 Jan 2025 10:17:09 -0800 Subject: [PATCH 7/9] Version change --- .../connectors/destination-s3-data-lake/metadata.yaml | 2 +- docs/integrations/destinations/s3-data-lake.md | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/metadata.yaml b/airbyte-integrations/connectors/destination-s3-data-lake/metadata.yaml index d25b6772b001..c500835d818e 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/metadata.yaml +++ b/airbyte-integrations/connectors/destination-s3-data-lake/metadata.yaml @@ -26,7 +26,7 @@ data: alias: airbyte-connector-testing-secret-store connectorType: destination definitionId: 716ca874-520b-4902-9f80-9fad66754b89 - dockerImageTag: 0.2.7 + dockerImageTag: 0.2.8 dockerRepository: airbyte/destination-s3-data-lake documentationUrl: https://docs.airbyte.com/integrations/destinations/s3-data-lake githubIssueLabel: destination-s3-data-lake diff --git a/docs/integrations/destinations/s3-data-lake.md b/docs/integrations/destinations/s3-data-lake.md index 2eace268e40b..29c390f1d9ad 100644 --- a/docs/integrations/destinations/s3-data-lake.md +++ b/docs/integrations/destinations/s3-data-lake.md @@ -15,9 +15,10 @@ for more information.
Expand to review -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------| -| 0.2.7 | 2025-01-00 | [\#50957](https://github.com/airbytehq/airbyte/pull/50991) | Add support for GLUE RBAC (Assume role) | -| 0.2.6 | 2025-01-08 | [\#50991](https://github.com/airbytehq/airbyte/pull/50991) | Initial public release. | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:------------------------------------------------------------|:--------------------------------------------| +| 0.2.8 | 2025-01-09 | [\#51012](https://github.com/airbytehq/airbyte/pull/50991) | Rename/Cleanup package name from Iceberg V2 | +| 0.2.7 | 2025-01-09 | [\#50957](https://github.com/airbytehq/airbyte/pull/50991) | Add support for GLUE RBAC (Assume role) | +| 0.2.6 | 2025-01-08 | [\#50991](https://github.com/airbytehq/airbyte/pull/50991) | Initial public release. |
From 05813dcb63c2f182579d84b9fc99baf29f9ae010 Mon Sep 17 00:00:00 2001 From: Francis Genet Date: Thu, 9 Jan 2025 11:20:50 -0800 Subject: [PATCH 8/9] Update entry point for the destination --- .../connectors/destination-s3-data-lake/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/build.gradle b/airbyte-integrations/connectors/destination-s3-data-lake/build.gradle index 69cdecf20932..c7ab2caf1318 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/build.gradle +++ b/airbyte-integrations/connectors/destination-s3-data-lake/build.gradle @@ -10,7 +10,7 @@ airbyteBulkConnector { } application { - mainClass = 'io.airbyte.integrations.destination.iceberg.v2.IcebergDestination' + mainClass = 'io.airbyte.integrations.destination.s3_data_lake.IcebergDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] From 2aba82860d84509e28d419d84085d5b85177b9bb Mon Sep 17 00:00:00 2001 From: Francis Genet Date: Thu, 9 Jan 2025 11:25:22 -0800 Subject: [PATCH 9/9] Change name of IcebergDestination to S3DataLakeDestination. Same thing with S3DataLakeDestinationCleaner --- .../connectors/destination-s3-data-lake/build.gradle | 2 +- .../destination/s3_data_lake/IcebergDestination.kt | 2 +- .../destination/s3_data_lake/IcebergDestinationCleaner.kt | 2 +- .../destination/s3_data_lake/IcebergV2WriteTest.kt | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/build.gradle b/airbyte-integrations/connectors/destination-s3-data-lake/build.gradle index c7ab2caf1318..e0fd03bf159a 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/build.gradle +++ b/airbyte-integrations/connectors/destination-s3-data-lake/build.gradle @@ -10,7 +10,7 @@ airbyteBulkConnector { } application { - mainClass = 'io.airbyte.integrations.destination.s3_data_lake.IcebergDestination' + mainClass = 'io.airbyte.integrations.destination.s3_data_lake.S3DataLakeDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergDestination.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergDestination.kt index 7f81a3854e6d..537716fd1905 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergDestination.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/main/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergDestination.kt @@ -6,7 +6,7 @@ package io.airbyte.integrations.destination.s3_data_lake import io.airbyte.cdk.AirbyteDestinationRunner -object IcebergDestination { +object S3DataLakeDestination { @JvmStatic fun main(args: Array) { AirbyteDestinationRunner.run(*args) diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergDestinationCleaner.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergDestinationCleaner.kt index ab4c4f1c989f..209801c44409 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergDestinationCleaner.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergDestinationCleaner.kt @@ -13,7 +13,7 @@ import org.apache.iceberg.catalog.Catalog import org.apache.iceberg.catalog.Namespace import org.apache.iceberg.catalog.SupportsNamespaces -class IcebergDestinationCleaner(private val catalog: Catalog) : DestinationCleaner { +class S3DataLakeDestinationCleaner(private val catalog: Catalog) : DestinationCleaner { override fun cleanup() { val namespaces: List = (catalog as SupportsNamespaces).listNamespaces().filter { diff --git a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2WriteTest.kt b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2WriteTest.kt index 46f6acd121c1..fe17eca2ff5b 100644 --- a/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2WriteTest.kt +++ b/airbyte-integrations/connectors/destination-s3-data-lake/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_data_lake/IcebergV2WriteTest.kt @@ -86,7 +86,7 @@ abstract class IcebergV2WriteTest( class IcebergGlueWriteTest : IcebergV2WriteTest( Files.readString(IcebergV2TestUtil.GLUE_CONFIG_PATH), - IcebergDestinationCleaner( + S3DataLakeDestinationCleaner( IcebergV2TestUtil.getCatalog( IcebergV2TestUtil.parseConfig(IcebergV2TestUtil.GLUE_CONFIG_PATH), IcebergV2TestUtil.getAWSSystemCredentials() @@ -97,7 +97,7 @@ class IcebergGlueWriteTest : class IcebergGlueAssumeRoleWriteTest : IcebergV2WriteTest( Files.readString(IcebergV2TestUtil.GLUE_ASSUME_ROLE_CONFIG_PATH), - IcebergDestinationCleaner( + S3DataLakeDestinationCleaner( IcebergV2TestUtil.getCatalog( IcebergV2TestUtil.parseConfig(IcebergV2TestUtil.GLUE_ASSUME_ROLE_CONFIG_PATH), IcebergV2TestUtil.getAWSSystemCredentials()