Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[S3-data-lake] Remove/Rename all occurences of iceberg.v2 #51012

Merged
merged 9 commits into from
Jan 9, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ airbyteBulkConnector {
}

application {
mainClass = 'io.airbyte.integrations.destination.iceberg.v2.IcebergDestination'
mainClass = 'io.airbyte.integrations.destination.s3_data_lake.S3DataLakeDestination'

applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ data:
alias: airbyte-connector-testing-secret-store
connectorType: destination
definitionId: 716ca874-520b-4902-9f80-9fad66754b89
dockerImageTag: 0.2.7
dockerImageTag: 0.2.8
dockerRepository: airbyte/destination-s3-data-lake
documentationUrl: https://docs.airbyte.com/integrations/destinations/s3-data-lake
githubIssueLabel: destination-s3-data-lake
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2.io;
package io.airbyte.integrations.destination.s3_data_lake.io;

import java.io.IOException;
import java.util.Set;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2
package io.airbyte.integrations.destination.s3_data_lake

import software.amazon.awssdk.auth.credentials.AwsBasicCredentials
import software.amazon.awssdk.auth.credentials.AwsCredentials
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2
package io.airbyte.integrations.destination.s3_data_lake

import io.airbyte.cdk.AirbyteDestinationRunner

object IcebergDestination {
object S3DataLakeDestination {
@JvmStatic
fun main(args: Array<String>) {
AirbyteDestinationRunner.run(*args)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2
package io.airbyte.integrations.destination.s3_data_lake

import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
import io.airbyte.cdk.load.command.DestinationStream
Expand All @@ -12,9 +12,9 @@ import io.airbyte.cdk.load.message.DestinationRecordAirbyteValue
import io.airbyte.cdk.load.message.SimpleBatch
import io.airbyte.cdk.load.state.StreamProcessingFailed
import io.airbyte.cdk.load.write.StreamLoader
import io.airbyte.integrations.destination.iceberg.v2.io.IcebergTableCleaner
import io.airbyte.integrations.destination.iceberg.v2.io.IcebergTableWriterFactory
import io.airbyte.integrations.destination.iceberg.v2.io.IcebergUtil
import io.airbyte.integrations.destination.s3_data_lake.io.IcebergTableCleaner
import io.airbyte.integrations.destination.s3_data_lake.io.IcebergTableWriterFactory
import io.airbyte.integrations.destination.s3_data_lake.io.IcebergUtil
import io.github.oshai.kotlinlogging.KotlinLogging
import org.apache.iceberg.Table

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2
package io.airbyte.integrations.destination.s3_data_lake

import io.airbyte.cdk.load.check.DestinationChecker
import io.airbyte.cdk.load.command.DestinationStream
import io.airbyte.integrations.destination.iceberg.v2.io.IcebergTableCleaner
import io.airbyte.integrations.destination.iceberg.v2.io.IcebergUtil
import io.airbyte.integrations.destination.s3_data_lake.io.IcebergTableCleaner
import io.airbyte.integrations.destination.s3_data_lake.io.IcebergUtil
import javax.inject.Singleton
import org.apache.iceberg.Schema
import org.apache.iceberg.types.Types
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2
package io.airbyte.integrations.destination.s3_data_lake

import io.airbyte.cdk.load.command.DestinationConfiguration
import io.airbyte.cdk.load.command.DestinationConfigurationFactory
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2
package io.airbyte.integrations.destination.s3_data_lake

import com.fasterxml.jackson.annotation.JsonPropertyDescription
import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,14 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2
package io.airbyte.integrations.destination.s3_data_lake

import io.airbyte.cdk.load.command.DestinationStream
import io.airbyte.cdk.load.data.iceberg.parquet.IcebergParquetPipelineFactory
import io.airbyte.cdk.load.write.DestinationWriter
import io.airbyte.cdk.load.write.StreamLoader
import io.airbyte.integrations.destination.iceberg.v2.io.IcebergTableWriterFactory
import io.airbyte.integrations.destination.iceberg.v2.io.IcebergUtil
import io.airbyte.integrations.destination.s3_data_lake.io.IcebergTableWriterFactory
import io.airbyte.integrations.destination.s3_data_lake.io.IcebergUtil
import javax.inject.Singleton
import org.apache.iceberg.Schema

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2
package io.airbyte.integrations.destination.s3_data_lake

import io.airbyte.cdk.load.command.DestinationStream
import io.airbyte.cdk.load.command.iceberg.parquet.GlueCatalogConfiguration
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2.io
package io.airbyte.integrations.destination.s3_data_lake.io

import jakarta.inject.Singleton
import org.apache.iceberg.Table
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2.io
package io.airbyte.integrations.destination.s3_data_lake.io

import io.airbyte.cdk.load.command.Append
import io.airbyte.cdk.load.command.Dedupe
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2.io
package io.airbyte.integrations.destination.s3_data_lake.io

import com.fasterxml.jackson.annotation.JsonProperty
import io.airbyte.cdk.load.command.Dedupe
Expand All @@ -18,11 +18,11 @@ import io.airbyte.cdk.load.data.iceberg.parquet.toIcebergRecord
import io.airbyte.cdk.load.data.iceberg.parquet.toIcebergSchema
import io.airbyte.cdk.load.data.withAirbyteMeta
import io.airbyte.cdk.load.message.DestinationRecordAirbyteValue
import io.airbyte.integrations.destination.iceberg.v2.ACCESS_KEY_ID
import io.airbyte.integrations.destination.iceberg.v2.GlueCredentialsProvider
import io.airbyte.integrations.destination.iceberg.v2.IcebergV2Configuration
import io.airbyte.integrations.destination.iceberg.v2.SECRET_ACCESS_KEY
import io.airbyte.integrations.destination.iceberg.v2.TableIdGenerator
import io.airbyte.integrations.destination.s3_data_lake.ACCESS_KEY_ID
import io.airbyte.integrations.destination.s3_data_lake.GlueCredentialsProvider
import io.airbyte.integrations.destination.s3_data_lake.IcebergV2Configuration
import io.airbyte.integrations.destination.s3_data_lake.SECRET_ACCESS_KEY
import io.airbyte.integrations.destination.s3_data_lake.TableIdGenerator
import io.github.oshai.kotlinlogging.KotlinLogging
import jakarta.inject.Singleton
import java.time.Duration
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2.io
package io.airbyte.integrations.destination.s3_data_lake.io

/** Delta operations for data. */
enum class Operation {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2.io
package io.airbyte.integrations.destination.s3_data_lake.io

import java.io.IOException
import java.io.UncheckedIOException
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2.io
package io.airbyte.integrations.destination.s3_data_lake.io

import org.apache.iceberg.data.Record

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2.io
package io.airbyte.integrations.destination.s3_data_lake.io

import org.apache.iceberg.FileFormat
import org.apache.iceberg.PartitionSpec
Expand Down Expand Up @@ -40,7 +40,7 @@ class UnpartitionedDeltaWriter(

private val writer = RowDataDeltaWriter(null)

override fun route(row: Record): RowDataDeltaWriter {
override fun route(row: Record): BaseDeltaTaskWriter.RowDataDeltaWriter {
return writer
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,18 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2
package io.airbyte.integrations.destination.s3_data_lake

import io.airbyte.cdk.load.test.util.DestinationCleaner
import io.airbyte.cdk.load.test.util.IntegrationTest.Companion.isNamespaceOld
import io.airbyte.cdk.load.test.util.IntegrationTest.Companion.randomizedNamespaceRegex
import io.airbyte.integrations.destination.iceberg.v2.io.IcebergTableCleaner
import io.airbyte.integrations.destination.iceberg.v2.io.IcebergUtil
import io.airbyte.integrations.destination.s3_data_lake.io.IcebergTableCleaner
import io.airbyte.integrations.destination.s3_data_lake.io.IcebergUtil
import org.apache.iceberg.catalog.Catalog
import org.apache.iceberg.catalog.Namespace
import org.apache.iceberg.catalog.SupportsNamespaces

class IcebergDestinationCleaner(private val catalog: Catalog) : DestinationCleaner {
class S3DataLakeDestinationCleaner(private val catalog: Catalog) : DestinationCleaner {
override fun cleanup() {
val namespaces: List<Namespace> =
(catalog as SupportsNamespaces).listNamespaces().filter {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2
package io.airbyte.integrations.destination.s3_data_lake

import io.airbyte.cdk.load.data.AirbyteType
import io.airbyte.cdk.load.data.AirbyteValue
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2
package io.airbyte.integrations.destination.s3_data_lake

import io.airbyte.cdk.load.check.CheckIntegrationTest
import io.airbyte.cdk.load.check.CheckTestConfig
import io.airbyte.integrations.destination.iceberg.v2.IcebergV2TestUtil.GLUE_CONFIG_PATH
import io.airbyte.integrations.destination.s3_data_lake.IcebergV2TestUtil.GLUE_CONFIG_PATH

class IcebergV2CheckTest :
CheckIntegrationTest<IcebergV2Specification>(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2
package io.airbyte.integrations.destination.s3_data_lake

import io.airbyte.cdk.command.ConfigurationSpecification
import io.airbyte.cdk.load.command.DestinationStream
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2
package io.airbyte.integrations.destination.s3_data_lake

import io.airbyte.cdk.load.spec.SpecTest

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,14 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2
package io.airbyte.integrations.destination.s3_data_lake

import com.fasterxml.jackson.core.type.TypeReference
import io.airbyte.cdk.command.ConfigurationSpecification
import io.airbyte.cdk.command.ValidatedJsonUtils
import io.airbyte.cdk.load.util.Jsons
import io.airbyte.integrations.destination.iceberg.v2.io.AWSSystemCredentials
import io.airbyte.integrations.destination.iceberg.v2.io.IcebergUtil
import io.airbyte.integrations.destination.s3_data_lake.io.AWSSystemCredentials
import io.airbyte.integrations.destination.s3_data_lake.io.IcebergUtil
import java.nio.file.Files
import java.nio.file.Path

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2
package io.airbyte.integrations.destination.s3_data_lake

import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.databind.ObjectMapper
Expand Down Expand Up @@ -86,7 +86,7 @@ abstract class IcebergV2WriteTest(
class IcebergGlueWriteTest :
IcebergV2WriteTest(
Files.readString(IcebergV2TestUtil.GLUE_CONFIG_PATH),
IcebergDestinationCleaner(
S3DataLakeDestinationCleaner(
IcebergV2TestUtil.getCatalog(
IcebergV2TestUtil.parseConfig(IcebergV2TestUtil.GLUE_CONFIG_PATH),
IcebergV2TestUtil.getAWSSystemCredentials()
Expand All @@ -97,7 +97,7 @@ class IcebergGlueWriteTest :
class IcebergGlueAssumeRoleWriteTest :
IcebergV2WriteTest(
Files.readString(IcebergV2TestUtil.GLUE_ASSUME_ROLE_CONFIG_PATH),
IcebergDestinationCleaner(
S3DataLakeDestinationCleaner(
IcebergV2TestUtil.getCatalog(
IcebergV2TestUtil.parseConfig(IcebergV2TestUtil.GLUE_ASSUME_ROLE_CONFIG_PATH),
IcebergV2TestUtil.getAWSSystemCredentials()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2
package io.airbyte.integrations.destination.s3_data_lake

import io.airbyte.cdk.load.util.setOnce
import java.io.File
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2
package io.airbyte.integrations.destination.s3_data_lake

import io.airbyte.cdk.load.command.Append
import io.airbyte.cdk.load.command.Dedupe
Expand All @@ -23,8 +23,8 @@ import io.airbyte.cdk.load.message.Meta.Companion.COLUMN_NAME_AB_EXTRACTED_AT
import io.airbyte.cdk.load.message.Meta.Companion.COLUMN_NAME_AB_GENERATION_ID
import io.airbyte.cdk.load.message.Meta.Companion.COLUMN_NAME_AB_META
import io.airbyte.cdk.load.message.Meta.Companion.COLUMN_NAME_AB_RAW_ID
import io.airbyte.integrations.destination.iceberg.v2.io.IcebergTableWriterFactory
import io.airbyte.integrations.destination.iceberg.v2.io.IcebergUtil
import io.airbyte.integrations.destination.s3_data_lake.io.IcebergTableWriterFactory
import io.airbyte.integrations.destination.s3_data_lake.io.IcebergUtil
import io.mockk.every
import io.mockk.mockk
import org.apache.iceberg.Schema
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2.io
package io.airbyte.integrations.destination.s3_data_lake.io

import io.airbyte.integrations.destination.iceberg.v2.io.BaseDeltaTaskWriter.RowDataDeltaWriter
import io.airbyte.integrations.destination.s3_data_lake.io.BaseDeltaTaskWriter.RowDataDeltaWriter
import io.mockk.every
import io.mockk.mockk
import io.mockk.verify
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2.io
package io.airbyte.integrations.destination.s3_data_lake.io

import io.mockk.Runs
import io.mockk.called
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
* Copyright (c) 2024 Airbyte, Inc., all rights reserved.
*/

package io.airbyte.integrations.destination.iceberg.v2.io
package io.airbyte.integrations.destination.s3_data_lake.io

import io.airbyte.cdk.load.command.Append
import io.airbyte.cdk.load.command.Dedupe
Expand Down
Loading
Loading