diff --git a/.github/workflows/testflows-sink-connector-kafka.yml b/.github/workflows/testflows-sink-connector-kafka.yml index 6bbad76eb..ac2963adf 100644 --- a/.github/workflows/testflows-sink-connector-kafka.yml +++ b/.github/workflows/testflows-sink-connector-kafka.yml @@ -48,6 +48,18 @@ jobs: working-directory: sink-connector/tests/integration run: echo "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)" + - name: Create a virtual environment + run: | + echo "Install Python modules..." + sudo apt-get clean + sudo apt-get update + sudo apt-get install -y python3.12-venv + + echo "Create and activate Python virtual environment..." + python3 -m venv venv + source venv/bin/activate + echo PATH=$PATH >> $GITHUB_ENV + - name: Install all dependencies working-directory: sink-connector/tests/integration run: pip3 install -r requirements.txt diff --git a/.github/workflows/testflows-sink-connector-lightweight.yml b/.github/workflows/testflows-sink-connector-lightweight.yml index 694235f1e..db7e320df 100644 --- a/.github/workflows/testflows-sink-connector-lightweight.yml +++ b/.github/workflows/testflows-sink-connector-lightweight.yml @@ -64,6 +64,18 @@ jobs: working-directory: sink-connector/tests/integration run: echo "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)" + - name: Create a virtual environment + run: | + echo "Install Python modules..." + sudo apt-get clean + sudo apt-get update + sudo apt-get install -y python3.12-venv + + echo "Create and activate Python virtual environment..." + python3 -m venv venv + source venv/bin/activate + echo PATH=$PATH >> $GITHUB_ENV + - name: Install all dependencies working-directory: sink-connector-lightweight/tests/integration run: pip3 install -r requirements.txt @@ -130,6 +142,18 @@ jobs: working-directory: sink-connector/tests/integration run: echo "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)" + - name: Create a virtual environment + run: | + echo "Install Python modules..." + sudo apt-get clean + sudo apt-get update + sudo apt-get install -y python3.12-venv + + echo "Create and activate Python virtual environment..." + python3 -m venv venv + source venv/bin/activate + echo PATH=$PATH >> $GITHUB_ENV + - name: Install all dependencies working-directory: sink-connector-lightweight/tests/integration run: pip3 install -r requirements.txt diff --git a/README.md b/README.md index 765672f6c..bc77c4e1e 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,9 @@ First two are good tutorials on MySQL and PostgreSQL respectively. * [Mutable Data Handling](doc/mutable_data.md) * [ClickHouse Table Engine Types](doc/clickhouse_engines.md) * [Troubleshooting](doc/Troubleshooting.md) +* [TimeZone and DATETIME/TIMESTAMP](doc/timezone.md) +* [Logging](doc/Logging.md) +* [Production Setup](doc/production_setup.md) ### Operations diff --git a/doc/OLD_README.md b/doc/OLD_README.md index 7cb7ce384..6e2cb9d6e 100644 --- a/doc/OLD_README.md +++ b/doc/OLD_README.md @@ -144,7 +144,6 @@ GLOBAL OPTIONS: | clickhouse.server.user | ClickHouse username | | clickhouse.server.password | ClickHouse password | | clickhouse.server.port | ClickHouse port, For TLS(use the correct port `8443` or `443` | -| clickhouse.server.database | ClickHouse destination database | | snapshot.mode | "initial" -> Data that already exists in source database will be replicated. "schema_only" -> Replicate data that is added/modified after the connector is started.\
MySQL: https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-property-snapshot-mode \
PostgreSQL: https://debezium.io/documentation/reference/stable/connectors/postgresql.html#postgresql-property-snapshot-mode
MongoDB: initial, never. https://debezium.io/documentation/reference/stable/connectors/mongodb.html | | connector.class | MySQL -> "io.debezium.connector.mysql.MySqlConnector"
PostgreSQL ->
Mongo ->
| | offset.storage.file.filename | Offset storage file(This stores the offsets of the source database) MySQL: mysql binlog file and position, gtid set. Make sure this file is durable and its not persisted in temp directories. | @@ -298,7 +297,6 @@ mvn install -DskipTests=true | clickhouse.server.url | | ClickHouse Server URL | | clickhouse.server.user | | ClickHouse Server username | | clickhouse.server.password | | ClickHouse Server password | -| clickhouse.server.database | | ClickHouse Database name | | clickhouse.server.port | 8123 | ClickHouse Server port | | clickhouse.topic2table.map | No | Map of Kafka topics to table names, :,: This variable will override the default mapping of topics to table names. | | store.kafka.metadata | false | If set to true, kafka metadata columns will be added to Clickhouse | diff --git a/doc/configuration.md b/doc/configuration.md index 21c412b90..53a218ca0 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -12,7 +12,6 @@ | clickhouse.server.user | ClickHouse username | | clickhouse.server.password | ClickHouse password | | clickhouse.server.port | ClickHouse port, For TLS(use the correct port `8443` or `443` | -| clickhouse.server.database | ClickHouse destination database | | snapshot.mode | "initial" -> Data that already exists in source database will be replicated. "schema_only" -> Replicate data that is added/modified after the connector is started.\
MySQL: https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-property-snapshot-mode \
PostgreSQL: https://debezium.io/documentation/reference/stable/connectors/postgresql.html#postgresql-property-snapshot-mode
MongoDB: initial, never. https://debezium.io/documentation/reference/stable/connectors/mongodb.html | | connector.class | MySQL -> "io.debezium.connector.mysql.MySqlConnector"
PostgreSQL ->
Mongo ->
| | offset.storage.file.filename | Offset storage file(This stores the offsets of the source database) MySQL: mysql binlog file and position, gtid set. Make sure this file is durable and its not persisted in temp directories. | diff --git a/doc/configuration_kafka.md b/doc/configuration_kafka.md index b828d728f..cd44a58e3 100644 --- a/doc/configuration_kafka.md +++ b/doc/configuration_kafka.md @@ -22,7 +22,6 @@ for both **Debezium** and **Sink** | clickhouse.server.url | | ClickHouse Server URL | | clickhouse.server.user | | ClickHouse Server username | | clickhouse.server.password | | ClickHouse Server password | -| clickhouse.server.database | | ClickHouse Database name | | clickhouse.server.port | 8123 | ClickHouse Server port | | clickhouse.topic2table.map | No | Map of Kafka topics to table names, :,: This variable will override the default mapping of topics to table names. | | store.kafka.metadata | false | If set to true, kafka metadata columns will be added to Clickhouse | diff --git a/doc/img/production_setup.drawio b/doc/img/production_setup.drawio new file mode 100644 index 000000000..0acde7cb1 --- /dev/null +++ b/doc/img/production_setup.drawio @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/img/production_setup.jpg b/doc/img/production_setup.jpg new file mode 100644 index 000000000..3c03a1e44 Binary files /dev/null and b/doc/img/production_setup.jpg differ diff --git a/doc/logging.md b/doc/logging.md new file mode 100644 index 000000000..22599a44b --- /dev/null +++ b/doc/logging.md @@ -0,0 +1,29 @@ +## Lightweight Sink Connector Logging + +Sink connector uses the log4j version 2 directly, however the dependendent libraries like debezium and clickhouse-jdbc +use different logging frameworks(JUL, slf4j). +There is logic to use adapters to bridge the logging frameworks. + +Logging is controlled by the `log4j2.xml` file in the 'docker' directory. +This file is mounted into the container and is passed to the JVM as a system property.(-Dlog4j.configurationFile) + +### Log Levels +If you need to change the Logging level , you can modify the rootLogger level in the `log4j2.xml` file. +By default its set to `info` level. + +```xml + + +``` + +## Changing Layout (Example JSON) +If you want to change the layout of the logs, you can modify the `log4j2.xml` file. +You can comment out the default `PatternLayout` and enable `JSONLayout` + +``` + + + + + +``` \ No newline at end of file diff --git a/doc/production_setup.md b/doc/production_setup.md new file mode 100644 index 000000000..0c799476c --- /dev/null +++ b/doc/production_setup.md @@ -0,0 +1,44 @@ +## Production setup +![](img/production_setup.jpg) + + +### Improving throughput and/or Memory usage. + +As detailed in the diagram above, there are components that store the messages and +can be configured to improve throughput and/or memory usage. + +1. **Debezium Queue**: + +The following configuration parameters are used to configure the size of the debezium queue +in terms of number of elements the queue can hold and the maximum size of the queue in bytes. + + ``` + #Positive integer value that specifies the maximum size of each batch of events that should be processed during each iteration of this connector. Defaults to 2048. + max.batch.size: 20000 + + #Positive integer value that specifies the maximum number of records that the blocking queue can hold. + max.queue.size: 100000 + + # A long integer value that specifies the maximum volume of the blocking queue in bytes. + max.queue.size.in.bytes: 1000000000 + ``` + +2. **Sink connector Queue**: + +``` + # The maximum number of records that should be loaded into memory while streaming data from MySQL to ClickHouse. + sink.connector.max.queue.size: "100000" + +``` + +3. **Thread Pool**: +``` + # Maximum number of threads in the thread pool for processing CDC records. + thread.pool.size: 10 + + # Max number of records for the flush buffer. + buffer.max.records: "1000000" + + Flush time of the buffer in milliseconds. The buffer that is stored in memory before being flushed to ClickHouse. + buffer.flush.time.ms: "1000" +``` \ No newline at end of file diff --git a/doc/sink_configuration.md b/doc/sink_configuration.md index 2ed93716b..f48d9b8e5 100644 --- a/doc/sink_configuration.md +++ b/doc/sink_configuration.md @@ -29,7 +29,6 @@ This is a sample configuration that's used in creating the connector using the K "clickhouse.server.url": "${CLICKHOUSE_HOST}", "clickhouse.server.user": "${CLICKHOUSE_USER}", "clickhouse.server.password": "${CLICKHOUSE_PASSWORD}", - "clickhouse.server.database": "${CLICKHOUSE_DATABASE}", "clickhouse.server.port": ${CLICKHOUSE_PORT}, "clickhouse.table.name": "${CLICKHOUSE_TABLE}", "key.converter": "io.apicurio.registry.utils.converter.AvroConverter", diff --git a/doc/sink_connector_cli.md b/doc/sink_connector_cli.md index 8fe0a5683..4a21df5df 100644 --- a/doc/sink_connector_cli.md +++ b/doc/sink_connector_cli.md @@ -13,10 +13,9 @@ This will give users option to set the binlog status/position, gtid ## Operations -1. **Startup**, the CLI application will send a REST API call to the server, if it gets a 200, it will continue or throw an error. -2. **Start_replica**, CLI application will send a REST API call to the server to start replication(start Debezium event loop) -3. **Stop_replica**, CLI application will send a REST API call to the server to stop replication(stop Debezium event loop) -4. **change_replication_source**, CLI application will send the gtid, binlog file, and binlog position to the server to change the replication source +1. **Start_replica**, CLI application will send a REST API call to the server to start replication(start Debezium event loop) +2. **Stop_replica**, CLI application will send a REST API call to the server to stop replication(stop Debezium event loop) +3. **change_replication_source**, CLI application will send the gtid, binlog file, and binlog position to the server to change the replication source Server will update the table with this information will restart the debezium event loop. -5. **show replica status** Return the information from the **replica_status** table. - +4. **show_replica_status** Return the information from the **replica_status** table. +5. **restart** Restart replication, this is also useful to reload the configuration file from disk. diff --git a/doc/timezone.md b/doc/timezone.md index 4d334874c..a8dfc6294 100644 --- a/doc/timezone.md +++ b/doc/timezone.md @@ -1,6 +1,31 @@ ## Time Zone +Ideally the timezone of the source database(MySQL/PostgreSQL) should be the same as the timezone of ClickHouse.(Preferably UTC) +If for some reason they are different, the following instructions can be used to set the timezone of the source database and ClickHouse. + +### Configuring the timezone for MySQL +The timezone of the source database(MySQL) can be set using the `database.connectionTimezone` configuration in the config.yml file. + +## Overriding the ClickHouse server timezone +The timezone of ClickHouse can be set using the `clickhouse.datetime.timezone` configuration in the config.yml file. + +## Debezium handling of time fields. +Its important to read this section to understand how the DATETIME/TIMESTAMP fields are handled by Debezium. +https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-temporal-types + +``` +The DATETIME type represents a local date and time such as "2018-01-13 09:48:27". As you can see, there is no time zone information. Such columns are converted into epoch milliseconds or microseconds based on the column’s precision by using UTC. The TIMESTAMP type represents a timestamp without time zone information. It is converted by MySQL from the server (or session’s) current time zone into UTC when writing and from UTC into the server (or session’s) current time zone when reading back the value. For example: + +DATETIME with a value of 2018-06-20 06:37:03 becomes 1529476623000. + +TIMESTAMP with a value of 2018-06-20 06:37:03 becomes 2018-06-20T13:37:03Z. + +Such columns are converted into an equivalent io.debezium.time.ZonedTimestamp in UTC based on the server (or session’s) current time zone. The time zone will be queried from the server by default. If this fails, it must be specified explicitly by the database connectionTimeZone MySQL configuration option. For example, if the database’s time zone (either globally or configured for the connector by means of the connectionTimeZone option) is "America/Los_Angeles", the TIMESTAMP value "2018-06-20 06:37:03" is represented by a ZonedTimestamp with the value "2018-06-20T13:37:03Z". + +``` + + +### Example of setting the timezone for MySQL and ClickHouse for DATETIME fields. -### MySQL 1. The environment variable `TZ=US/Central` in docker-compose under mysql can be used to set the timezone for MySQL. 2. To make sure the timezone is properly set in MySQL, run the following SQL on MySQL Server. diff --git a/pom.xml b/pom.xml index f9aa5c09d..b122cfd0e 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ 17 UTF-8 UTF-8 - 2.4.0 + 2.5.0.Beta1 5.9.1 3.1.1 UTF-8 diff --git a/sink-connector-client/main.go b/sink-connector-client/main.go index 2f690fe83..0585664cd 100644 --- a/sink-connector-client/main.go +++ b/sink-connector-client/main.go @@ -35,11 +35,12 @@ const ( UPDATE_LSN_COMMAND = "lsn" ) const ( - START_REPLICATION = "start" - STOP_REPLICATION = "stop" - STATUS = "status" - UPDATE_BINLOG = "binlog" - UPDATE_LSN = "lsn" + START_REPLICATION = "start" + STOP_REPLICATION = "stop" + RESTART_REPLICATION = "restart" + STATUS = "status" + UPDATE_BINLOG = "binlog" + UPDATE_LSN = "lsn" ) // Fetches the repos for the given Github users @@ -77,6 +78,7 @@ func getServerUrl(action string, c *cli.Context) string { func main() { app := cli.NewApp() + app.EnableBashCompletion = true app.Name = "Sink Connector Lightweight CLI" app.Usage = "CLI for Sink Connector Lightweight, operations to get status, start/stop replication and set binlog/gtid position" app.Flags = []cli.Flag{ @@ -122,6 +124,19 @@ func main() { return nil }, }, + // Restart + { + Name: RESTART_REPLICATION, + Usage: "Restart the replication, this was also reload the configuration file from disk", + Action: func(c *cli.Context) error { + log.Println("***** Restarting replication..... *****") + var serverUrl = getServerUrl(RESTART_REPLICATION, c) + resp := getHTTPCall(serverUrl) + log.Println(resp.String()) + log.Println("***** Replication restarted successfully and configuration file reloaded from disk *****") + return nil + }, + }, { Name: STATUS_COMMAND, //Aliases: []string{"c"}, diff --git a/sink-connector-client/sink-connector-client b/sink-connector-client/sink-connector-client index 33cb470c8..f6bec4bb8 100755 Binary files a/sink-connector-client/sink-connector-client and b/sink-connector-client/sink-connector-client differ diff --git a/sink-connector-lightweight/clickhouse/config.xml b/sink-connector-lightweight/clickhouse/config.xml index a94e439cd..34d1259ae 100644 --- a/sink-connector-lightweight/clickhouse/config.xml +++ b/sink-connector-lightweight/clickhouse/config.xml @@ -8,6 +8,7 @@ 15000 + default clickhouse 02 diff --git a/sink-connector-lightweight/config.examples/mysql_config.yaml b/sink-connector-lightweight/config.examples/mysql_config.yaml index 785c712f6..80bbe7621 100644 --- a/sink-connector-lightweight/config.examples/mysql_config.yaml +++ b/sink-connector-lightweight/config.examples/mysql_config.yaml @@ -8,7 +8,6 @@ clickhouse.server.url: "localhost" clickhouse.server.user: "root" clickhouse.server.password: "root" clickhouse.server.port: "8123" -clickhouse.server.database: "test" database.allowPublicKeyRetrieval: "true" snapshot.mode: "schema_only" connector.class: "io.debezium.connector.mysql.MySqlConnector" diff --git a/sink-connector-lightweight/dependency-reduced-pom.xml b/sink-connector-lightweight/dependency-reduced-pom.xml index e2e59b558..ae2ba9236 100644 --- a/sink-connector-lightweight/dependency-reduced-pom.xml +++ b/sink-connector-lightweight/dependency-reduced-pom.xml @@ -72,6 +72,7 @@ + filesystem listener com.altinity.clickhouse.debezium.embedded.FailFastListener @@ -79,6 +80,7 @@ true true true + ${surefire.test.runOrder} @@ -116,7 +118,7 @@ io.debezium debezium-connector-mongodb - 2.5.0.Beta1 + 2.7.0.Alpha2 test @@ -297,7 +299,6 @@ quarkus-bom 1.19.1 3.0.0-M7 - 5.2.1 UTF-8 0.0.8 5.9.1 @@ -305,7 +306,7 @@ UTF-8 3.1.1 17 - 2.5.0.Beta1 + 2.7.0.Alpha2 io.quarkus.platform diff --git a/sink-connector-lightweight/docker/clickhouse-sink-connector-lt-service.yml b/sink-connector-lightweight/docker/clickhouse-sink-connector-lt-service.yml index e72c2ee32..c14b6f29c 100644 --- a/sink-connector-lightweight/docker/clickhouse-sink-connector-lt-service.yml +++ b/sink-connector-lightweight/docker/clickhouse-sink-connector-lt-service.yml @@ -3,7 +3,7 @@ version: "3.4" services: clickhouse-sink-connector-lt: image: ${CLICKHOUSE_SINK_CONNECTOR_LT_IMAGE} - entrypoint: ["sh", "-c", "java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Xms4g -Xmx4g -jar /app.jar /config.yml com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication"] + entrypoint: ["sh", "-c", "java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Xms4g -Xmx4g -Dlog4j2.configurationFile=log4j2.xml -jar /app.jar /config.yml com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication"] restart: "no" ports: - "8083:8083" @@ -12,4 +12,5 @@ services: extra_hosts: - "host.docker.internal:host-gateway" volumes: + - ./log4j2.xml:/log4j2.xml - ./config.yml:/config.yml diff --git a/sink-connector-lightweight/docker/config.yml b/sink-connector-lightweight/docker/config.yml index 8b8996503..527912994 100644 --- a/sink-connector-lightweight/docker/config.yml +++ b/sink-connector-lightweight/docker/config.yml @@ -41,9 +41,6 @@ clickhouse.server.password: "root" # Clickhouse Server Port clickhouse.server.port: "8123" -# Clickhouse Server Database -clickhouse.server.database: "test" - # database.allowPublicKeyRetrieval: "true" https://rmoff.net/2019/10/23/debezium-mysql-v8-public-key-retrieval-is-not-allowed/ database.allowPublicKeyRetrieval: "true" @@ -51,6 +48,9 @@ database.allowPublicKeyRetrieval: "true" # The default value of the property is initial. You can customize the way that the connector creates snapshots by changing the value of the snapshot.mode property snapshot.mode: "initial" +# Snapshot.locking.mode Required for Debezium 2.7.0 and later. The snapshot.locking.mode configuration property specifies the mode that the connector uses to lock tables during snapshotting. +#snapshot.locking.mode: "minimal" + # offset.flush.interval.ms: The number of milliseconds to wait before flushing recent offsets to Kafka. This ensures that offsets are committed within the specified time interval. offset.flush.interval.ms: 5000 @@ -123,6 +123,10 @@ auto.create.tables: "true" # database.connectionTimeZone: The timezone of the MySQL database server used to correctly shift the commit transaction timestamp. database.connectionTimeZone: "UTC" +# Configuration to override the clickhouse database name for a given MySQL database name. If this configuration is not +# provided, the MySQL database name will be used as the ClickHouse database name. +#clickhouse.database.override.map: "employees:employees2, products:productsnew + # clickhouse.datetime.timezone: This timezone will override the default timezone of ClickHouse server. Timezone columns will be set to this timezone. #clickhouse.datetime.timezone: "UTC" @@ -147,6 +151,9 @@ restart.event.loop: "true" #restart.event.loop.timeout.period.secs: Defines the restart timeout period. restart.event.loop.timeout.period.secs: "3000" +# Flush time of the buffer in milliseconds. The buffer that is stored in memory before being flushed to ClickHouse. +#buffer.flush.time.ms: "1000" + # Max number of records for the flush buffer. #buffer.max.records: "10000" diff --git a/sink-connector-lightweight/docker/config/grafana/config/datasource.yml b/sink-connector-lightweight/docker/config/grafana/config/datasource.yml index ca317ef25..2164bf945 100644 --- a/sink-connector-lightweight/docker/config/grafana/config/datasource.yml +++ b/sink-connector-lightweight/docker/config/grafana/config/datasource.yml @@ -1,3 +1,5 @@ +apiVersion: 1 + datasources: - access: 'proxy' # make grafana perform the requests editable: true # whether it should be editable @@ -8,18 +10,17 @@ datasources: url: 'http://prometheus:9090' # url of the prom instance database: 'prometheus' version: 1 # well, versioning - - name: Clickhouse - type: vertamedia-clickhouse-datasource - access: 'proxy' - url: http://clickhouse:8123 - basicAuth: true - withCredentials: true - user: "ch_user" - database: "" - secureJsonData: - password: "root" + - name: Clickhouse + type: vertamedia-clickhouse-datasource + access: proxy + url: http://clickhouse:8123 + basicAuth: true + secureJsonData: basicAuthPassword: "root" - isDefault: false - readOnly: false - editable: true - version: 1 \ No newline at end of file + basicAuthUser: "root" + editable: true + jsonData: + addCorsHeader: true + usePOST: true + useCompression: true + compressionType: gzip \ No newline at end of file diff --git a/sink-connector-lightweight/docker/config_local.yml b/sink-connector-lightweight/docker/config_local.yml index 9d35d4d0c..ba18ebf44 100644 --- a/sink-connector-lightweight/docker/config_local.yml +++ b/sink-connector-lightweight/docker/config_local.yml @@ -24,7 +24,7 @@ database.server.name: "ER54" database.include.list: sbtest # table.include.list An optional list of regular expressions that match fully-qualified table identifiers for tables to be monitored; -table.include.list: "sbtest.table1,sbtest.table2,sbtest.table3" +#table.include.list: "sbtest.table1,sbtest.table2,sbtest.table3" # Clickhouse Server URL, Specify only the hostname. clickhouse.server.url: "localhost" @@ -38,9 +38,6 @@ clickhouse.server.password: "root" # Clickhouse Server Port clickhouse.server.port: "8123" -# Clickhouse Server Database -clickhouse.server.database: "test" - # database.allowPublicKeyRetrieval: "true" https://rmoff.net/2019/10/23/debezium-mysql-v8-public-key-retrieval-is-not-allowed/ database.allowPublicKeyRetrieval: "true" @@ -69,6 +66,8 @@ offset.storage.jdbc.user: "root" # offset.storage.jdbc.password: The password of the database user to be used when connecting to the database where connector offsets are to be stored. offset.storage.jdbc.password: "root" +clickhouse.database.override.map: "sbtest:chtest" + # offset.storage.jdbc.offset.table.ddl: The DDL statement used to create the database table where connector offsets are to be stored.(Advanced) offset.storage.jdbc.offset.table.ddl: "CREATE TABLE if not exists %s ( diff --git a/sink-connector-lightweight/docker/config_postgres.yml b/sink-connector-lightweight/docker/config_postgres.yml index 491a544b8..981763787 100644 --- a/sink-connector-lightweight/docker/config_postgres.yml +++ b/sink-connector-lightweight/docker/config_postgres.yml @@ -3,6 +3,8 @@ # name: Unique name for the connector. Attempting to register again with the same name will fail. name: "debezium-embedded-postgres" +auto.create.tables.replicated: "true" + # database.hostname: IP address or hostname of the PostgreSQL database server. database.hostname: "postgres" @@ -19,13 +21,15 @@ database.password: "root" database.server.name: "ER54" # schema.include.list: An optional list of regular expressions that match schema names to be monitored; -schema.include.list: public +schema.include.list: public,public2 + +slot.name: connector2 # plugin.name: The name of the PostgreSQL logical decoding plug-in installed on the PostgreSQL server. Supported values are decoderbufs, and pgoutput. plugin.name: "pgoutput" # table.include.list: An optional list of regular expressions that match fully-qualified table identifiers for tables to be monitored; -table.include.list: "public.tm,public.tm2" +#table.include.list: "public.tm,public.tm2" # clickhouse.server.url: Specify only the hostname of the Clickhouse Server. clickhouse.server.url: "clickhouse" @@ -39,9 +43,6 @@ clickhouse.server.password: "root" # clickhouse.server.port: Clickhouse Server Port clickhouse.server.port: "8123" -# clickhouse.server.database: Clickhouse Server Database -clickhouse.server.database: "test" - # database.allowPublicKeyRetrieval: "true" https://rmoff.net/2019/10/23/debezium-mysql-v8-public-key-retrieval-is-not-allowed/ database.allowPublicKeyRetrieval: "true" @@ -118,4 +119,4 @@ database.dbname: "public" #disable.ddl: "false" #disable.drop.truncate: If set to true, the connector will ignore drop and truncate events. The default is false. -#disable.drop.truncate: "false" \ No newline at end of file +#disable.drop.truncate: "false" diff --git a/sink-connector-lightweight/docker/docker-compose-postgres.yml b/sink-connector-lightweight/docker/docker-compose-postgres.yml index 484f9a214..587a3db36 100644 --- a/sink-connector-lightweight/docker/docker-compose-postgres.yml +++ b/sink-connector-lightweight/docker/docker-compose-postgres.yml @@ -17,6 +17,14 @@ services: extends: file: clickhouse-service.yml service: clickhouse + depends_on: + zookeeper: + condition: service_healthy + + zookeeper: + extends: + file: zookeeper-service.yml + service: zookeeper clickhouse-sink-connector-lt: extends: diff --git a/sink-connector-lightweight/docker/grafana-service.yml b/sink-connector-lightweight/docker/grafana-service.yml index c809b2ea5..fee429890 100644 --- a/sink-connector-lightweight/docker/grafana-service.yml +++ b/sink-connector-lightweight/docker/grafana-service.yml @@ -10,4 +10,5 @@ services: - DS_PROMETHEUS=prometheus - GF_USERS_DEFAULT_THEME=light - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=vertamedia-clickhouse-datasource,grafana-clickhouse-datasource - - GF_INSTALL_PLUGINS=vertamedia-clickhouse-datasource,grafana-clickhouse-datasource \ No newline at end of file + - GF_INSTALL_PLUGINS=vertamedia-clickhouse-datasource + - GF_LOG_LEVEL=debug \ No newline at end of file diff --git a/sink-connector-lightweight/helm/sink-connector-lightweight/templates/configmap.yaml b/sink-connector-lightweight/helm/sink-connector-lightweight/templates/configmap.yaml index d666e7b13..e72d89e06 100644 --- a/sink-connector-lightweight/helm/sink-connector-lightweight/templates/configmap.yaml +++ b/sink-connector-lightweight/helm/sink-connector-lightweight/templates/configmap.yaml @@ -18,7 +18,6 @@ data: clickhouse.server.user: "root" clickhouse.server.password: "root" clickhouse.server.port: "8123" - clickhouse.server.database: "test" database.allowPublicKeyRetrieval: "true" snapshot.mode: "initial" offset.flush.interval.ms: 5000 @@ -71,7 +70,6 @@ data: clickhouse.server.user: "root" clickhouse.server.pass: "secretsecret" clickhouse.server.port: "8123" - clickhouse.server.database: "default" database.allowPublicKeyRetrieval: "true" snapshot.mode: "initial" offset.flush.interval.ms: "5000" diff --git a/sink-connector-lightweight/pom.xml b/sink-connector-lightweight/pom.xml index 136498017..c6bacc575 100644 --- a/sink-connector-lightweight/pom.xml +++ b/sink-connector-lightweight/pom.xml @@ -13,7 +13,7 @@ 17 UTF-8 UTF-8 - 2.5.0.Beta1 + 2.7.0.Beta2 5.9.1 1.19.1 3.1.1 @@ -23,7 +23,7 @@ io.quarkus.platform 2.14.0.Final 3.0.0-M7 - 0.0.8 + 0.0.9 @@ -191,6 +191,18 @@ log4j-api 2.17.1 + + + org.apache.logging.log4j + log4j-slf4j2-impl + 2.23.1 + + + org.antlr antlr4-runtime @@ -395,6 +407,13 @@ 2.23.1 compile + + + org.apache.logging.log4j + log4j-jul + 2.23.1 + + org.slf4j slf4j-api @@ -504,6 +523,7 @@ + com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication diff --git a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedApplication.java b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedApplication.java index 168f7fcd2..6842b8e43 100644 --- a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedApplication.java +++ b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedApplication.java @@ -14,6 +14,7 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.jul.Log4jBridgeHandler; import java.io.IOException; import java.util.Properties; @@ -30,7 +31,6 @@ public class ClickHouseDebeziumEmbeddedApplication { private static DebeziumChangeEventCapture debeziumChangeEventCapture; - private static Properties userProperties = new Properties(); private static Injector injector; @@ -41,6 +41,9 @@ public class ClickHouseDebeziumEmbeddedApplication { private static TimerTask monitoringTimerTask; + // store the configuration file + // so that it can be refreshed on restart. + private static String configurationFile; /** * Main Entry for the application * @param args arguments @@ -48,15 +51,18 @@ public class ClickHouseDebeziumEmbeddedApplication { */ public static void main(String[] args) throws Exception { + //BasicConfigurator.configure(); - System.setProperty("log4j.configurationFile", "resources/log4j2.xml"); - //org.apache.log4j.Logger root = org.apache.logging.log4j.getRootLogger(); - // root.addAppender(new ConsoleAppender(new PatternLayout("%r %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %p %c %x - %m%n"))); + Log4jBridgeHandler.install(false, "", true); + System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); + + System.setProperty("log4j.configurationFile", "resources/log4j2.xml"); + embeddedApplication = new ClickHouseDebeziumEmbeddedApplication(); String loggingLevel = System.getenv("LOGGING_LEVEL"); if(loggingLevel != null) { - // If the user passes a wrong level, it defaults to DEBUG + // If the user passes a wrong level, it defaults to DEBUGl LogManager.getRootLogger().atLevel(Level.toLevel(loggingLevel)); } else { LogManager.getRootLogger().atLevel(Level.INFO); @@ -68,22 +74,16 @@ public static void main(String[] args) throws Exception { log.info(String.format("****** CONFIGURATION FILE: %s ********", args[0])); try { - Properties defaultProperties = PropertiesHelper.getProperties("config.properties"); - - props.putAll(defaultProperties); - Properties fileProps = new ConfigLoader().loadFromFile(args[0]); - props.putAll(fileProps); + configurationFile = args[0]; + embeddedApplication.loadPropertiesFile(configurationFile); } catch(Exception e) { log.error("Error parsing configuration file, USAGE: java -jar : \n" + e.toString()); System.exit(-1); } } else { - props = injector.getInstance(ConfigurationService.class).parse(); } - embeddedApplication = new ClickHouseDebeziumEmbeddedApplication(); - setupMonitoringThread(new ClickHouseSinkConnectorConfig(PropertiesHelper.toMap(props)), props); embeddedApplication.start(injector.getInstance(DebeziumRecordParserService.class), @@ -97,7 +97,21 @@ public static void main(String[] args) throws Exception { } /** - * Force start replication from REST API. + * Function to load properties from user provided file path + * @param filePath user provided file path + * @throws Exception + */ + private static void loadPropertiesFile(String filePath) throws Exception { + props.clear(); + Properties defaultProperties = PropertiesHelper.getProperties("config.properties"); + props.putAll(defaultProperties); + Properties fileProps = new ConfigLoader().loadFromFile(filePath); + props.putAll(fileProps); + + } + + /** + * Force start replication from REST API.l * @param injector * @param props * @return @@ -123,11 +137,13 @@ public static CompletableFuture startDebeziumEventLoop(Injector injector public static void start(DebeziumRecordParserService recordParserService, DDLParserService ddlParserService, Properties props, boolean forceStart) throws Exception { - + if(forceStart == true) { + // Reload the configuration file. + log.info(String.format("******* Reloading configuration file (%s) from disk ******", configurationFile)); + loadPropertiesFile(configurationFile); + } debeziumChangeEventCapture = new DebeziumChangeEventCapture(); debeziumChangeEventCapture.setup(props, recordParserService, ddlParserService, forceStart); - - } public static void stop() throws IOException { @@ -181,8 +197,10 @@ public void run() { start(injector.getInstance(DebeziumRecordParserService.class), injector.getInstance(DDLParserService.class), props, true); } catch (IOException e) { + log.error("**** ERROR: Restarting Event Loop ****", e); throw new RuntimeException(e); } catch (Exception e) { + log.error("**** ERROR: Restarting Event Loop ****", e); throw new RuntimeException(e); } diff --git a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/api/DebeziumEmbeddedRestApi.java b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/api/DebeziumEmbeddedRestApi.java index 85d5033e9..b2e3fcdb6 100644 --- a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/api/DebeziumEmbeddedRestApi.java +++ b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/api/DebeziumEmbeddedRestApi.java @@ -42,8 +42,21 @@ public static void startRestApi(Properties props, Injector injector, Properties finalProps1 = props; app.get("/status", ctx -> { ClickHouseSinkConnectorConfig config = new ClickHouseSinkConnectorConfig(PropertiesHelper.toMap(finalProps1)); - - ctx.result(debeziumChangeEventCapture.getDebeziumStorageStatus(config, finalProps1)); + String response = ""; + + try { + response = debeziumChangeEventCapture.getDebeziumStorageStatus(config, finalProps1); + } catch (Exception e) { + log.error("Client - Error getting status", e); + // Create JSON response + JSONObject jsonObject = new JSONObject(); + jsonObject.put("error", e.toString()); + response = jsonObject.toJSONString(); + ctx.result(response); + ctx.status(HttpStatus.INTERNAL_SERVER_ERROR); + return; + } + ctx.result(response); }); @@ -83,7 +96,6 @@ public static void startRestApi(Properties props, Injector injector, if(userProperties.size() > 0) { log.info("User Overridden properties: " + userProperties); - } debeziumChangeEventCapture.updateDebeziumStorageStatus(config, finalProps1, binlogFile, binlogPosition, @@ -109,6 +121,15 @@ public static void startRestApi(Properties props, Injector injector, ctx.result("Started Replication...., this might take 60 seconds...."); }); + app.get("/restart", ctx -> { + log.info("Restarting sink connector"); + ClickHouseDebeziumEmbeddedApplication.stop(); + + finalProps.putAll(userProperties); + CompletableFuture cf = ClickHouseDebeziumEmbeddedApplication.startDebeziumEventLoop(injector, finalProps); + ctx.result("Started Replication...."); + + }); } // Stop the javalin server public static void stop() { diff --git a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCapture.java b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCapture.java index bfc7d38f0..23e966bd2 100644 --- a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCapture.java +++ b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCapture.java @@ -1,6 +1,5 @@ package com.altinity.clickhouse.debezium.embedded.cdc; -import com.altinity.clickhouse.debezium.embedded.api.DebeziumEmbeddedRestApi; import com.altinity.clickhouse.debezium.embedded.common.PropertiesHelper; import com.altinity.clickhouse.debezium.embedded.config.SinkConnectorLightWeightConfig; import com.altinity.clickhouse.debezium.embedded.ddl.parser.DDLParserService; @@ -24,6 +23,7 @@ import io.debezium.engine.DebeziumEngine; import io.debezium.engine.spi.OffsetCommitPolicy; import io.debezium.storage.jdbc.offset.JdbcOffsetBackingStoreConfig; +import org.apache.commons.lang3.tuple.Pair; import org.apache.kafka.connect.data.Field; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.source.SourceRecord; @@ -85,35 +85,20 @@ public DebeziumChangeEventCapture() { singleThreadDebeziumEventExecutor = Executors.newFixedThreadPool(1); } - // Function to store the DBwriter instance for a given database. - private Map dbWriterMap = new HashMap<>(); - - // Function to retrieve the DBWriter from dbWriterMap for a given database. - // if it exists or create a new one. - private BaseDbWriter getDbWriter(String databaseName, ClickHouseSinkConnectorConfig config) { - if(dbWriterMap.containsKey(databaseName)) { - return dbWriterMap.get(databaseName); - } else { - DBCredentials dbCredentials = parseDBConfiguration(config); - String jdbcUrl = BaseDbWriter.getConnectionString(dbCredentials.getHostName(), dbCredentials.getPort(), - databaseName); - ClickHouseConnection conn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", - dbCredentials.getUserName(), dbCredentials.getPassword(), config); - BaseDbWriter writer = new BaseDbWriter(dbCredentials.getHostName(), dbCredentials.getPort(), - databaseName, dbCredentials.getUserName(), - dbCredentials.getPassword(), config, conn); - dbWriterMap.put(databaseName, writer); - return writer; - } - } + /** + * Function to perform DDL operation on the main thread. + * @param DDL DDL to be executed. + * @param props + * @param sr + * @param config + */ private void performDDLOperation(String DDL, Properties props, SourceRecord sr, ClickHouseSinkConnectorConfig config) { String databaseName = "system"; if(sr != null && sr.key() != null) { if(sr.key() instanceof Struct) { Struct keyStruct = (Struct) sr.key(); - //System.out.println("Do something"); String recordDbName = (String) keyStruct.get("databaseName"); if(recordDbName != null && recordDbName.isEmpty() == false) { databaseName = recordDbName; @@ -132,8 +117,6 @@ private void performDDLOperation(String DDL, Properties props, SourceRecord sr, databaseName, dbCredentials.getUserName(), dbCredentials.getPassword(), config, conn); } - //BaseDbWriter writer = getDbWriter(databaseName, config); - StringBuffer clickHouseQuery = new StringBuffer(); AtomicBoolean isDropOrTruncate = new AtomicBoolean(false); @@ -148,7 +131,6 @@ private void performDDLOperation(String DDL, Properties props, SourceRecord sr, log.info("Executed Source DB DDL: " + DDL + " Snapshot:" + isSnapshotDDL(sr)); } - long currentTime = System.currentTimeMillis(); boolean ddlProcessingResult = true; Metrics.updateDdlMetrics(DDL, currentTime, 0, ddlProcessingResult); @@ -173,7 +155,6 @@ private void performDDLOperation(String DDL, Properties props, SourceRecord sr, } catch (Exception e) { log.error("Error running DDL Query: " + e); ddlProcessingResult = false; - //throw new RuntimeException(e); } try { @@ -188,7 +169,6 @@ private void performDDLOperation(String DDL, Properties props, SourceRecord sr, Metrics.updateDdlMetrics(DDL, currentTime, elapsedTime, ddlProcessingResult); } - /** * Function to process every change event record * as received from Debezium @@ -253,8 +233,6 @@ private ClickHouseStruct processEveryChangeRecord(Properties props, ChangeEvent< } } - String value = String.valueOf(record.value()); - //log.debug(String.format("Record %s", value)); } catch (Exception e) { log.error("Exception processing record", e); } @@ -321,73 +299,111 @@ private boolean checkIfDDLNeedsToBeIgnored(Properties props, SourceRecord sr, At private void createDatabaseForDebeziumStorage(ClickHouseSinkConnectorConfig config, Properties props) { try { DBCredentials dbCredentials = parseDBConfiguration(config); - //if (writer == null) { - String jdbcUrl = BaseDbWriter.getConnectionString(dbCredentials.getHostName(), dbCredentials.getPort(), - dbCredentials.getDatabase()); - ClickHouseConnection conn = BaseDbWriter.createConnection(jdbcUrl, "Client_1",dbCredentials.getUserName(), dbCredentials.getPassword(), config); - BaseDbWriter writer = new BaseDbWriter(dbCredentials.getHostName(), dbCredentials.getPort(), - dbCredentials.getDatabase(), dbCredentials.getUserName(), + + String jdbcUrl = BaseDbWriter.getConnectionString(dbCredentials.getHostName(), dbCredentials.getPort(), + "system"); + ClickHouseConnection conn = BaseDbWriter.createConnection(jdbcUrl, "Client_1",dbCredentials.getUserName(), dbCredentials.getPassword(), config); + BaseDbWriter writer = new BaseDbWriter(dbCredentials.getHostName(), dbCredentials.getPort(), + "system", dbCredentials.getUserName(), dbCredentials.getPassword(), config, conn); - //} - - String tableName = props.getProperty(JdbcOffsetBackingStoreConfig.OFFSET_STORAGE_PREFIX + - JdbcOffsetBackingStoreConfig.PROP_TABLE_NAME.name()); - if(tableName.contains(".")) { - String[] dbTableNameArray = tableName.split("\\."); - if(dbTableNameArray.length >= 2) { - String dbName = dbTableNameArray[0].replace("\"", ""); - String createDbQuery = String.format("create database if not exists %s", dbName); - log.info("CREATING DEBEZIUM STORAGE Database: " + createDbQuery); - writer.executeQuery(createDbQuery); - - - // Also create view. - String view = " CREATE VIEW IF NOT EXISTS %s.show_replica_status\n" + - " AS\n" + - " SELECT\n" + - " now() - fromUnixTimestamp(JSONExtractUInt(offset_val, 'ts_sec')) AS seconds_behind_source,\n" + - " toDateTime(fromUnixTimestamp(JSONExtractUInt(offset_val, 'ts_sec')), 'UTC') AS utc_time,\n" + - " fromUnixTimestamp(JSONExtractUInt(offset_val, 'ts_sec')) AS local_time,\n" + - " *\n" + - " FROM %s\n" + - " FINAL"; - String formattedView = String.format(view, dbName, tableName); - try { - writer.executeQuery(formattedView); - } catch(Exception e) { - log.error("**** Error creating VIEW **** " + formattedView); - } - } - } + + Pair tableNameDatabaseName = getDebeziumStorageDatabaseName(props); + String databaseName = tableNameDatabaseName.getRight(); + + String createDbQuery = String.format("create database if not exists %s", databaseName); + log.info("CREATING DEBEZIUM STORAGE Database: " + createDbQuery); + writer.executeQuery(createDbQuery); + } catch(Exception e) { log.error("Error creating Debezium storage database", e); } } /** - * Function to get the status of Debezium storage. + * Function to create view for show_replica_status + * @param config + * @param props + */ + private void createViewForShowReplicaStatus(ClickHouseSinkConnectorConfig config, Properties props) { + String view = props.getProperty(ClickHouseSinkConnectorConfigVariables.REPLICA_STATUS_VIEW.toString()); + if(view == null || view.isEmpty() == true) { + log.warn("Skipping creating view for replica_status as the query was not provided in configuration"); + return; + } + DBCredentials dbCredentials = parseDBConfiguration(config); + + String jdbcUrl = BaseDbWriter.getConnectionString(dbCredentials.getHostName(), dbCredentials.getPort(), + "system"); + ClickHouseConnection conn = BaseDbWriter.createConnection(jdbcUrl, "Client_1",dbCredentials.getUserName(), dbCredentials.getPassword(), config); + BaseDbWriter writer = new BaseDbWriter(dbCredentials.getHostName(), dbCredentials.getPort(), + "system", dbCredentials.getUserName(), + dbCredentials.getPassword(), config, conn); + Pair tableNameDatabaseName = getDebeziumStorageDatabaseName(props); + + String tableName = tableNameDatabaseName.getLeft(); + String dbName = tableNameDatabaseName.getRight(); + + String formattedView = String.format(view, dbName, dbName + "." + tableName); + // Remove quotes. + formattedView = formattedView.replace("\"", ""); + try { + writer.executeQuery(formattedView); + } catch(Exception e) { + log.error("**** Error creating VIEW **** " + formattedView); + } + } + + /** + * * @param props * @return */ - public String getDebeziumStorageStatus(ClickHouseSinkConnectorConfig config, Properties props) throws SQLException { - String response = ""; + private Pair getDebeziumStorageDatabaseName(Properties props) { + + String tableName = props.getProperty(JdbcOffsetBackingStoreConfig.OFFSET_STORAGE_PREFIX + JdbcOffsetBackingStoreConfig.PROP_TABLE_NAME.name()); + // if tablename is dbname.tablename and contains a dot. + String databaseName = "system"; + // split tablename with dot. + if(tableName.contains(".")) { + String[] dbTableNameArray = tableName.split("\\."); + if(dbTableNameArray.length >= 2) { + databaseName = dbTableNameArray[0].replace("\"", ""); + tableName = dbTableNameArray[1].replace("\"", ""); + } + } + + return Pair.of(tableName, databaseName); + } + + /** + * Function to get the status of Debezium storage. + * @param props + * @return + */ + public String getDebeziumStorageStatus(ClickHouseSinkConnectorConfig config, Properties props) throws Exception { + String response = ""; + + Pair tableNameDatabaseName = getDebeziumStorageDatabaseName(props); + String tableName = tableNameDatabaseName.getLeft(); + String databaseName = tableNameDatabaseName.getRight(); + DBCredentials dbCredentials = parseDBConfiguration(config); if (writer == null || writer.getConnection().isClosed() == true) { // Json error string log.error("**** Connection to ClickHouse is not established, re-initiating ****"); String jdbcUrl = BaseDbWriter.getConnectionString(dbCredentials.getHostName(), dbCredentials.getPort(), - dbCredentials.getDatabase()); + databaseName); ClickHouseConnection conn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", dbCredentials.getUserName(), dbCredentials.getPassword(), config); writer = new BaseDbWriter(dbCredentials.getHostName(), dbCredentials.getPort(), - dbCredentials.getDatabase(), dbCredentials.getUserName(), + databaseName, dbCredentials.getUserName(), dbCredentials.getPassword(), config, conn); } //DBCredentials dbCredentials = parseDBConfiguration(config); - String debeziumStorageStatusQuery = String.format("select * from %s limit 1", tableName); + String debeziumStorageStatusQuery = String.format("select * from %s limit 1", databaseName + "." + tableName); ResultSet resultSet = writer.executeQueryWithResultSet(debeziumStorageStatusQuery); if(resultSet != null) { @@ -443,8 +459,12 @@ public long getLatestRecordTimestamp(ClickHouseSinkConnectorConfig config, Prope long result = -1; DBCredentials dbCredentials = parseDBConfiguration(config); + Pair tableNameDatabaseName = getDebeziumStorageDatabaseName(props); + String tableName = tableNameDatabaseName.getLeft(); + String databaseName = tableNameDatabaseName.getRight(); + BaseDbWriter writer = new BaseDbWriter(dbCredentials.getHostName(), dbCredentials.getPort(), - dbCredentials.getDatabase(), dbCredentials.getUserName(), + databaseName, dbCredentials.getUserName(), dbCredentials.getPassword(), config, this.conn); String latestRecordTs = new DebeziumOffsetStorage().getDebeziumLatestRecordTimestamp(props, writer); @@ -477,12 +497,14 @@ public void updateDebeziumStorageStatus(ClickHouseSinkConnectorConfig config, Pr String binlogFile, String binLogPosition, String gtid) throws SQLException, ParseException { - String tableName = props.getProperty(JdbcOffsetBackingStoreConfig.OFFSET_STORAGE_PREFIX + - JdbcOffsetBackingStoreConfig.PROP_TABLE_NAME.name()); + Pair tableNameDatabaseName = getDebeziumStorageDatabaseName(props); + String tableName = tableNameDatabaseName.getLeft(); + String databaseName = tableNameDatabaseName.getRight(); + DBCredentials dbCredentials = parseDBConfiguration(config); BaseDbWriter writer = new BaseDbWriter(dbCredentials.getHostName(), dbCredentials.getPort(), - dbCredentials.getDatabase(), dbCredentials.getUserName(), + databaseName, dbCredentials.getUserName(), dbCredentials.getPassword(), config, this.conn); String offsetValue = new DebeziumOffsetStorage().getDebeziumStorageStatusQuery(props, writer); @@ -508,12 +530,13 @@ public void updateDebeziumStorageStatus(ClickHouseSinkConnectorConfig config, Pr String lsn) throws SQLException, ParseException { - String tableName = props.getProperty(JdbcOffsetBackingStoreConfig.OFFSET_STORAGE_PREFIX + - JdbcOffsetBackingStoreConfig.PROP_TABLE_NAME.name()); + Pair tableNameDatabaseName = getDebeziumStorageDatabaseName(props); + String tableName = tableNameDatabaseName.getLeft(); + String databaseName = tableNameDatabaseName.getRight(); DBCredentials dbCredentials = parseDBConfiguration(config); BaseDbWriter writer = new BaseDbWriter(dbCredentials.getHostName(), dbCredentials.getPort(), - dbCredentials.getDatabase(), dbCredentials.getUserName(), + databaseName, dbCredentials.getUserName(), dbCredentials.getPassword(), config, this.conn); String offsetValue = new DebeziumOffsetStorage().getDebeziumStorageStatusQuery(props, writer); @@ -532,6 +555,14 @@ public void updateDebeziumStorageStatus(ClickHouseSinkConnectorConfig config, Pr public int numRetries = 0; + /** + * + * @param props + * @param debeziumRecordParserService + * @param config + * @throws IOException + * @throws ClassNotFoundException + */ public void setupDebeziumEventCapture(Properties props, DebeziumRecordParserService debeziumRecordParserService, ClickHouseSinkConnectorConfig config) throws IOException, ClassNotFoundException { @@ -576,16 +607,21 @@ public void handle(boolean b, String s, Throwable throwable) { if (b == false) { log.error("Error starting connector" + throwable + " Message:" + s); + if(throwable != null && throwable.getCause() != null && throwable.getCause().getLocalizedMessage() != null) + log.error("Error stating connector: Cause" + throwable.getCause().getLocalizedMessage()); + log.error("Retrying - try number:" + numRetries); if (numRetries++ <= MAX_RETRIES) { try { Thread.sleep(SLEEP_TIME); } catch (InterruptedException e) { + log.error("Error sleeping", e); throw new RuntimeException(e); } try { setupDebeziumEventCapture(props, debeziumRecordParserService, config); } catch (IOException | ClassNotFoundException e) { + log.error("Error setting up debezium event capture", e); throw new RuntimeException(e); } } @@ -599,6 +635,8 @@ public void handle(boolean b, String s, Throwable throwable) { public void connectorStarted() { isReplicationRunning = true; log.debug("Connector started"); + // Create view. + createViewForShowReplicaStatus(config, props); } @Override @@ -722,10 +760,10 @@ DBCredentials parseDBConfiguration(ClickHouseSinkConnectorConfig config) { DBCredentials dbCredentials = new DBCredentials(); dbCredentials.setHostName(config.getString(ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_URL.toString())); - dbCredentials.setDatabase(config.getString(ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_DATABASE.toString())); dbCredentials.setPort(config.getInt(ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_PORT.toString())); dbCredentials.setUserName(config.getString(ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_USER.toString())); dbCredentials.setPassword(config.getString(ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_PASS.toString())); + dbCredentials.setDatabase("system"); return dbCredentials; } @@ -744,7 +782,8 @@ private void setupProcessingThread(ClickHouseSinkConnectorConfig config) { new ThreadFactoryBuilder().setNameFormat("Sink Connector thread-pool-%d").build(); this.executor = new ClickHouseBatchExecutor(config.getInt(ClickHouseSinkConnectorConfigVariables.THREAD_POOL_SIZE.toString()), namedThreadFactory); for(int i = 0; i < config.getInt(ClickHouseSinkConnectorConfigVariables.THREAD_POOL_SIZE.toString()); i++) { - this.executor.scheduleAtFixedRate(new ClickHouseBatchRunnable(this.records, config, new HashMap()), 0, config.getLong(ClickHouseSinkConnectorConfigVariables.BUFFER_FLUSH_TIME.toString()), TimeUnit.MILLISECONDS); + this.executor.scheduleAtFixedRate(new ClickHouseBatchRunnable(this.records, config, new HashMap()), 0, + config.getLong(ClickHouseSinkConnectorConfigVariables.BUFFER_FLUSH_TIME.toString()), TimeUnit.MILLISECONDS); } //this.executor.scheduleAtFixedRate(this.runnable, 0, config.getLong(ClickHouseSinkConnectorConfigVariables.BUFFER_FLUSH_TIME.toString()), TimeUnit.MILLISECONDS); } diff --git a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumOffsetStorage.java b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumOffsetStorage.java index eef531303..0fb0f61c4 100644 --- a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumOffsetStorage.java +++ b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumOffsetStorage.java @@ -86,7 +86,7 @@ public String getDebeziumStorageStatusQuery( */ public String updateBinLogInformation(String record, String binLogFile, String binLogPosition, String gtids) throws ParseException { JSONObject jsonObject = new JSONObject(); - if(record != null || !record.isEmpty()) { + if(record != null && !record.isEmpty()) { jsonObject = (JSONObject) new JSONParser().parse(record); } else { jsonObject.put("ts_sec", System.currentTimeMillis() / 1000); diff --git a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/Constants.java b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/Constants.java index 3972a7b9e..a76f04cb4 100644 --- a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/Constants.java +++ b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/Constants.java @@ -45,7 +45,7 @@ public class Constants { public static final String DROP_TABLE = "DROP TABLE"; - public static final String CREATE_DATABASE = "CREATE DATABASE %s"; + public static final String CREATE_DATABASE = "CREATE DATABASE IF NOT EXISTS %s"; public static final String DROP_COLUMN = "DROP COLUMN %s"; diff --git a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MySQLDDLParserBaseListener.java b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MySQLDDLParserBaseListener.java index bbc266578..1fe8dfaf9 100644 --- a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MySQLDDLParserBaseListener.java +++ b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MySQLDDLParserBaseListener.java @@ -1288,6 +1288,36 @@ public void exitPartitionFunctionList(MySqlParser.PartitionFunctionListContext p } + @Override + public void enterPartitionSystemVersion(MySqlParser.PartitionSystemVersionContext partitionSystemVersionContext) { + + } + + @Override + public void exitPartitionSystemVersion(MySqlParser.PartitionSystemVersionContext partitionSystemVersionContext) { + + } + + @Override + public void enterPartitionSystemVersionDefinitions(MySqlParser.PartitionSystemVersionDefinitionsContext partitionSystemVersionDefinitionsContext) { + + } + + @Override + public void exitPartitionSystemVersionDefinitions(MySqlParser.PartitionSystemVersionDefinitionsContext partitionSystemVersionDefinitionsContext) { + + } + + @Override + public void enterPartitionSystemVersionDefinition(MySqlParser.PartitionSystemVersionDefinitionContext partitionSystemVersionDefinitionContext) { + + } + + @Override + public void exitPartitionSystemVersionDefinition(MySqlParser.PartitionSystemVersionDefinitionContext partitionSystemVersionDefinitionContext) { + + } + @Override public void enterSubPartitionFunctionHash(MySqlParser.SubPartitionFunctionHashContext subPartitionFunctionHashContext) { diff --git a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MySqlDDLParserListenerImpl.java b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MySqlDDLParserListenerImpl.java index 3efa17330..e6b297a84 100644 --- a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MySqlDDLParserListenerImpl.java +++ b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MySqlDDLParserListenerImpl.java @@ -7,6 +7,7 @@ import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfigVariables; +import com.altinity.clickhouse.sink.connector.common.Utils; import io.debezium.ddl.parser.mysql.generated.MySqlParser; import io.debezium.ddl.parser.mysql.generated.MySqlParser.AlterByAddColumnContext; import io.debezium.ddl.parser.mysql.generated.MySqlParser.TableNameContext; @@ -17,10 +18,7 @@ import org.apache.logging.log4j.Logger; import java.time.ZoneId; -import java.util.HashSet; -import java.util.List; -import java.util.ListIterator; -import java.util.Set; +import java.util.*; /** @@ -71,7 +69,26 @@ public ZoneId parseTimeZone() { public void enterCreateDatabase(MySqlParser.CreateDatabaseContext createDatabaseContext) { for (ParseTree tree : createDatabaseContext.children) { if (tree instanceof MySqlParser.UidContext) { - this.query.append(String.format(Constants.CREATE_DATABASE, tree.getText())); + + String databaseName = tree.getText(); + if(!databaseName.isEmpty()) { + // Check if the database is overridden + Map sourceToDestinationMap = new HashMap<>(); + + try { + if (this.config.getString(ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_DATABASE_OVERRIDE_MAP.toString()) != null) + sourceToDestinationMap = Utils.parseSourceToDestinationDatabaseMap(this.config. + getString(ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_DATABASE_OVERRIDE_MAP.toString())); + } catch(Exception e) { + log.error("enterCreateDatabase: Error parsing source to destination database map:" + e.toString()); + } + + if(sourceToDestinationMap.containsKey(databaseName)) { + this.query.append(String.format(Constants.CREATE_DATABASE, sourceToDestinationMap.get(databaseName))); + } else { + this.query.append(String.format(Constants.CREATE_DATABASE, databaseName)); + } + } } } } @@ -109,8 +126,17 @@ public void enterColumnCreateTable(MySqlParser.ColumnCreateTableContext columnCr Set columnNames = parseCreateTable(columnCreateTableContext, orderByColumns, partitionByColumn); //this.query.append(" Engine=") String isDeletedColumn = IS_DELETED_COLUMN; - if(columnNames.contains(isDeletedColumn)) { - isDeletedColumn = "__" + IS_DELETED_COLUMN; + // Iterate through columnNames and match isDeletedColumn with elements in columnNames. + // remove the backticks from elements in columnNames. + for(String columnName: columnNames) { + if(columnName.contains("`")) { + // replace backticks with empty string. + columnName = columnName.replace("`", ""); + } + if(columnName.equalsIgnoreCase(isDeletedColumn)) { + isDeletedColumn = "__" + IS_DELETED_COLUMN; + break; + } } // Check if the destination is ReplicatedReplacingMergeTree. @@ -128,12 +154,12 @@ public void enterColumnCreateTable(MySqlParser.ColumnCreateTableContext columnCr this.query.append(")"); if(DebeziumChangeEventCapture.isNewReplacingMergeTreeEngine == true) { if(isReplicatedReplacingMergeTree == true) { - this.query.append(String.format("Engine=ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/%s', '{replica}', %s, %s)", tableName, VERSION_COLUMN, isDeletedColumn)); + this.query.append(String.format("Engine=ReplicatedReplacingMergeTree(%s, %s)", VERSION_COLUMN, isDeletedColumn)); } else this.query.append(" Engine=ReplacingMergeTree(").append(VERSION_COLUMN).append(",").append(isDeletedColumn).append(")"); } else { if (isReplicatedReplacingMergeTree == true) { - this.query.append(String.format("Engine=ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/%s', '{replica}', %s)", tableName, VERSION_COLUMN)); + this.query.append(String.format("Engine=ReplicatedReplacingMergeTree(%s)", VERSION_COLUMN)); } else this.query.append(" Engine=ReplacingMergeTree(").append(VERSION_COLUMN).append(")"); } diff --git a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/parser/DataTypeConverter.java b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/parser/DataTypeConverter.java index 69b6558c1..f5b7c2c8f 100644 --- a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/parser/DataTypeConverter.java +++ b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/parser/DataTypeConverter.java @@ -4,7 +4,7 @@ import com.clickhouse.data.ClickHouseDataType; import io.debezium.antlr.DataTypeResolver; import io.debezium.config.CommonConnectorConfig; -import io.debezium.connector.mysql.MySqlValueConverters; +import io.debezium.connector.mysql.jdbc.MySqlValueConverters; import io.debezium.ddl.parser.mysql.generated.MySqlParser; import io.debezium.jdbc.JdbcValueConverters; import io.debezium.jdbc.TemporalPrecisionMode; @@ -34,8 +34,8 @@ public static ClickHouseDataType convert(String columnName, MySqlParser.DataType JdbcValueConverters.DecimalMode.PRECISE, TemporalPrecisionMode.ADAPTIVE, JdbcValueConverters.BigIntUnsignedMode.LONG, - CommonConnectorConfig.BinaryHandlingMode.BYTES - ); + CommonConnectorConfig.BinaryHandlingMode.BYTES, + x ->x, CommonConnectorConfig.EventConvertingFailureHandlingMode.WARN); DataType dataType = initializeDataTypeResolver().resolveDataType(columnDefChild); @@ -51,8 +51,8 @@ public static String convertToString(String columnName, int scale, int precision JdbcValueConverters.DecimalMode.PRECISE, TemporalPrecisionMode.ADAPTIVE, JdbcValueConverters.BigIntUnsignedMode.LONG, - CommonConnectorConfig.BinaryHandlingMode.BYTES - ); + CommonConnectorConfig.BinaryHandlingMode.BYTES, + x ->x, CommonConnectorConfig.EventConvertingFailureHandlingMode.WARN); String convertedDataType = null; diff --git a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/parser/SourceRecordParserService.java b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/parser/SourceRecordParserService.java index b9cf35c67..750eaf058 100644 --- a/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/parser/SourceRecordParserService.java +++ b/sink-connector-lightweight/src/main/java/com/altinity/clickhouse/debezium/embedded/parser/SourceRecordParserService.java @@ -137,6 +137,7 @@ private ClickHouseStruct readBeforeOrAfterSection(Map convertedV } } } catch (ParseException e) { + log.error("Error parsing JSON", e); throw new RuntimeException(e); } diff --git a/sink-connector-lightweight/src/main/resources/config.properties b/sink-connector-lightweight/src/main/resources/config.properties index 495e361ee..7c9c39e2b 100644 --- a/sink-connector-lightweight/src/main/resources/config.properties +++ b/sink-connector-lightweight/src/main/resources/config.properties @@ -1,45 +1,24 @@ name= "engine" -#offset.storage=org.apache.kafka.connect.storage.FileOffsetBackingStore -#offset.storage.file.filename=/tmp/offsets.dat - database.server.name= "clickhouse-ddl" database.server.id= 976 -#database.history= "io.debezium.relational.history.FileDatabaseHistory" -#database.history.file.filename=/tmp/dbhistory.dat -#connector.class= io.debezium.connector.mysql.MySqlConnector converter.schemas.enable= "true" schemas.enable= true topic.prefix=embeddedconnector offset.storage=io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore offset.storage.jdbc.offset.table.name= "default.replica_source_info" -offset.storage.jdbc.url: "jdbc:clickhouse://clickhouse:8123" -offset.storage.jdbc.user: "root" -offset.storage.jdbc.password: "root" -offset.storage.jdbc.offset.table.ddl: "CREATE TABLE %s -( -`id` String, -`offset_key` String, -`offset_val` String, -`record_insert_ts` DateTime, -`record_insert_seq` UInt64, -`_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9)) -) -ENGINE = ReplacingMergeTree(_version) -ORDER BY id -SETTINGS index_granularity = 8192" -offset.storage.jdbc.offset.table.delete: "delete from %s where 1=1" -schema.history.internal: io.debezium.storage.jdbc.history.JdbcSchemaHistory -schema.history.internal.jdbc.url: "jdbc:clickhouse://clickhouse:8123" -schema.history.internal.jdbc.user: "root" -schema.history.internal.jdbc.password: "root" -schema.history.internal.schema.history.table.ddl: "CREATE TABLE %s -(`id` VARCHAR(36) NOT NULL, `history_data` VARCHAR(65000), `history_data_seq` INTEGER, `record_insert_ts` TIMESTAMP NOT NULL, -`record_insert_seq` INTEGER NOT NULL) Engine=ReplacingMergeTree(record_insert_seq) order by id" - -schema.history.internal.schema.history.table.name: "default.replicate_schema_history" - +offset.storage.jdbc.url= "jdbc:clickhouse://clickhouse:8123" +offset.storage.jdbc.user= "root" +offset.storage.jdbc.password= "root" +offset.storage.jdbc.offset.table.ddl="CREATE TABLE %s(`id` String, `offset_key` String, `offset_val` String, `record_insert_ts` DateTime, `record_insert_seq` UInt64, `_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9)))ENGINE = ReplacingMergeTree(_version) ORDER BY id SETTINGS index_granularity = 8192" +offset.storage.jdbc.offset.table.delete="delete from %s where 1=1" +schema.history.internal=io.debezium.storage.jdbc.history.JdbcSchemaHistory +schema.history.internal.jdbc.url="jdbc:clickhouse://clickhouse:8123" +schema.history.internal.jdbc.user="root" +schema.history.internal.jdbc.password="root" +schema.history.internal.schema.history.table.ddl="CREATE TABLE %s(`id` VARCHAR(36) NOT NULL, `history_data` VARCHAR(65000), `history_data_seq` INTEGER, `record_insert_ts` TIMESTAMP NOT NULL, `record_insert_seq` INTEGER NOT NULL) Engine=ReplacingMergeTree(record_insert_seq) order by id" +schema.history.internal.schema.history.table.name="default.replicate_schema_history" auto.create.tables= false replacingmergetree.delete.column=_sign -metrics.enable= true metrics.port= 8083 -snapshot.mode= "initial" \ No newline at end of file +snapshot.mode= "initial" +replica.status.view="CREATE VIEW IF NOT EXISTS %s.show_replica_status AS SELECT now() - fromUnixTimestamp(JSONExtractUInt(offset_val, 'ts_sec')) AS seconds_behind_source, toDateTime(fromUnixTimestamp(JSONExtractUInt(offset_val, 'ts_sec')), 'UTC') AS utc_time, fromUnixTimestamp(JSONExtractUInt(offset_val, 'ts_sec')) AS local_time FROM %s FINAL" \ No newline at end of file diff --git a/sink-connector-lightweight/src/main/resources/log4j2.xml b/sink-connector-lightweight/src/main/resources/log4j2.xml index 9321838cb..5e7aae4fa 100644 --- a/sink-connector-lightweight/src/main/resources/log4j2.xml +++ b/sink-connector-lightweight/src/main/resources/log4j2.xml @@ -2,12 +2,23 @@ + + - + + + + + + - \ No newline at end of file + diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedMySqlDockerIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedMySqlDockerIT.java index 64b7a83ab..6611f9a8e 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedMySqlDockerIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ClickHouseDebeziumEmbeddedMySqlDockerIT.java @@ -40,7 +40,7 @@ // // @BeforeEach // public void startContainers() throws InterruptedException { -// mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") +// mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") // .asCompatibleSubstituteFor("mysql")) // .withDatabaseName("employees").withUsername("root").withPassword("adminpass") // .withInitScript("alter_ddl_add_column.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ITCommon.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ITCommon.java index 9cb7c48db..61b89c384 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ITCommon.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ITCommon.java @@ -4,6 +4,7 @@ import com.altinity.clickhouse.debezium.embedded.config.ConfigLoader; import org.testcontainers.clickhouse.ClickHouseContainer; import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.containers.PostgreSQLContainer; import java.sql.Connection; import java.sql.DriverManager; @@ -28,6 +29,22 @@ static public Connection connectToMySQL(MySQLContainer mySqlContainer) { return conn; } + // Function to connect to Postgres. + static public Connection connectToPostgreSQL(PostgreSQLContainer postgreSQLContainer) { + Connection conn = null; + try { + + String connectionUrl = String.format("jdbc:postgresql://%s:%s/%s?user=%s&password=%s", postgreSQLContainer.getHost(), + postgreSQLContainer.getFirstMappedPort(), + postgreSQLContainer.getDatabaseName(), postgreSQLContainer.getUsername(), postgreSQLContainer.getPassword()); + conn = DriverManager.getConnection(connectionUrl); + + } catch (SQLException ex) { + + } + return conn; + } + static public Properties getDebeziumProperties(MySQLContainer mySqlContainer, ClickHouseContainer clickHouseContainer) throws Exception { // Start the debezium embedded application. @@ -48,7 +65,6 @@ static public Properties getDebeziumProperties(MySQLContainer mySqlContainer, Cl defaultProps.setProperty("clickhouse.server.port", String.valueOf(clickHouseContainer.getFirstMappedPort())); defaultProps.setProperty("clickhouse.server.user", clickHouseContainer.getUsername()); defaultProps.setProperty("clickhouse.server.password", clickHouseContainer.getPassword()); - defaultProps.setProperty("clickhouse.server.database", "employees"); defaultProps.setProperty("offset.storage.jdbc.url", String.format("jdbc:clickhouse://%s:%s", clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort())); diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/MySQLGenerateColumnsTest.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/MySQLGenerateColumnsTest.java index 357de7eab..cb7348279 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/MySQLGenerateColumnsTest.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/MySQLGenerateColumnsTest.java @@ -51,7 +51,7 @@ public class MySQLGenerateColumnsTest { @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") // .withInitScript("data_types.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/MySQLJsonIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/MySQLJsonIT.java new file mode 100644 index 000000000..ccf9409ea --- /dev/null +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/MySQLJsonIT.java @@ -0,0 +1,154 @@ +package com.altinity.clickhouse.debezium.embedded; + +import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; +import com.altinity.clickhouse.debezium.embedded.ddl.parser.MySQLDDLParserService; +import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; +import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; +import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; +import com.clickhouse.jdbc.ClickHouseConnection; +import org.apache.log4j.BasicConfigurator; +import org.junit.Assert; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.testcontainers.clickhouse.ClickHouseContainer; +import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Integration test to validate support for replication of multiple databases. + */ +@Testcontainers +@DisplayName("Integration Test that validates handling of multiple databases") +public class MySQLJsonIT +{ + + protected MySQLContainer mySqlContainer; + static ClickHouseContainer clickHouseContainer; + + @BeforeEach + public void startContainers() throws InterruptedException { + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") + .asCompatibleSubstituteFor("mysql")) + .withDatabaseName("employees").withUsername("root").withPassword("adminpass") + // .withInitScript("data_types.sql") + .withExtraHost("mysql-server", "0.0.0.0") + .waitingFor(new HttpWaitStrategy().forPort(3306)); + + BasicConfigurator.configure(); + mySqlContainer.start(); + Thread.sleep(15000); + } + + static { + clickHouseContainer = new ClickHouseContainer(DockerImageName.parse("clickhouse/clickhouse-server:latest") + .asCompatibleSubstituteFor("clickhouse")) + .withInitScript("init_clickhouse_it.sql") + .withUsername("ch_user") + .withPassword("password") + .withExposedPorts(8123); + + clickHouseContainer.start(); + } + + @DisplayName("Integration Test that validates handle of JSON data type from MySQL") + @Test + public void testMultipleDatabases() throws Exception { + + AtomicReference engine = new AtomicReference<>(); + + Properties props = ITCommon.getDebeziumProperties(mySqlContainer, clickHouseContainer); + // Set the list of databases captured. + props.put("database.whitelist", "employees,test_db,test_db2"); + props.put("database.include.list", "employees,test_db,test_db2"); + + ExecutorService executorService = Executors.newFixedThreadPool(1); + executorService.execute(() -> { + try { + + engine.set(new DebeziumChangeEventCapture()); + engine.get().setup(props, new SourceRecordParserService(), + new MySQLDDLParserService(new ClickHouseSinkConnectorConfig(new HashMap<>()), "test_db"),false); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + Thread.sleep(30000); + Connection conn = ITCommon.connectToMySQL(mySqlContainer); + + conn.createStatement().execute("CREATE DATABASE test_db2"); + Thread.sleep(5000); + // Create a new database + conn.createStatement().execute("CREATE TABLE employees.audience (" + + "id int unsigned NOT NULL AUTO_INCREMENT, " + + "client_id int unsigned NOT NULL, " + + "list_id int unsigned NOT NULL, " + + "status tinyint NOT NULL, " + + "email varchar(200) CHARACTER SET utf16 COLLATE utf16_unicode_ci NOT NULL, " + + "custom_properties JSON, " + + "source tinyint unsigned NOT NULL DEFAULT '0', " + + "created_date datetime DEFAULT NULL, " + + "modified_date datetime DEFAULT NULL, " + + "property_update_date datetime DEFAULT NULL, " + + "PRIMARY KEY (id), " + + "KEY cid_email (client_id,email), " + + "KEY cid (client_id,list_id,status), " + + "KEY contact_created (created_date), " + + "KEY idx_email (email)" + + ") ENGINE=InnoDB CHARSET=utf16 COLLATE=utf16_unicode_ci"); + + + Thread.sleep(5000); + // Insert a new row. + conn.createStatement().execute("INSERT INTO employees.audience (client_id, list_id, status, email, custom_properties, source, created_date, modified_date, property_update_date)" + + " VALUES (1, 100, 1, 'example@example.com', '{\"name\": \"John\", \"age\": 30}', 1, '2024-05-13 12:00:00', '2024-05-13 12:00:00', '2024-05-13 12:00:00')"); + + Thread.sleep(10000); + conn.close(); + + // Create connection to clickhouse and validate if the tables are replicated. + String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), + "system"); + ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", + clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); + + BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), + "system", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + // query clickhouse connection and get data for test_table1 and test_table2 + + + ResultSet rs = writer.executeQueryWithResultSet("SELECT * FROM employees.audience"); + // Validate the data + boolean recordFound = false; + while(rs.next()) { + recordFound = true; + assert rs.getInt("id") == 1; + //assert rs.getString("name").equalsIgnoreCase("test"); + } + Assert.assertTrue(recordFound); + + + + if(engine.get() != null) { + engine.get().stop(); + } + // Files.deleteIfExists(tmpFilePath); + executorService.shutdown(); + + writer.getConnection().close(); + + + } +} diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/OffsetManagementIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/OffsetManagementIT.java index ff33fe472..0c89e3020 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/OffsetManagementIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/OffsetManagementIT.java @@ -1,12 +1,10 @@ package com.altinity.clickhouse.debezium.embedded; import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; -import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumOffsetStorageIT; import com.altinity.clickhouse.debezium.embedded.common.PropertiesHelper; import com.altinity.clickhouse.debezium.embedded.ddl.parser.MySQLDDLParserService; import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; -import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; import org.apache.log4j.BasicConfigurator; import org.junit.Assert; import org.junit.jupiter.api.AfterEach; @@ -22,7 +20,6 @@ import org.testcontainers.utility.DockerImageName; import java.sql.Connection; -import java.sql.ResultSet; import java.util.HashMap; import java.util.Properties; import java.util.concurrent.ExecutorService; @@ -40,7 +37,7 @@ public class OffsetManagementIT { @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") // .withInitScript("data_types.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/PostgresPgoutputMultipleSchemaIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/PostgresPgoutputMultipleSchemaIT.java new file mode 100644 index 000000000..87e78b7b5 --- /dev/null +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/PostgresPgoutputMultipleSchemaIT.java @@ -0,0 +1,145 @@ +package com.altinity.clickhouse.debezium.embedded; + +import com.altinity.clickhouse.debezium.embedded.cdc.DebeziumChangeEventCapture; +import com.altinity.clickhouse.debezium.embedded.ddl.parser.MySQLDDLParserService; +import com.altinity.clickhouse.debezium.embedded.parser.SourceRecordParserService; +import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; +import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; +import com.clickhouse.jdbc.ClickHouseConnection; +import org.junit.Assert; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.testcontainers.Testcontainers; +import org.testcontainers.clickhouse.ClickHouseContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.utility.DockerImageName; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicReference; + +import static com.altinity.clickhouse.debezium.embedded.PostgresProperties.getDefaultProperties; + + +/** + * This is a test for "plugin.name", "pgoutput" + */ + +public class PostgresPgoutputMultipleSchemaIT { + + @Container + public static ClickHouseContainer clickHouseContainer = new ClickHouseContainer(DockerImageName.parse("clickhouse/clickhouse-server:latest") + .asCompatibleSubstituteFor("clickhouse")) + .withInitScript("init_clickhouse_it.sql") + .withUsername("ch_user") + .withPassword("password") + .withExposedPorts(8123); + + @Container + public static PostgreSQLContainer postgreSQLContainer = new PostgreSQLContainer<>("postgres:latest") + .withInitScript("init_postgres.sql") + .withDatabaseName("public") + .withUsername("root") + .withPassword("root") + .withExposedPorts(5432) + .withCommand("postgres -c wal_level=logical") + .withNetworkAliases("postgres").withAccessToHost(true); + + + public Properties getProperties() throws Exception { + + + Properties properties = getDefaultProperties(postgreSQLContainer, clickHouseContainer); + properties.put("plugin.name", "pgoutput" ); + properties.put("plugin.path", "/" ); + properties.put("table.include.list", "public.tm" ); + properties.put("topic.prefix", "test-server" ); + properties.put("slot.max.retries", "6" ); + properties.put("slot.retry.delay.ms", "5000" ); + properties.put("database.allowPublicKeyRetrieval", "true" ); + properties.put("schema.include.list", "public,public2"); + properties.put("table.include.list", "public.tm,public2.tm2" ); + return properties; + } + + @Test + @DisplayName("Integration Test - Validates postgresql replication works with multiple schemas") + public void testMultipleSchemaReplication() throws Exception { + Network network = Network.newNetwork(); + + postgreSQLContainer.withNetwork(network).start(); + clickHouseContainer.withNetwork(network).start(); + Thread.sleep(10000); + + Testcontainers.exposeHostPorts(postgreSQLContainer.getFirstMappedPort()); + AtomicReference engine = new AtomicReference<>(); + + ExecutorService executorService = Executors.newFixedThreadPool(1); + executorService.execute(() -> { + try { + + engine.set(new DebeziumChangeEventCapture()); + engine.get().setup(getProperties(), new SourceRecordParserService(), + new MySQLDDLParserService(new ClickHouseSinkConnectorConfig(new HashMap<>()), "system"), false); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + Thread.sleep(50000); + + // Create a postgres connection and insert new records + // to public2.tm CREATE TABLE "public2.tm" (id uuid DEFAULT gen_random_uuid() NOT NULL PRIMARY KEY, secid uuid, acc_id uuid); + Connection postgresConn = ITCommon.connectToPostgreSQL(postgreSQLContainer); + postgresConn.createStatement().execute("insert into public2.tm2 (id, secid, acc_id) values (gen_random_uuid(), gen_random_uuid(), gen_random_uuid())"); + postgresConn.close(); + + Thread.sleep(10000); + + // Create connection. + String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), + "public"); + ClickHouseConnection conn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", + clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); + + BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), + "public", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, conn); + Map tmColumns = writer.getColumnsDataTypesForTable("tm"); + Assert.assertTrue(tmColumns.size() == 22); + Assert.assertTrue(tmColumns.get("id").equalsIgnoreCase("UUID")); + Assert.assertTrue(tmColumns.get("secid").equalsIgnoreCase("Nullable(UUID)")); + //Assert.assertTrue(tmColumns.get("am").equalsIgnoreCase("Nullable(Decimal(21,5))")); + Assert.assertTrue(tmColumns.get("created").equalsIgnoreCase("Nullable(DateTime64(6))")); + + + int tmCount = 0; + ResultSet chRs = writer.getConnection().prepareStatement("select count(*) from public.tm final").executeQuery(); + while(chRs.next()) { + tmCount = chRs.getInt(1); + } + + Assert.assertTrue(tmCount == 2); + + + int tm2Count = 0; + ResultSet chRs2 = writer.getConnection().prepareStatement("select count(*) from public.tm2 final").executeQuery(); + while(chRs2.next()) { + tm2Count = chRs2.getInt(1); + } + Assert.assertTrue(tm2Count == 1); + + if(engine.get() != null) { + engine.get().stop(); + } + // Files.deleteIfExists(tmpFilePath); + executorService.shutdown(); + + } +} diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTClickHouse22TIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTClickHouse22TIT.java index 55d5a51ea..08ea89ba8 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTClickHouse22TIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTClickHouse22TIT.java @@ -46,7 +46,7 @@ public void startContainers() throws InterruptedException { zookeeperContainer.withNetwork(network).withNetworkAliases("zookeeper"); zookeeperContainer.start(); - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("data_types_test.sql") @@ -76,7 +76,7 @@ public void startContainers() throws InterruptedException { @CsvSource({ "clickhouse/clickhouse-server:22.3" }) - @DisplayName("Test that validates creation of Replicated Replacing Merge Tree") + @DisplayName("Test that validates creation of Replicated Replacing Merge Tree on ClickHouse 22.3 ") public void testReplicatedRMTAutoCreate(String clickHouseServerVersion) throws Exception { AtomicReference engine = new AtomicReference<>(); diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTDDLClickHouse22TIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTDDLClickHouse22TIT.java index dcb7bab50..21e839d84 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTDDLClickHouse22TIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTDDLClickHouse22TIT.java @@ -47,7 +47,7 @@ public void startContainers() throws InterruptedException { zookeeperContainer.withNetwork(network).withNetworkAliases("zookeeper"); zookeeperContainer.start(); - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("data_types_test.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTDDLIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTDDLIT.java index c2dcb4676..fc5278dd8 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTDDLIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTDDLIT.java @@ -47,7 +47,7 @@ public void startContainers() throws InterruptedException { zookeeperContainer.withNetwork(network).withNetworkAliases("zookeeper"); zookeeperContainer.start(); - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("data_types_test.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTIT.java index 428354816..65a385561 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ReplicatedRMTIT.java @@ -47,7 +47,7 @@ public void startContainers() throws InterruptedException { zookeeperContainer.withNetwork(network).withNetworkAliases("zookeeper"); zookeeperContainer.start(); - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("data_types_test.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/BatchRetryOnFailureIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/BatchRetryOnFailureIT.java index 27f3c3325..ce1aacd5f 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/BatchRetryOnFailureIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/BatchRetryOnFailureIT.java @@ -47,7 +47,7 @@ public class BatchRetryOnFailureIT { .withExposedPorts(8123); @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("datetime.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/ClickHouseDelayedStartIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/ClickHouseDelayedStartIT.java index 3fd472ce2..bdc18d15d 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/ClickHouseDelayedStartIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/ClickHouseDelayedStartIT.java @@ -50,7 +50,7 @@ public class ClickHouseDelayedStartIT { .withExposedPorts(8123); @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("datetime.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DatabaseOverrideIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DatabaseOverrideIT.java new file mode 100644 index 000000000..c711c9ff0 --- /dev/null +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DatabaseOverrideIT.java @@ -0,0 +1,158 @@ +package com.altinity.clickhouse.debezium.embedded.cdc; + +import com.altinity.clickhouse.debezium.embedded.AppInjector; +import com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication; +import com.altinity.clickhouse.debezium.embedded.ITCommon; +import com.altinity.clickhouse.debezium.embedded.api.DebeziumEmbeddedRestApi; +import com.altinity.clickhouse.debezium.embedded.ddl.parser.DDLParserService; +import com.altinity.clickhouse.debezium.embedded.parser.DebeziumRecordParserService; +import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; +import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; +import com.clickhouse.jdbc.ClickHouseConnection; +import com.google.inject.Guice; +import com.google.inject.Injector; +import org.apache.log4j.BasicConfigurator; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.clickhouse.ClickHouseContainer; +import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.utility.DockerImageName; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.HashMap; +import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static com.altinity.clickhouse.debezium.embedded.ITCommon.getDebeziumProperties; +import static org.junit.Assert.assertTrue; + +public class DatabaseOverrideIT { + + private static final Logger log = LoggerFactory.getLogger(DatabaseOverrideIT.class); + + + protected MySQLContainer mySqlContainer; + + @Container + public static ClickHouseContainer clickHouseContainer = new ClickHouseContainer(DockerImageName.parse("clickhouse/clickhouse-server:latest") + .asCompatibleSubstituteFor("clickhouse")) + .withInitScript("init_clickhouse_schema_only_column_timezone.sql") + // .withCopyFileToContainer(MountableFile.forClasspathResource("config.xml"), "/etc/clickhouse-server/config.d/config.xml") + .withUsername("ch_user") + .withPassword("password") + .withExposedPorts(8123); + + @BeforeEach + public void startContainers() throws InterruptedException { + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") + .asCompatibleSubstituteFor("mysql")) + .withDatabaseName("employees").withUsername("root").withPassword("adminpass") +// .withInitScript("15k_tables_mysql.sql") + .withExtraHost("mysql-server", "0.0.0.0") + .waitingFor(new HttpWaitStrategy().forPort(3306)); + + BasicConfigurator.configure(); + mySqlContainer.start(); + clickHouseContainer.start(); + Thread.sleep(35000); + } + + + @DisplayName("Test that validates overriding database name in ClickHouse") + @Test + public void testDatabaseOverride() throws Exception { + + Injector injector = Guice.createInjector(new AppInjector()); + + Properties props = getDebeziumProperties(mySqlContainer, clickHouseContainer); + props.setProperty("snapshot.mode", "schema_only"); + props.setProperty("schema.history.internal.store.only.captured.tables.ddl", "true"); + props.setProperty("schema.history.internal.store.only.captured.databases.ddl", "true"); + props.setProperty("clickhouse.database.override.map", "employees:employees2, products:productsnew"); + props.setProperty("database.include.list", "employees, products, customers"); + + // Override clickhouse server timezone. + ClickHouseDebeziumEmbeddedApplication clickHouseDebeziumEmbeddedApplication = new ClickHouseDebeziumEmbeddedApplication(); + + + ExecutorService executorService = Executors.newFixedThreadPool(1); + executorService.execute(() -> { + try { + clickHouseDebeziumEmbeddedApplication.start(injector.getInstance(DebeziumRecordParserService.class), + injector.getInstance(DDLParserService.class), props, false); + DebeziumEmbeddedRestApi.startRestApi(props, injector, clickHouseDebeziumEmbeddedApplication.getDebeziumEventCapture() + , new Properties()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + }); + + Thread.sleep(25000); + + // Employees table + Connection conn = ITCommon.connectToMySQL(mySqlContainer); + conn.prepareStatement("create table `newtable`(col1 varchar(255) not null, col2 int, col3 int, primary key(col1))").execute(); + + // Insert a new row in the table + conn.prepareStatement("insert into newtable values('a', 1, 1)").execute(); + + + conn.prepareStatement("create database products").execute(); + conn.prepareStatement("create table products.prodtable(col1 varchar(255) not null, col2 int, col3 int, primary key(col1))").execute(); + conn.prepareStatement("insert into products.prodtable values('a', 1, 1)").execute(); + + conn.prepareStatement("create database customers").execute(); + conn.prepareStatement("create table customers.custtable(col1 varchar(255) not null, col2 int, col3 int, primary key(col1))").execute(); + conn.prepareStatement("insert into customers.custtable values('a', 1, 1)").execute(); + + Thread.sleep(10000); + + // Validate in Clickhouse the last record written is 29999 + String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), + "system"); + ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", + clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); + BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), + "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + + long col2 = 0L; + ResultSet version1Result = writer.executeQueryWithResultSet("select col2 from employees2.newtable final where col1 = 'a'"); + while(version1Result.next()) { + col2 = version1Result.getLong("col2"); + } + Thread.sleep(10000); + assertTrue(col2 == 1); + + long productsCol2 = 0L; + ResultSet productsVersionResult = writer.executeQueryWithResultSet("select col2 from productsnew.prodtable final where col1 = 'a'"); + while(productsVersionResult.next()) { + productsCol2 = productsVersionResult.getLong("col2"); + } + assertTrue(productsCol2 == 1); + Thread.sleep(10000); + + long customersCol2 = 0L; + ResultSet customersVersionResult = writer.executeQueryWithResultSet("select col2 from customers.custtable final where col1 = 'a'"); + while(customersVersionResult.next()) { + customersCol2 = customersVersionResult.getLong("col2"); + } + assertTrue(customersCol2 == 1); + + + + clickHouseDebeziumEmbeddedApplication.getDebeziumEventCapture().engine.close(); + + conn.close(); + // Files.deleteIfExists(tmpFilePath); + executorService.shutdown(); + } +} diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/Debezium15KTablesLoadIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/Debezium15KTablesLoadIT.java index aa87d3cce..6e5948729 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/Debezium15KTablesLoadIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/Debezium15KTablesLoadIT.java @@ -45,7 +45,7 @@ public class Debezium15KTablesLoadIT { .withExposedPorts(8123); @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("15k_tables_mysql.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCaptureIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCaptureIT.java index f60e5707c..c8a5bf35c 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCaptureIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumChangeEventCaptureIT.java @@ -91,7 +91,7 @@ public void testDeleteOffsetStorageRow2() { .withExposedPorts(8123); @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") // .withInitScript("15k_tables_mysql.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumOffsetStorageIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumOffsetStorageIT.java deleted file mode 100644 index e3dfe83af..000000000 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumOffsetStorageIT.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.altinity.clickhouse.debezium.embedded.cdc; - -import org.testcontainers.junit.jupiter.Testcontainers; - -@Testcontainers -public class DebeziumOffsetStorageIT { -} diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumStorageViewIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumStorageViewIT.java index 2a150048d..368aec496 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumStorageViewIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DebeziumStorageViewIT.java @@ -48,7 +48,7 @@ public class DebeziumStorageViewIT { .withExposedPorts(8123); @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("datetime.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DestinationDBColumnMissingIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DestinationDBColumnMissingIT.java new file mode 100644 index 000000000..3f4c8d73e --- /dev/null +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/DestinationDBColumnMissingIT.java @@ -0,0 +1,134 @@ +package com.altinity.clickhouse.debezium.embedded.cdc; + +import com.altinity.clickhouse.debezium.embedded.AppInjector; +import com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication; +import com.altinity.clickhouse.debezium.embedded.ITCommon; +import com.altinity.clickhouse.debezium.embedded.api.DebeziumEmbeddedRestApi; +import com.altinity.clickhouse.debezium.embedded.ddl.parser.DDLParserService; +import com.altinity.clickhouse.debezium.embedded.parser.DebeziumRecordParserService; +import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; +import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; +import com.clickhouse.jdbc.ClickHouseConnection; +import com.google.inject.Guice; +import com.google.inject.Injector; +import org.apache.log4j.BasicConfigurator; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.clickhouse.ClickHouseContainer; +import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.util.HashMap; +import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static com.altinity.clickhouse.debezium.embedded.ITCommon.getDebeziumProperties; +import static org.junit.Assert.assertTrue; + +@Testcontainers +@Disabled +@DisplayName("Test that validates behavior when source column is not present in ClickHouse(Destination)") +public class DestinationDBColumnMissingIT { + private static final Logger log = LoggerFactory.getLogger(MultipleUpdatesWSameTimestampIT.class); + + + protected MySQLContainer mySqlContainer; + + @Container + public static ClickHouseContainer clickHouseContainer = new ClickHouseContainer(DockerImageName.parse("clickhouse/clickhouse-server:latest") + .asCompatibleSubstituteFor("clickhouse")) + .withInitScript("init_clickhouse_column_mismatch.sql") + // .withCopyFileToContainer(MountableFile.forClasspathResource("config.xml"), "/etc/clickhouse-server/config.d/config.xml") + .withUsername("ch_user") + .withPassword("password") + .withExposedPorts(8123); + @BeforeEach + public void startContainers() throws InterruptedException { + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") + .asCompatibleSubstituteFor("mysql")) + .withDatabaseName("employees").withUsername("root").withPassword("adminpass") +// .withInitScript("15k_tables_mysql.sql") + .withExtraHost("mysql-server", "0.0.0.0") + .waitingFor(new HttpWaitStrategy().forPort(3306)); + + BasicConfigurator.configure(); + mySqlContainer.start(); + clickHouseContainer.start(); + Thread.sleep(35000); + } + + + @Test + public void testColumnMismatch() throws Exception { + + Injector injector = Guice.createInjector(new AppInjector()); + + Properties props = getDebeziumProperties(mySqlContainer, clickHouseContainer); + props.setProperty("snapshot.mode", "schema_only"); + props.setProperty("schema.history.internal.store.only.captured.tables.ddl", "true"); + props.setProperty("schema.history.internal.store.only.captured.databases.ddl", "true"); + + // Override clickhouse server timezone. + ClickHouseDebeziumEmbeddedApplication clickHouseDebeziumEmbeddedApplication = new ClickHouseDebeziumEmbeddedApplication(); + + + ExecutorService executorService = Executors.newFixedThreadPool(1); + executorService.execute(() -> { + try { + clickHouseDebeziumEmbeddedApplication.start(injector.getInstance(DebeziumRecordParserService.class), + injector.getInstance(DDLParserService.class), props, false); + DebeziumEmbeddedRestApi.startRestApi(props, injector, clickHouseDebeziumEmbeddedApplication.getDebeziumEventCapture() + , new Properties()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + }); + + Thread.sleep(25000); + + Connection conn = ITCommon.connectToMySQL(mySqlContainer); + conn.prepareStatement("create table `newtable`(col111 varchar(255) not null, primary key(col111))").execute(); + // Insert a new row in the table + conn.prepareStatement("insert into newtable values('a')").execute(); + + + Thread.sleep(10000); + + // Validate in Clickhouse the last record written is 29999 + String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), + "employees"); + ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", + clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); + BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), + "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + + long col2 = 1L; + ResultSet version1Result = writer.executeQueryWithResultSet("select col2 from newtable final where col1 = 'a'"); + while(version1Result.next()) { + col2 = version1Result.getLong("col2"); + } + Thread.sleep(10000); + + while(true) { + ; + } + +// assertTrue(col2 == 1); +// clickHouseDebeziumEmbeddedApplication.getDebeziumEventCapture().engine.close(); +// +// conn.close(); +// executorService.shutdown(); + } + +} diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/MultipleUpdatesWSameTimestampIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/MultipleUpdatesWSameTimestampIT.java index 0ef1ca3a4..14b8f3170 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/MultipleUpdatesWSameTimestampIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/MultipleUpdatesWSameTimestampIT.java @@ -59,7 +59,7 @@ public class MultipleUpdatesWSameTimestampIT { .withExposedPorts(8123); @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") // .withInitScript("15k_tables_mysql.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/SourceDBColumnMissingIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/SourceDBColumnMissingIT.java new file mode 100644 index 000000000..6ef0bd20a --- /dev/null +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/cdc/SourceDBColumnMissingIT.java @@ -0,0 +1,128 @@ +package com.altinity.clickhouse.debezium.embedded.cdc; + +import com.altinity.clickhouse.debezium.embedded.AppInjector; +import com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication; +import com.altinity.clickhouse.debezium.embedded.ITCommon; +import com.altinity.clickhouse.debezium.embedded.api.DebeziumEmbeddedRestApi; +import com.altinity.clickhouse.debezium.embedded.ddl.parser.DDLParserService; +import com.altinity.clickhouse.debezium.embedded.parser.DebeziumRecordParserService; +import com.altinity.clickhouse.sink.connector.ClickHouseSinkConnectorConfig; +import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; +import com.clickhouse.jdbc.ClickHouseConnection; +import com.google.inject.Guice; +import com.google.inject.Injector; +import org.apache.log4j.BasicConfigurator; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.clickhouse.ClickHouseContainer; +import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.util.HashMap; +import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static com.altinity.clickhouse.debezium.embedded.ITCommon.getDebeziumProperties; +import static org.junit.Assert.assertTrue; + +@Testcontainers +@DisplayName("Test that validates behavior when source columns are not present in ClickHouse.") +public class SourceDBColumnMissingIT { + private static final Logger log = LoggerFactory.getLogger(MultipleUpdatesWSameTimestampIT.class); + + + protected MySQLContainer mySqlContainer; + + @Container + public static ClickHouseContainer clickHouseContainer = new ClickHouseContainer(DockerImageName.parse("clickhouse/clickhouse-server:latest") + .asCompatibleSubstituteFor("clickhouse")) + .withInitScript("init_clickhouse_column_mismatch.sql") + // .withCopyFileToContainer(MountableFile.forClasspathResource("config.xml"), "/etc/clickhouse-server/config.d/config.xml") + .withUsername("ch_user") + .withPassword("password") + .withExposedPorts(8123); + @BeforeEach + public void startContainers() throws InterruptedException { + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") + .asCompatibleSubstituteFor("mysql")) + .withDatabaseName("employees").withUsername("root").withPassword("adminpass") +// .withInitScript("15k_tables_mysql.sql") + .withExtraHost("mysql-server", "0.0.0.0") + .waitingFor(new HttpWaitStrategy().forPort(3306)); + + BasicConfigurator.configure(); + mySqlContainer.start(); + clickHouseContainer.start(); + Thread.sleep(35000); + } + + + @Test + public void testColumnMismatch() throws Exception { + + Injector injector = Guice.createInjector(new AppInjector()); + + Properties props = getDebeziumProperties(mySqlContainer, clickHouseContainer); + props.setProperty("snapshot.mode", "schema_only"); + props.setProperty("schema.history.internal.store.only.captured.tables.ddl", "true"); + props.setProperty("schema.history.internal.store.only.captured.databases.ddl", "true"); + + // Override clickhouse server timezone. + ClickHouseDebeziumEmbeddedApplication clickHouseDebeziumEmbeddedApplication = new ClickHouseDebeziumEmbeddedApplication(); + + + ExecutorService executorService = Executors.newFixedThreadPool(1); + executorService.execute(() -> { + try { + clickHouseDebeziumEmbeddedApplication.start(injector.getInstance(DebeziumRecordParserService.class), + injector.getInstance(DDLParserService.class), props, false); + DebeziumEmbeddedRestApi.startRestApi(props, injector, clickHouseDebeziumEmbeddedApplication.getDebeziumEventCapture() + , new Properties()); + } catch (Exception e) { + throw new RuntimeException(e); + } + + }); + + Thread.sleep(25000); + + Connection conn = ITCommon.connectToMySQL(mySqlContainer); + conn.prepareStatement("create table `newtable`(col1 varchar(255) not null, col2 int, col3 int, primary key(col1))").execute(); + // Insert a new row in the table + conn.prepareStatement("insert into newtable values('a', 1, 1)").execute(); + + + Thread.sleep(10000); + + // Validate in Clickhouse the last record written is 29999 + String jdbcUrl = BaseDbWriter.getConnectionString(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), + "employees"); + ClickHouseConnection chConn = BaseDbWriter.createConnection(jdbcUrl, "Client_1", + clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), new ClickHouseSinkConnectorConfig(new HashMap<>())); + BaseDbWriter writer = new BaseDbWriter(clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort(), + "employees", clickHouseContainer.getUsername(), clickHouseContainer.getPassword(), null, chConn); + + long col2 = 1L; + ResultSet version1Result = writer.executeQueryWithResultSet("select col2 from newtable final where col1 = 'a'"); + while(version1Result.next()) { + col2 = version1Result.getLong("col2"); + } + Thread.sleep(10000); + + assertTrue(col2 == 1); + clickHouseDebeziumEmbeddedApplication.getDebeziumEventCapture().engine.close(); + + conn.close(); + executorService.shutdown(); + } + +} diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/client/SinkConnectorClientRestAPITest.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/client/SinkConnectorClientRestAPITest.java index 6cddf159d..a82932f36 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/client/SinkConnectorClientRestAPITest.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/client/SinkConnectorClientRestAPITest.java @@ -53,7 +53,7 @@ public class SinkConnectorClientRestAPITest { .withExposedPorts(8123); @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("datetime.sql") @@ -74,7 +74,6 @@ public void testRestClient() throws Exception { Properties props = ITCommon.getDebeziumProperties(mySqlContainer, clickHouseContainer); props.setProperty("database.include.list", "datatypes"); - props.setProperty("clickhouse.server.database", "datatypes"); // Override clickhouse server timezone. ClickHouseDebeziumEmbeddedApplication clickHouseDebeziumEmbeddedApplication = new ClickHouseDebeziumEmbeddedApplication(); diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/config/mysql_config.yaml b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/config/mysql_config.yaml index 89052b819..e06e18979 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/config/mysql_config.yaml +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/config/mysql_config.yaml @@ -8,7 +8,6 @@ clickhouse.server.url: "localhost" clickhouse.server.user: "root" clickhouse.server.password: "root" clickhouse.server.port: "8123" -clickhouse.server.database: "test" database.allowPublicKeyRetrieval: "true" snapshot.mode: "schema_only" connector.class: "io.debezium.connector.mysql.MySqlConnector" \ No newline at end of file diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableAddColumnIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableAddColumnIT.java index 219c6dcb8..dc968d75f 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableAddColumnIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableAddColumnIT.java @@ -28,7 +28,7 @@ public class AlterTableAddColumnIT extends DDLBaseIT { @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("alter_ddl_add_column.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableChangeColumnIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableChangeColumnIT.java index 53835230c..799bb662a 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableChangeColumnIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableChangeColumnIT.java @@ -28,7 +28,7 @@ public class AlterTableChangeColumnIT extends DDLBaseIT { @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("alter_ddl_change_column.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableModifyColumnIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableModifyColumnIT.java index 0689ccbdc..efb67010f 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableModifyColumnIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AlterTableModifyColumnIT.java @@ -28,7 +28,7 @@ public class AlterTableModifyColumnIT extends DDLBaseIT { @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("alter_ddl_modify_column.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AutoCreateTableIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AutoCreateTableIT.java index 9dfcc505b..898f1bb47 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AutoCreateTableIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/AutoCreateTableIT.java @@ -33,7 +33,7 @@ public class AutoCreateTableIT { @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") // .withInitScript("data_types.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesIT.java index e73ec7860..20c230d3c 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesIT.java @@ -30,7 +30,7 @@ public class CreateTableDataTypesIT extends DDLBaseIT { @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("data_types.sql") @@ -53,7 +53,6 @@ public void testCreateTable() throws Exception { Properties props = getDebeziumProperties(); props.setProperty("database.include.list", "datatypes"); - props.setProperty("clickhouse.server.database", "datatypes"); engine.set(new DebeziumChangeEventCapture()); engine.get().setup(getDebeziumProperties(), new SourceRecordParserService(), diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesTimeZoneIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesTimeZoneIT.java index ec4c6427c..93bdc1dc6 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesTimeZoneIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/CreateTableDataTypesTimeZoneIT.java @@ -44,7 +44,7 @@ public class CreateTableDataTypesTimeZoneIT { @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("data_types.sql") @@ -67,7 +67,6 @@ public void testCreateTable() throws Exception { Properties props = ITCommon.getDebeziumProperties(mySqlContainer, clickHouseContainer); props.setProperty("database.include.list", "datatypes"); - props.setProperty("clickhouse.server.database", "datatypes"); engine.set(new DebeziumChangeEventCapture()); engine.get().setup(ITCommon.getDebeziumProperties(mySqlContainer, clickHouseContainer), new SourceRecordParserService(), diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DDLBaseIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DDLBaseIT.java index e00962377..51409f8e0 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DDLBaseIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DDLBaseIT.java @@ -31,7 +31,7 @@ public class DDLBaseIT { @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("alter_ddl_add_column.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneColumnSchemaOnlyIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneColumnSchemaOnlyIT.java index dee4ac2e7..785145afa 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneColumnSchemaOnlyIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneColumnSchemaOnlyIT.java @@ -44,7 +44,7 @@ public class DateTimeWithTimeZoneColumnSchemaOnlyIT { .withExposedPorts(8123); @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("datetime.sql") @@ -82,7 +82,6 @@ protected Properties getDebeziumProperties() throws Exception { defaultProps.setProperty("clickhouse.server.port", String.valueOf(clickHouseContainer.getFirstMappedPort())); defaultProps.setProperty("clickhouse.server.user", clickHouseContainer.getUsername()); defaultProps.setProperty("clickhouse.server.password", clickHouseContainer.getPassword()); - defaultProps.setProperty("clickhouse.server.database", "employees"); defaultProps.setProperty("offset.storage.jdbc.url", String.format("jdbc:clickhouse://%s:%s", clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort())); diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneIT.java index d4213adbb..599f82372 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneIT.java @@ -45,7 +45,7 @@ public class DateTimeWithTimeZoneIT { @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("datetime.sql") @@ -68,7 +68,6 @@ public void testCreateTable() throws Exception { Properties props = ITCommon.getDebeziumProperties(mySqlContainer, clickHouseContainer); props.setProperty("database.include.list", "datatypes"); - props.setProperty("clickhouse.server.database", "datatypes"); engine.set(new DebeziumChangeEventCapture()); engine.get().setup(ITCommon.getDebeziumProperties(mySqlContainer, clickHouseContainer), new SourceRecordParserService(), diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneSchemaOnlyIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneSchemaOnlyIT.java index 44b376c22..6ade4e240 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneSchemaOnlyIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithTimeZoneSchemaOnlyIT.java @@ -47,7 +47,7 @@ public class DateTimeWithTimeZoneSchemaOnlyIT { @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("datetime.sql") @@ -70,7 +70,6 @@ public void testCreateTable() throws Exception { Properties props = getDebeziumProperties(); props.setProperty("database.include.list", "datatypes"); - props.setProperty("clickhouse.server.database", "datatypes"); engine.set(new DebeziumChangeEventCapture()); engine.get().setup(getDebeziumProperties(), new SourceRecordParserService(), @@ -275,7 +274,6 @@ protected Properties getDebeziumProperties() throws Exception { defaultProps.setProperty("clickhouse.server.port", String.valueOf(clickHouseContainer.getFirstMappedPort())); defaultProps.setProperty("clickhouse.server.user", clickHouseContainer.getUsername()); defaultProps.setProperty("clickhouse.server.password", clickHouseContainer.getPassword()); - defaultProps.setProperty("clickhouse.server.database", "employees"); defaultProps.setProperty("offset.storage.jdbc.url", String.format("jdbc:clickhouse://%s:%s", clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort())); diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithUserProvidedDifferentTimeZoneIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithUserProvidedDifferentTimeZoneIT.java index 2d7ebd02e..34985b83f 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithUserProvidedDifferentTimeZoneIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithUserProvidedDifferentTimeZoneIT.java @@ -47,7 +47,7 @@ public class DateTimeWithUserProvidedDifferentTimeZoneIT { @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("datetime.sql") @@ -290,7 +290,6 @@ protected Properties getDebeziumProperties() throws Exception { defaultProps.setProperty("clickhouse.server.port", String.valueOf(clickHouseContainer.getFirstMappedPort())); defaultProps.setProperty("clickhouse.server.user", clickHouseContainer.getUsername()); defaultProps.setProperty("clickhouse.server.password", clickHouseContainer.getPassword()); - defaultProps.setProperty("clickhouse.server.database", "employees"); defaultProps.setProperty("offset.storage.jdbc.url", String.format("jdbc:clickhouse://%s:%s", clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort())); diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithUserProvidedTimeZoneSchemaOnlyIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithUserProvidedTimeZoneSchemaOnlyIT.java index 8bca5c3ea..cb3bf551e 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithUserProvidedTimeZoneSchemaOnlyIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/DateTimeWithUserProvidedTimeZoneSchemaOnlyIT.java @@ -48,7 +48,7 @@ public class DateTimeWithUserProvidedTimeZoneSchemaOnlyIT { @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("datetime.sql") @@ -72,9 +72,6 @@ public void testCreateTable() throws Exception { Properties props = getDebeziumProperties(); props.setProperty("database.include.list", "datatypes"); - props.setProperty("clickhouse.server.database", "datatypes"); - // Override clickhouse server timezone. - // props.setProperty("clickhouse.datetime.timezone", "UTC"); engine.set(new DebeziumChangeEventCapture()); engine.get().setup(getDebeziumProperties(), new SourceRecordParserService(), @@ -269,7 +266,6 @@ protected Properties getDebeziumProperties() throws Exception { defaultProps.setProperty("clickhouse.server.port", String.valueOf(clickHouseContainer.getFirstMappedPort())); defaultProps.setProperty("clickhouse.server.user", clickHouseContainer.getUsername()); defaultProps.setProperty("clickhouse.server.password", clickHouseContainer.getPassword()); - defaultProps.setProperty("clickhouse.server.database", "employees"); defaultProps.setProperty("offset.storage.jdbc.url", String.format("jdbc:clickhouse://%s:%s", clickHouseContainer.getHost(), clickHouseContainer.getFirstMappedPort())); diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/EmployeesDBIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/EmployeesDBIT.java index f9ef972df..760e452d8 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/EmployeesDBIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/EmployeesDBIT.java @@ -34,7 +34,7 @@ public class EmployeesDBIT extends DDLBaseIT { @BeforeEach @Override public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("employees.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/IsDeletedColumnsIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/IsDeletedColumnsIT.java index b3b199d29..43fcd20ce 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/IsDeletedColumnsIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/IsDeletedColumnsIT.java @@ -34,7 +34,7 @@ public class IsDeletedColumnsIT { @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("data_types.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MultipleDatabaseIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MultipleDatabaseIT.java index 69bcb8755..0126b6293 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MultipleDatabaseIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MultipleDatabaseIT.java @@ -39,7 +39,7 @@ public class MultipleDatabaseIT @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") // .withInitScript("data_types.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MySqlDDLParserListenerImplTest.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MySqlDDLParserListenerImplTest.java index d595be418..0e790698d 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MySqlDDLParserListenerImplTest.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/MySqlDDLParserListenerImplTest.java @@ -606,7 +606,7 @@ public void testCreateDatabase() { String sql = "create database test_ddl"; mySQLDDLParserService.parseSql(sql, "table1", clickHouseQuery); - Assert.assertTrue(clickHouseQuery.toString().equalsIgnoreCase(sql)); + Assert.assertTrue(clickHouseQuery.toString().equalsIgnoreCase("create database if not exists test_ddl")); } @Test @@ -708,7 +708,7 @@ public void testReplicatedReplacingMergeTreeWithoutIsDeletedColumn() { String sql = "CREATE TABLE temporal_types_TIMESTAMP1(`Mid_Value` timestamp(1) NOT NULL) ENGINE=InnoDB;"; mySQLDDLParserService.parseSql(sql, "temporal_types_DATETIME4", clickHouseQuery, isDropOrTruncate); - String expectedResult = "CREATE TABLE datatypes.temporal_types_TIMESTAMP1 ON CLUSTER `{cluster}`(`Mid_Value` DateTime64(1, 0) NOT NULL ,`_version` UInt64,`is_deleted` UInt8)Engine=ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/temporal_types_TIMESTAMP1', '{replica}', _version, is_deleted) ORDER BY tuple()"; + String expectedResult = "CREATE TABLE datatypes.temporal_types_TIMESTAMP1 ON CLUSTER `{cluster}`(`Mid_Value` DateTime64(1, 0) NOT NULL ,`_version` UInt64,`is_deleted` UInt8)Engine=ReplicatedReplacingMergeTree(_version, is_deleted) ORDER BY tuple()"; Assert.assertTrue(clickHouseQuery.toString().equalsIgnoreCase(expectedResult)); @@ -777,6 +777,33 @@ public void testAlterDatabaseAddColumnJson() { Assert.assertTrue(clickHouseQuery.toString().equalsIgnoreCase(clickhouseExpectedQuery)); } + @Test + public void testRenameIsDeletedColumn() { + String sql = "CREATE TABLE `city` (\n" + + " `ID` int NOT NULL AUTO_INCREMENT,\n" + + " `Name` char(35) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '',\n" + + " `CountryCode` char(3) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '',\n" + + " `District` char(20) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '',\n" + + " `Population` int NOT NULL DEFAULT '0',\n" + + " `is_deleted` tinyint(1) DEFAULT '0',\n" + + " PRIMARY KEY (`ID`)\n" + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;"; + + StringBuffer clickHouseQuery = new StringBuffer(); + mySQLDDLParserService.parseSql(sql, "employees", clickHouseQuery); + + Assert.assertTrue(clickHouseQuery.toString().equalsIgnoreCase( + "CREATE TABLE employees.`city`(`ID` Int32 NOT NULL ,`Name` String NOT NULL ,`CountryCode` String NOT NULL ,`District` String NOT NULL ,`Population` Int32 NOT NULL ,`is_deleted` Nullable(Int16),`_version` UInt64,`__is_deleted` UInt8) Engine=ReplacingMergeTree(_version,__is_deleted) ORDER BY (`ID`)")); + + + String sqlWithoutBackticks = "create table city(id int not null auto_increment, Name char(35) , is_deleted tinyint(1) DEFAULT 0, primary key(id))"; + + StringBuffer clickHouseQuery2 = new StringBuffer(); + mySQLDDLParserService.parseSql(sqlWithoutBackticks, "employees", clickHouseQuery2); + + Assert.assertTrue(clickHouseQuery2.toString().equalsIgnoreCase( + "CREATE TABLE employees.city(id Int32 NOT NULL ,Name Nullable(String),is_deleted Nullable(Int16),`_version` UInt64,`__is_deleted` UInt8) Engine=ReplacingMergeTree(_version,__is_deleted) ORDER BY (id)")); + } // @Test // public void deleteData() { // String sql = "DELETE FROM Customers WHERE CustomerName='Alfreds Futterkiste'"; diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/TableOperationsIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/TableOperationsIT.java index 0f8e8bd9d..4d3e7942a 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/TableOperationsIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/TableOperationsIT.java @@ -34,7 +34,7 @@ public class TableOperationsIT { @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("data_types.sql") diff --git a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/TruncateTableIT.java b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/TruncateTableIT.java index 990e6743d..626d68b2b 100644 --- a/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/TruncateTableIT.java +++ b/sink-connector-lightweight/src/test/java/com/altinity/clickhouse/debezium/embedded/ddl/parser/TruncateTableIT.java @@ -35,7 +35,7 @@ public class TruncateTableIT { @BeforeEach public void startContainers() throws InterruptedException { - mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:latest") + mySqlContainer = new MySQLContainer<>(DockerImageName.parse("docker.io/bitnami/mysql:8.0.36") .asCompatibleSubstituteFor("mysql")) .withDatabaseName("employees").withUsername("root").withPassword("adminpass") .withInitScript("truncate_table.sql") diff --git a/sink-connector-lightweight/src/test/resources/config.yml b/sink-connector-lightweight/src/test/resources/config.yml index 4d99d5aea..e39ce2d93 100644 --- a/sink-connector-lightweight/src/test/resources/config.yml +++ b/sink-connector-lightweight/src/test/resources/config.yml @@ -10,7 +10,6 @@ clickhouse.server.url: "clickhouse" clickhouse.server.user: "default" clickhouse.server.password: "root" clickhouse.server.port: "8123" -clickhouse.server.database: "test" database.allowPublicKeyRetrieval: "true" snapshot.mode: "initial" offset.flush.interval.ms: "5000" diff --git a/sink-connector-lightweight/src/test/resources/config_postgres.yml b/sink-connector-lightweight/src/test/resources/config_postgres.yml index fc617e69f..82b8db310 100644 --- a/sink-connector-lightweight/src/test/resources/config_postgres.yml +++ b/sink-connector-lightweight/src/test/resources/config_postgres.yml @@ -11,7 +11,6 @@ clickhouse.server.url: "clickhouse" clickhouse.server.user: "default" clickhouse.server.password: "root" clickhouse.server.port: "8123" -clickhouse.server.database: "public" database.allowPublicKeyRetrieval: "true" snapshot.mode: "initial" offset.flush.interval.ms: "5000" diff --git a/sink-connector-lightweight/src/test/resources/init_clickhouse_column_mismatch.sql b/sink-connector-lightweight/src/test/resources/init_clickhouse_column_mismatch.sql new file mode 100644 index 000000000..6d6060dca --- /dev/null +++ b/sink-connector-lightweight/src/test/resources/init_clickhouse_column_mismatch.sql @@ -0,0 +1,7 @@ +CREATE database datatypes; +CREATE database employees; +CREATE database public; +CREATE database project; + +CREATE TABLE employees.`newtable`(`col1` String NOT NULL,`col2` Int32 NULL,`_version` UInt64,`_sign` UInt8) + Engine=ReplacingMergeTree(_version,_sign) PRIMARY KEY(col1) ORDER BY(col1) \ No newline at end of file diff --git a/sink-connector-lightweight/src/test/resources/init_postgres.sql b/sink-connector-lightweight/src/test/resources/init_postgres.sql index 8066f74c3..2a81ea7ce 100644 --- a/sink-connector-lightweight/src/test/resources/init_postgres.sql +++ b/sink-connector-lightweight/src/test/resources/init_postgres.sql @@ -1,3 +1,4 @@ + CREATE TABLE "tm" ( id uuid DEFAULT gen_random_uuid() NOT NULL PRIMARY KEY, secid uuid, @@ -134,3 +135,6 @@ INSERT INTO protocol_test VALUES ('1778432', '21481203', 'Edward V prisoners Pe --); -- +create schema public2; +set schema 'public2'; +CREATE TABLE "tm2" (id uuid DEFAULT gen_random_uuid() NOT NULL PRIMARY KEY, secid uuid, acc_id uuid); \ No newline at end of file diff --git a/sink-connector-lightweight/tests/integration/env/auto/configs/config.yml b/sink-connector-lightweight/tests/integration/env/auto/configs/config.yml index c98a77ad1..7d4dd6db6 100644 --- a/sink-connector-lightweight/tests/integration/env/auto/configs/config.yml +++ b/sink-connector-lightweight/tests/integration/env/auto/configs/config.yml @@ -9,10 +9,10 @@ clickhouse.server.url: clickhouse clickhouse.server.user: root clickhouse.server.password: root clickhouse.server.port: '8123' -clickhouse.server.database: test database.allowPublicKeyRetrieval: 'true' snapshot.mode: initial offset.flush.interval.ms: '5000' +snapshot.locking.mode: none connector.class: io.debezium.connector.mysql.MySqlConnector offset.storage: io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore offset.storage.jdbc.offset.table.name: altinity_sink_connector.replica_source_info diff --git a/sink-connector-lightweight/tests/integration/env/auto/configs/logs/log4j2.xml b/sink-connector-lightweight/tests/integration/env/auto/configs/logs/log4j2.xml new file mode 100644 index 000000000..5b81cd712 --- /dev/null +++ b/sink-connector-lightweight/tests/integration/env/auto/configs/logs/log4j2.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/sink-connector-lightweight/tests/integration/env/auto/docker-compose.yml b/sink-connector-lightweight/tests/integration/env/auto/docker-compose.yml index 080c314ec..d5af369e7 100644 --- a/sink-connector-lightweight/tests/integration/env/auto/docker-compose.yml +++ b/sink-connector-lightweight/tests/integration/env/auto/docker-compose.yml @@ -3,7 +3,7 @@ version: "2.3" services: mysql-master: - image: docker.io/bitnami/mysql:8.0 + image: docker.io/bitnami/mysql:8.0.36 restart: "no" expose: - "3306" diff --git a/sink-connector-lightweight/tests/integration/env/auto_replicated/configs/logs/log4j2.xml b/sink-connector-lightweight/tests/integration/env/auto_replicated/configs/logs/log4j2.xml new file mode 100644 index 000000000..5b81cd712 --- /dev/null +++ b/sink-connector-lightweight/tests/integration/env/auto_replicated/configs/logs/log4j2.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/sink-connector-lightweight/tests/integration/env/auto_replicated/docker-compose.yml b/sink-connector-lightweight/tests/integration/env/auto_replicated/docker-compose.yml index 080c314ec..d5af369e7 100644 --- a/sink-connector-lightweight/tests/integration/env/auto_replicated/docker-compose.yml +++ b/sink-connector-lightweight/tests/integration/env/auto_replicated/docker-compose.yml @@ -3,7 +3,7 @@ version: "2.3" services: mysql-master: - image: docker.io/bitnami/mysql:8.0 + image: docker.io/bitnami/mysql:8.0.36 restart: "no" expose: - "3306" diff --git a/sink-connector-lightweight/tests/integration/helpers/cluster.py b/sink-connector-lightweight/tests/integration/helpers/cluster.py index abab8532d..717def898 100755 --- a/sink-connector-lightweight/tests/integration/helpers/cluster.py +++ b/sink-connector-lightweight/tests/integration/helpers/cluster.py @@ -596,15 +596,14 @@ def up(self, timeout=30 * 60): "IMAGE_DEPENDENCY_PROXY", "" ) self.environ["COMPOSE_HTTP_TIMEOUT"] = "300" - self.environ["CLICKHOUSE_TESTS_SERVER_BIN_PATH"] = ( - self.clickhouse_binary_path - ) - self.environ["CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH"] = ( - self.clickhouse_odbc_bridge_binary_path - or os.path.join( - os.path.dirname(self.clickhouse_binary_path), - "clickhouse-odbc-bridge", - ) + self.environ[ + "CLICKHOUSE_TESTS_SERVER_BIN_PATH" + ] = self.clickhouse_binary_path + self.environ[ + "CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH" + ] = self.clickhouse_odbc_bridge_binary_path or os.path.join( + os.path.dirname(self.clickhouse_binary_path), + "clickhouse-odbc-bridge", ) self.environ["CLICKHOUSE_TESTS_DIR"] = self.configs_dir @@ -890,6 +889,10 @@ def query( if raise_on_exception: raise QueryRuntimeException(r.output) assert False, error(r.output) + elif "ERROR" in r.output: + if raise_on_exception: + raise QueryRuntimeException(r.output) + assert False, error(r.output) return r @@ -970,7 +973,7 @@ def parse_value(input_string): else: raise ValueError(f"Failed to parse value from string: {input_string}") - def start_sink_connector(self, timeout=300, config_file=None): + def start_sink_connector(self, timeout=300, config_file=None, log_file=None): """Start ClickHouse Sink Connector.""" if config_file is None: config_file = "config.yml" @@ -978,8 +981,15 @@ def start_sink_connector(self, timeout=300, config_file=None): config_file = config_file.rsplit("/", 1)[-1] with Given("I start ClickHouse Sink Connector"): + command = f"java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Xms4g -Xmx4g " + + if log_file is None: + log_file = "log4j2.xml" + + command += f"-Dlog4j2.configurationFile={log_file} -jar /app.jar /configs/{config_file} com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication >> /logs/sink-connector-lt.log 2>&1 &" + start_command = self.command( - command=f"java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -jar /app.jar /configs/{config_file} com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication > /logs/sink-connector-lt.log 2>&1 &", + command=command, exitcode=0, timeout=timeout, ) diff --git a/sink-connector-lightweight/tests/integration/helpers/create_config.py b/sink-connector-lightweight/tests/integration/helpers/create_config.py index 0f7ef03f4..e49a5e54e 100644 --- a/sink-connector-lightweight/tests/integration/helpers/create_config.py +++ b/sink-connector-lightweight/tests/integration/helpers/create_config.py @@ -26,7 +26,8 @@ def update(self, new_data): def remove(self, key): """Remove the ClickHouse Sink Connector configuration key.""" - self.data.pop(key) + if key in self.data: + self.data.pop(key) def display_config(self): """Print out the ClickHouse Sink Connector configuration.""" diff --git a/sink-connector-lightweight/tests/integration/helpers/default_config.py b/sink-connector-lightweight/tests/integration/helpers/default_config.py index 6fb40ad26..fd873999f 100644 --- a/sink-connector-lightweight/tests/integration/helpers/default_config.py +++ b/sink-connector-lightweight/tests/integration/helpers/default_config.py @@ -7,12 +7,10 @@ "database.user": "root", "database.password": "root", "database.server.name": "ER54", - "database.include.list": "test", "clickhouse.server.url": "clickhouse", "clickhouse.server.user": "root", "clickhouse.server.password": "root", "clickhouse.server.port": "8123", - "clickhouse.server.database": "test", "database.allowPublicKeyRetrieval": "true", "snapshot.mode": "initial", "offset.flush.interval.ms": "5000", diff --git a/sink-connector-lightweight/tests/integration/regression_auto.py b/sink-connector-lightweight/tests/integration/regression_auto.py index 767d1c69f..39dec83a9 100755 --- a/sink-connector-lightweight/tests/integration/regression_auto.py +++ b/sink-connector-lightweight/tests/integration/regression_auto.py @@ -123,11 +123,15 @@ ), "/mysql to clickhouse replication/auto table creation/schema only/*": ( Skip, - "Seems to be broken in CI/CD. need oto fix.", + "Seems to be broken in CI/CD. need to fix.", ), "/mysql to clickhouse replication/auto table creation/cli/*": ( Skip, - "Seems to be broken in CI/CD. need oto fix.", + "Seems to be broken in CI/CD. need to fix.", + ), + "/mysql to clickhouse replication/auto table creation/parallel alters/multiple parallel add modify drop column": ( + Skip, + "Test requires fixing.", ), } @@ -166,7 +170,6 @@ def regression( "clickhouse-sink-connector-lt": ("clickhouse-sink-connector-lt",), "mysql-master": ("mysql-master",), "clickhouse": ("clickhouse", "clickhouse1", "clickhouse2", "clickhouse3"), - "bash-tools": ("bash-tools",), "zookeeper": ("zookeeper",), } diff --git a/sink-connector-lightweight/tests/integration/regression_auto_replicated.py b/sink-connector-lightweight/tests/integration/regression_auto_replicated.py index 4a3cd5661..7757ab5ff 100755 --- a/sink-connector-lightweight/tests/integration/regression_auto_replicated.py +++ b/sink-connector-lightweight/tests/integration/regression_auto_replicated.py @@ -127,11 +127,15 @@ ), "/mysql to clickhouse replication/auto replicated table creation/schema only/*": ( Skip, - "Seems to be broken in CI/CD. need oto fix.", + "Seems to be broken in CI/CD. need to fix.", ), "/mysql to clickhouse replication/auto replicated table creation/cli/*": ( Skip, - "Seems to be broken in CI/CD. need oto fix.", + "Seems to be broken in CI/CD. need to fix.", + ), + "/mysql to clickhouse replication/auto replicated table creation/parallel alters/multiple parallel add modify drop column": ( + Skip, + "Test requires fixing.", ), } @@ -164,7 +168,6 @@ def regression( "clickhouse-sink-connector-lt": ("clickhouse-sink-connector-lt",), "mysql-master": ("mysql-master",), "clickhouse": ("clickhouse", "clickhouse1", "clickhouse2", "clickhouse3"), - "bash-tools": ("bash-tools",), "zookeeper": ("zookeeper",), } @@ -304,6 +307,7 @@ def regression( join() + Feature(run=load("tests.databases", "module")) Feature( run=load("tests.schema_only", "module"), ) diff --git a/sink-connector-lightweight/tests/integration/regression_manual.py b/sink-connector-lightweight/tests/integration/regression_manual.py index a0f93747f..8739d0967 100755 --- a/sink-connector-lightweight/tests/integration/regression_manual.py +++ b/sink-connector-lightweight/tests/integration/regression_manual.py @@ -130,7 +130,6 @@ def regression( "debezium": ("debezium",), "mysql-master": ("mysql-master",), "clickhouse": ("clickhouse", "clickhouse1", "clickhouse2", "clickhouse3"), - "bash-tools": ("bash-tools",), "zookeeper": ("zookeeper",), } diff --git a/sink-connector-lightweight/tests/integration/requirements.txt b/sink-connector-lightweight/tests/integration/requirements.txt index 28ce2b366..2e9be8ba7 100644 --- a/sink-connector-lightweight/tests/integration/requirements.txt +++ b/sink-connector-lightweight/tests/integration/requirements.txt @@ -1,5 +1,10 @@ -docker-compose==1.29.2 testflows==2.1.5 +python-dateutil==2.9.0 +numpy==1.26.4 +pyarrow==16.1.0 +pandas==2.2.0 +PyYAML==5.3.1 +docker-compose==1.29.2 awscli==1.27.36 docker==6.1.3 -requests==2.31.0 +requests==2.31.0 \ No newline at end of file diff --git a/sink-connector-lightweight/tests/integration/requirements/requirements.md b/sink-connector-lightweight/tests/integration/requirements/requirements.md index 983162238..098f17608 100644 --- a/sink-connector-lightweight/tests/integration/requirements/requirements.md +++ b/sink-connector-lightweight/tests/integration/requirements/requirements.md @@ -190,6 +190,8 @@ * 28 [Column Names](#column-names) * 28.1 [Replicate Tables With Special Column Names](#replicate-tables-with-special-column-names) * 28.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ColumnNames.Special](#rqsrs-030clickhousemysqltoclickhousereplicationcolumnnamesspecial) + * 28.2 [Replicate Tables With Backticks in Column Names](#replicate-tables-with-backticks-in-column-names) + * 28.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ColumnNames.Backticks](#rqsrs-030clickhousemysqltoclickhousereplicationcolumnnamesbackticks) * 29 [Replication Interruption](#replication-interruption) * 29.1 [Retry Replication When ClickHouse Instance Is Not Active](#retry-replication-when-clickhouse-instance-is-not-active) * 29.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Interruption.ClickHouse.Instance.Stopped](#rqsrs-030clickhousemysqltoclickhousereplicationinterruptionclickhouseinstancestopped) @@ -273,17 +275,21 @@ * 34.4.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.IncludeList](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesincludelist) * 34.4.2 [Replicate All Databases](#replicate-all-databases) * 34.4.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.ReplicateAll](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesreplicateall) - * 34.5 [Table Operations](#table-operations) - * 34.5.1 [Specify Database Name in Table Operations](#specify-database-name-in-table-operations) - * 34.5.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.SpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsspecifydatabasename) - * 34.5.2 [Table Operations Without Specifying Database Name](#table-operations-without-specifying-database-name) - * 34.5.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.NoSpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsnospecifydatabasename) - * 34.6 [Error Handling](#error-handling) - * 34.6.1 [When Replicated Database Does Not Exist on the Destination](#when-replicated-database-does-not-exist-on-the-destination) - * 34.6.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ErrorHandling.DatabaseNotExist](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabaseserrorhandlingdatabasenotexist) - * 34.7 [Concurrent Actions](#concurrent-actions) - * 34.7.1 [Perform Table Operations on Each Database Concurrently](#perform-table-operations-on-each-database-concurrently) - * 34.7.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConcurrentActions](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconcurrentactions) + * 34.5 [Overriding Source To Destination Database Name Mapping](#overriding-source-to-destination-database-name-mapping) + * 34.5.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.OverrideMap](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesoverridemap) + * 34.5.2 [Multiple Database Names](#multiple-database-names) + * 34.5.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.OverrideMap.MultipleValues](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesoverridemapmultiplevalues) + * 34.6 [Table Operations](#table-operations) + * 34.6.1 [Specify Database Name in Table Operations](#specify-database-name-in-table-operations) + * 34.6.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.SpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsspecifydatabasename) + * 34.6.2 [Table Operations Without Specifying Database Name](#table-operations-without-specifying-database-name) + * 34.6.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.NoSpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsnospecifydatabasename) + * 34.7 [Error Handling](#error-handling) + * 34.7.1 [When Replicated Database Does Not Exist on the Destination](#when-replicated-database-does-not-exist-on-the-destination) + * 34.7.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ErrorHandling.DatabaseNotExist](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabaseserrorhandlingdatabasenotexist) + * 34.8 [Concurrent Actions](#concurrent-actions) + * 34.8.1 [Perform Table Operations on Each Database Concurrently](#perform-table-operations-on-each-database-concurrently) + * 34.8.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConcurrentActions](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconcurrentactions) ## Introduction @@ -1458,6 +1464,21 @@ CREATE TABLE new_table(col1 VARCHAR(255), col2 INT, is_deleted INT) The `ReplacingMergeTree` table created on ClickHouse side SHALL be updated and the `is_deleted` column should be renamed to `_is_deleted` so there are no column name conflicts between ClickHouse and source table. +### Replicate Tables With Backticks in Column Names + +#### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ColumnNames.Backticks +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from the source tables that have backticks in column names. + +For example, + +If we create a source table that contains the column with the `is_deleted` name, + +```sql +CREATE TABLE new_table(col1 VARCHAR(255), `col2` INT, `is_deleted` INT) +``` + ## Replication Interruption ### Retry Replication When ClickHouse Instance Is Not Active @@ -1796,10 +1817,12 @@ Multiple Databases: actions: - Perform table operations on each database sequentially - Perform table operations on all databases simultaneously + - Create database on source and map it to the database with different name on destination - Remove database configValues: - - database.include.list: database1, database2, ... , databaseN - Don't specify database.include.list + - database.include.list: database1, database2, ... , databaseN + - clickhouse.database.override.map: "test:test2, products:products2" TableOperations: - types: - With database name @@ -1937,6 +1960,53 @@ version: 1.0 [Altinity Sink Connector] SHALL support the ability to monitor all databases from the source and replicate them to the destination without specifying the `database.include.list` configuration value. +### Overriding Source To Destination Database Name Mapping + +#### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.OverrideMap +version: 1.0 + +[Altinity Sink Connector] SHALL support the usage of the `clickhouse.database.override.map` configuration value to allow the user to replicate the data from the source database to the destination database with a different name. + +For example, when using the following value in configuration, + +```yaml +clickhouse.database.override.map: "mysql1:ch1" +``` + +The source database `mysql1` SHALL be mapped to the destination database `ch1`, and the data from the source `mysql1` SHALL only be replicated to the destination database `ch1`. + +```mermaid +flowchart TD + B[Read clickhouse.database.override.map] --> D[Identify Source Database mysql1] + D --> E[Map to Destination Database ch1] + E --> F[Replicate Data to ch1] +``` + +#### Multiple Database Names + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.OverrideMap.MultipleValues +version: 1.0 + +[Altinity Sink Connector] SHALL support the usage of the `clickhouse.database.override.map` configuration value to map multiple source databases to different databases on the destination. + +For example, when using the following value in configuration, + +```yaml +clickhouse.database.override.map: "mysql1:ch1, mysql2:ch2" +``` + +The source databases `mysql1` and `mysql2` SHALL be mapped to the destination databases `ch1` and `ch2`, and the data from these source databases SHALL only be replicated to the destination databases `ch1` and `ch2`. + +```mermaid +flowchart TD + B[Read clickhouse.database.override.map] + B --> C[Parse Override Map] + C --> E[Map mysql1 to ch1] + C --> F[Map mysql2 to ch2] + E --> G[Replicate Data from mysql1 to ch1] + F --> H[Replicate Data from mysql2 to ch2] +``` + ### Table Operations #### Specify Database Name in Table Operations diff --git a/sink-connector-lightweight/tests/integration/requirements/requirements.py b/sink-connector-lightweight/tests/integration/requirements/requirements.py index 4c38c388a..010729d31 100644 --- a/sink-connector-lightweight/tests/integration/requirements/requirements.py +++ b/sink-connector-lightweight/tests/integration/requirements/requirements.py @@ -1,6 +1,6 @@ # These requirements were auto generated # from software requirements specification (SRS) -# document by TestFlows v2.0.240111.1210833. +# document by TestFlows v2.0.240708.1162538. # Do not edit by hand but re-generate instead # using 'tfs requirements generate' command. from testflows.core import Specification @@ -1739,6 +1739,30 @@ num="28.1.1", ) +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ColumnNames_Backticks = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ColumnNames.Backticks", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL support replication from the source tables that have backticks in column names.\n" + "\n" + "For example,\n" + "\n" + "If we create a source table that contains the column with the `is_deleted` name,\n" + "\n" + "```sql\n" + "CREATE TABLE new_table(col1 VARCHAR(255), `col2` INT, `is_deleted` INT)\n" + "```\n" + "\n" + ), + link=None, + level=3, + num="28.2.1", +) + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_Interruption_ClickHouse_Instance_Stopped = Requirement( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Interruption.ClickHouse.Instance.Stopped", version="1.0", @@ -2453,6 +2477,71 @@ num="34.4.2.1", ) +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_ConfigValues_OverrideMap = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.OverrideMap", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL support the usage of the `clickhouse.database.override.map` configuration value to allow the user to replicate the data from the source database to the destination database with a different name.\n" + "\n" + "For example, when using the following value in configuration,\n" + "\n" + "```yaml\n" + 'clickhouse.database.override.map: "mysql1:ch1"\n' + "```\n" + "\n" + "The source database `mysql1` SHALL be mapped to the destination database `ch1`, and the data from the source `mysql1` SHALL only be replicated to the destination database `ch1`.\n" + "\n" + "```mermaid\n" + "flowchart TD\n" + " B[Read clickhouse.database.override.map] --> D[Identify Source Database mysql1]\n" + " D --> E[Map to Destination Database ch1]\n" + " E --> F[Replicate Data to ch1]\n" + "```\n" + "\n" + ), + link=None, + level=3, + num="34.5.1", +) + +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_ConfigValues_OverrideMap_MultipleValues = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.OverrideMap.MultipleValues", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL support the usage of the `clickhouse.database.override.map` configuration value to map multiple source databases to different databases on the destination.\n" + "\n" + "For example, when using the following value in configuration,\n" + "\n" + "```yaml\n" + 'clickhouse.database.override.map: "mysql1:ch1, mysql2:ch2"\n' + "```\n" + "\n" + "The source databases `mysql1` and `mysql2` SHALL be mapped to the destination databases `ch1` and `ch2`, and the data from these source databases SHALL only be replicated to the destination databases `ch1` and `ch2`.\n" + "\n" + "```mermaid\n" + "flowchart TD\n" + " B[Read clickhouse.database.override.map]\n" + " B --> C[Parse Override Map]\n" + " C --> E[Map mysql1 to ch1]\n" + " C --> F[Map mysql2 to ch2]\n" + " E --> G[Replicate Data from mysql1 to ch1]\n" + " F --> H[Replicate Data from mysql2 to ch2]\n" + "```\n" + "\n" + ), + link=None, + level=4, + num="34.5.2.1", +) + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_TableOperations_SpecifyDatabaseName = Requirement( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.SpecifyDatabaseName", version="1.0", @@ -2472,7 +2561,7 @@ ), link=None, level=4, - num="34.5.1.1", + num="34.6.1.1", ) RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_TableOperations_NoSpecifyDatabaseName = Requirement( @@ -2494,7 +2583,7 @@ ), link=None, level=4, - num="34.5.2.1", + num="34.6.2.1", ) RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_ErrorHandling_DatabaseNotExist = Requirement( @@ -2510,7 +2599,7 @@ ), link=None, level=4, - num="34.6.1.1", + num="34.7.1.1", ) RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_ConcurrentActions = Requirement( @@ -2536,7 +2625,7 @@ ), link=None, level=4, - num="34.7.1.1", + num="34.8.1.1", ) SRS030_MySQL_to_ClickHouse_Replication = Specification( @@ -3127,6 +3216,14 @@ level=3, num="28.1.1", ), + Heading( + name="Replicate Tables With Backticks in Column Names", level=2, num="28.2" + ), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ColumnNames.Backticks", + level=3, + num="28.2.1", + ), Heading(name="Replication Interruption", level=1, num="29"), Heading( name="Retry Replication When ClickHouse Instance Is Not Active", @@ -3414,46 +3511,62 @@ level=4, num="34.4.2.1", ), - Heading(name="Table Operations", level=2, num="34.5"), Heading( - name="Specify Database Name in Table Operations", level=3, num="34.5.1" + name="Overriding Source To Destination Database Name Mapping", + level=2, + num="34.5", + ), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.OverrideMap", + level=3, + num="34.5.1", + ), + Heading(name="Multiple Database Names", level=3, num="34.5.2"), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.OverrideMap.MultipleValues", + level=4, + num="34.5.2.1", + ), + Heading(name="Table Operations", level=2, num="34.6"), + Heading( + name="Specify Database Name in Table Operations", level=3, num="34.6.1" ), Heading( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.SpecifyDatabaseName", level=4, - num="34.5.1.1", + num="34.6.1.1", ), Heading( name="Table Operations Without Specifying Database Name", level=3, - num="34.5.2", + num="34.6.2", ), Heading( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.NoSpecifyDatabaseName", level=4, - num="34.5.2.1", + num="34.6.2.1", ), - Heading(name="Error Handling", level=2, num="34.6"), + Heading(name="Error Handling", level=2, num="34.7"), Heading( name="When Replicated Database Does Not Exist on the Destination", level=3, - num="34.6.1", + num="34.7.1", ), Heading( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ErrorHandling.DatabaseNotExist", level=4, - num="34.6.1.1", + num="34.7.1.1", ), - Heading(name="Concurrent Actions", level=2, num="34.7"), + Heading(name="Concurrent Actions", level=2, num="34.8"), Heading( name="Perform Table Operations on Each Database Concurrently", level=3, - num="34.7.1", + num="34.8.1", ), Heading( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConcurrentActions", level=4, - num="34.7.1.1", + num="34.8.1.1", ), ), requirements=( @@ -3551,6 +3664,7 @@ RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_TableNames_Valid, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_TableNames_Invalid, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ColumnNames_Special, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ColumnNames_Backticks, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_Interruption_ClickHouse_Instance_Stopped, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_CLI, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_CLI_StartReplication, @@ -3588,12 +3702,14 @@ RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_Tables_DifferentNameSameStructure, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_ConfigValues_IncludeList, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_ConfigValues_ReplicateAll, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_ConfigValues_OverrideMap, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_ConfigValues_OverrideMap_MultipleValues, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_TableOperations_SpecifyDatabaseName, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_TableOperations_NoSpecifyDatabaseName, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_ErrorHandling_DatabaseNotExist, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_ConcurrentActions, ), - content=""" + content=r""" # SRS030 MySQL to ClickHouse Replication # Software Requirements Specification @@ -3786,6 +3902,8 @@ * 28 [Column Names](#column-names) * 28.1 [Replicate Tables With Special Column Names](#replicate-tables-with-special-column-names) * 28.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ColumnNames.Special](#rqsrs-030clickhousemysqltoclickhousereplicationcolumnnamesspecial) + * 28.2 [Replicate Tables With Backticks in Column Names](#replicate-tables-with-backticks-in-column-names) + * 28.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ColumnNames.Backticks](#rqsrs-030clickhousemysqltoclickhousereplicationcolumnnamesbackticks) * 29 [Replication Interruption](#replication-interruption) * 29.1 [Retry Replication When ClickHouse Instance Is Not Active](#retry-replication-when-clickhouse-instance-is-not-active) * 29.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Interruption.ClickHouse.Instance.Stopped](#rqsrs-030clickhousemysqltoclickhousereplicationinterruptionclickhouseinstancestopped) @@ -3869,17 +3987,21 @@ * 34.4.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.IncludeList](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesincludelist) * 34.4.2 [Replicate All Databases](#replicate-all-databases) * 34.4.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.ReplicateAll](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesreplicateall) - * 34.5 [Table Operations](#table-operations) - * 34.5.1 [Specify Database Name in Table Operations](#specify-database-name-in-table-operations) - * 34.5.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.SpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsspecifydatabasename) - * 34.5.2 [Table Operations Without Specifying Database Name](#table-operations-without-specifying-database-name) - * 34.5.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.NoSpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsnospecifydatabasename) - * 34.6 [Error Handling](#error-handling) - * 34.6.1 [When Replicated Database Does Not Exist on the Destination](#when-replicated-database-does-not-exist-on-the-destination) - * 34.6.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ErrorHandling.DatabaseNotExist](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabaseserrorhandlingdatabasenotexist) - * 34.7 [Concurrent Actions](#concurrent-actions) - * 34.7.1 [Perform Table Operations on Each Database Concurrently](#perform-table-operations-on-each-database-concurrently) - * 34.7.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConcurrentActions](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconcurrentactions) + * 34.5 [Overriding Source To Destination Database Name Mapping](#overriding-source-to-destination-database-name-mapping) + * 34.5.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.OverrideMap](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesoverridemap) + * 34.5.2 [Multiple Database Names](#multiple-database-names) + * 34.5.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.OverrideMap.MultipleValues](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesoverridemapmultiplevalues) + * 34.6 [Table Operations](#table-operations) + * 34.6.1 [Specify Database Name in Table Operations](#specify-database-name-in-table-operations) + * 34.6.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.SpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsspecifydatabasename) + * 34.6.2 [Table Operations Without Specifying Database Name](#table-operations-without-specifying-database-name) + * 34.6.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.NoSpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsnospecifydatabasename) + * 34.7 [Error Handling](#error-handling) + * 34.7.1 [When Replicated Database Does Not Exist on the Destination](#when-replicated-database-does-not-exist-on-the-destination) + * 34.7.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ErrorHandling.DatabaseNotExist](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabaseserrorhandlingdatabasenotexist) + * 34.8 [Concurrent Actions](#concurrent-actions) + * 34.8.1 [Perform Table Operations on Each Database Concurrently](#perform-table-operations-on-each-database-concurrently) + * 34.8.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConcurrentActions](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconcurrentactions) ## Introduction @@ -5054,6 +5176,21 @@ The `ReplacingMergeTree` table created on ClickHouse side SHALL be updated and the `is_deleted` column should be renamed to `_is_deleted` so there are no column name conflicts between ClickHouse and source table. +### Replicate Tables With Backticks in Column Names + +#### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ColumnNames.Backticks +version: 1.0 + +[Altinity Sink Connector] SHALL support replication from the source tables that have backticks in column names. + +For example, + +If we create a source table that contains the column with the `is_deleted` name, + +```sql +CREATE TABLE new_table(col1 VARCHAR(255), `col2` INT, `is_deleted` INT) +``` + ## Replication Interruption ### Retry Replication When ClickHouse Instance Is Not Active @@ -5392,10 +5529,12 @@ actions: - Perform table operations on each database sequentially - Perform table operations on all databases simultaneously + - Create database on source and map it to the database with different name on destination - Remove database configValues: - - database.include.list: database1, database2, ... , databaseN - Don't specify database.include.list + - database.include.list: database1, database2, ... , databaseN + - clickhouse.database.override.map: "test:test2, products:products2" TableOperations: - types: - With database name @@ -5533,6 +5672,53 @@ [Altinity Sink Connector] SHALL support the ability to monitor all databases from the source and replicate them to the destination without specifying the `database.include.list` configuration value. +### Overriding Source To Destination Database Name Mapping + +#### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.OverrideMap +version: 1.0 + +[Altinity Sink Connector] SHALL support the usage of the `clickhouse.database.override.map` configuration value to allow the user to replicate the data from the source database to the destination database with a different name. + +For example, when using the following value in configuration, + +```yaml +clickhouse.database.override.map: "mysql1:ch1" +``` + +The source database `mysql1` SHALL be mapped to the destination database `ch1`, and the data from the source `mysql1` SHALL only be replicated to the destination database `ch1`. + +```mermaid +flowchart TD + B[Read clickhouse.database.override.map] --> D[Identify Source Database mysql1] + D --> E[Map to Destination Database ch1] + E --> F[Replicate Data to ch1] +``` + +#### Multiple Database Names + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.OverrideMap.MultipleValues +version: 1.0 + +[Altinity Sink Connector] SHALL support the usage of the `clickhouse.database.override.map` configuration value to map multiple source databases to different databases on the destination. + +For example, when using the following value in configuration, + +```yaml +clickhouse.database.override.map: "mysql1:ch1, mysql2:ch2" +``` + +The source databases `mysql1` and `mysql2` SHALL be mapped to the destination databases `ch1` and `ch2`, and the data from these source databases SHALL only be replicated to the destination databases `ch1` and `ch2`. + +```mermaid +flowchart TD + B[Read clickhouse.database.override.map] + B --> C[Parse Override Map] + C --> E[Map mysql1 to ch1] + C --> F[Map mysql2 to ch2] + E --> G[Replicate Data from mysql1 to ch1] + F --> H[Replicate Data from mysql2 to ch2] +``` + ### Table Operations #### Specify Database Name in Table Operations diff --git a/sink-connector-lightweight/tests/integration/tests/calculated_columns.py b/sink-connector-lightweight/tests/integration/tests/calculated_columns.py index 4dd72c916..010baebb6 100644 --- a/sink-connector-lightweight/tests/integration/tests/calculated_columns.py +++ b/sink-connector-lightweight/tests/integration/tests/calculated_columns.py @@ -34,7 +34,7 @@ def string_concatenation(self): f"I create a {table_name} table with calculated column with string concatenation" ): create_mysql_table( - table_name=f"\`{table_name}\`", + table_name=rf"\`{table_name}\`", columns=f"first_name VARCHAR(50) NOT NULL,last_name VARCHAR(50) NOT NULL,fullname varchar(101) " f"GENERATED ALWAYS AS (CONCAT(first_name,' ',last_name)),email VARCHAR(100) NOT NULL", ) @@ -60,7 +60,7 @@ def basic_arithmetic_operations(self): with Given(f"I create a {table_name} table with calculated column"): create_mysql_table( - table_name=f"\`{table_name}\`", + table_name=rf"\`{table_name}\`", columns=f"a INT, b INT, sum_col INT AS (a + b), diff_col INT AS (a - b), prod_col INT AS (a * b), div_col DOUBLE AS (a / b)", ) @@ -86,7 +86,7 @@ def complex_expressions(self): with Given(f"I create a {table_name} table with calculated column"): create_mysql_table( - table_name=f"\`{table_name}\`", + table_name=rf"\`{table_name}\`", columns=f"base_salary DECIMAL(10,2), bonus_rate DECIMAL(5,2), total_compensation DECIMAL(12,2) AS (base_salary + (base_salary * bonus_rate / 100))", ) @@ -114,7 +114,7 @@ def nested(self): with Given(f"I create a {table_name} table with calculated column"): create_mysql_table( - table_name=f"\`{table_name}\`", + table_name=rf"\`{table_name}\`", columns=f"a INT, b INT, c INT AS (a + b), d INT AS (c * 2)", ) diff --git a/sink-connector-lightweight/tests/integration/tests/databases.py b/sink-connector-lightweight/tests/integration/tests/databases.py index efafd3b50..fa4f52e25 100644 --- a/sink-connector-lightweight/tests/integration/tests/databases.py +++ b/sink-connector-lightweight/tests/integration/tests/databases.py @@ -4,18 +4,16 @@ from integration.tests.steps.clickhouse import * -@TestOutline -@Requirements() +@TestScenario def databases_tables( self, - clickhouse_table_engine, version_column="_version", - clickhouse_columns=None, mysql_columns=" MyData DATETIME", ): """Check correctness of virtual column names.""" + database_1 = "test" + database_2 = "test2" - clickhouse = self.context.cluster.node("clickhouse") mysql = self.context.cluster.node("mysql-master") table_name = f"databases_{getuid()}" @@ -24,48 +22,50 @@ def databases_tables( create_mysql_table( table_name=table_name, columns=mysql_columns, + database_name=database_1, ) with And( f"I create another database in Clickhouse and table with the same name {table_name} in it" ): - clickhouse_node = self.context.cluster.node("clickhouse") + create_clickhouse_database(name=database_2) + create_mysql_database(database_name=database_2) - create_clickhouse_database(name="test2") + create_mysql_table( + table_name=table_name, + columns=mysql_columns, + database_name=database_2, + ) - clickhouse_node.query( - f"CREATE TABLE IF NOT EXISTS test2.{table_name} " - f"(id Int32, MyData Nullable(DateTime64(3)), _sign " - f"Int8, {version_column} UInt64) " - f"ENGINE = ReplacingMergeTree({version_column}) " - f" PRIMARY KEY (id ) ORDER BY (id)" - f" SETTINGS " - f"index_granularity = 8192;" + with When(f"I insert data in MySQL table {table_name} on {database_1}"): + mysql.query( + f"INSERT INTO {database_1}.{table_name} VALUES (1, '2018-09-08 17:51:05.777')" + ) + mysql.query( + f"INSERT INTO {database_1}.{table_name} VALUES (2, '2018-09-08 17:51:05.777')" + ) + mysql.query( + f"INSERT INTO {database_1}.{table_name} VALUES (3, '2018-09-08 17:51:05.777')" ) - with When(f"I insert data in MySQL table {table_name}"): - mysql.query(f"INSERT INTO {table_name} VALUES (1, '2018-09-08 17:51:05.777')") - mysql.query(f"INSERT INTO {table_name} VALUES (2, '2018-09-08 17:51:05.777')") - mysql.query(f"INSERT INTO {table_name} VALUES (3, '2018-09-08 17:51:05.777')") + with And(f"I insert data in MySQL table {table_name} on {database_2}"): + mysql.query( + f"INSERT INTO {database_2}.{table_name} VALUES (1, '2018-09-08 17:51:05.777')" + ) + mysql.query( + f"INSERT INTO {database_2}.{table_name} VALUES (2, '2018-09-08 17:51:05.777')" + ) + mysql.query( + f"INSERT INTO {database_2}.{table_name} VALUES (3, '2018-09-08 17:51:05.777')" + ) with Then(f"I check that data is replicated to the correct table"): - verify_table_creation_in_clickhouse( - table_name=table_name, - clickhouse_table_engine=clickhouse_table_engine, - statement="count(*)", - with_final=True, + Check(test=check_if_table_was_created)( + table_name=table_name, database_name=database_1 + ) + Check(test=check_if_table_was_created)( + table_name=table_name, database_name=database_2 ) - - -@TestFeature -def tables_in_different_databases(self): - """Check correctness replication when we have two tables with the same name in different databases.""" - # xfail("https://github.com/Altinity/clickhouse-sink-connector/issues/247") - - for clickhouse_table_engine in self.context.clickhouse_table_engines: - if self.context.env.endswith("auto"): - with Example({clickhouse_table_engine}, flags=TE): - databases_tables(clickhouse_table_engine=clickhouse_table_engine) @TestModule @@ -73,9 +73,4 @@ def tables_in_different_databases(self): def module(self): """Section to check behavior replication in different databases.""" - with Pool(1) as executor: - try: - for feature in loads(current_module(), Feature): - Feature(test=feature, parallel=True, executor=executor)() - finally: - join() + Scenario(run=databases_tables) diff --git a/sink-connector-lightweight/tests/integration/tests/datatypes.py b/sink-connector-lightweight/tests/integration/tests/datatypes.py index 0e15866d0..b59e72e74 100644 --- a/sink-connector-lightweight/tests/integration/tests/datatypes.py +++ b/sink-connector-lightweight/tests/integration/tests/datatypes.py @@ -36,7 +36,7 @@ def create_table_with_datetime_column(self, table_name, data, precision): with By(f"creating a {table_name} table with datetime column"): create_mysql_table( - table_name=f"\`{table_name}\`", + table_name=rf"\`{table_name}\`", columns=f"date DATETIME({precision})", ) diff --git a/sink-connector-lightweight/tests/integration/tests/is_deleted.py b/sink-connector-lightweight/tests/integration/tests/is_deleted.py index 61a951806..b35e722aa 100644 --- a/sink-connector-lightweight/tests/integration/tests/is_deleted.py +++ b/sink-connector-lightweight/tests/integration/tests/is_deleted.py @@ -9,18 +9,23 @@ @TestStep(Given) def create_table_with_is_deleted( - self, table_name, datatype="int", data="5", column="is_deleted" + self, table_name, datatype="int", data="5", column="is_deleted", backticks=False ): """Create mysql table that contains the column with the name 'is_deleted'""" mysql_node = self.context.mysql_node clickhouse_node = self.context.clickhouse_node + if not backticks: + columns = "col1 varchar(255), col2 int, " + else: + columns = "\`col1\` varchar(255), \`col2\` int, " + with By( f"creating a {table_name} table with is_deleted column and {datatype} datatype" ): create_mysql_table( - table_name=f"\`{table_name}\`", - columns=f"col1 varchar(255), col2 int, {column} {datatype}", + table_name=rf"\`{table_name}\`", + columns=f"{columns}{column} {datatype}", ) with And(f"inserting data into the {table_name} table"): @@ -158,6 +163,52 @@ def is_deleted_different_datatypes(self): )(datatype=datatype) +@TestScenario +@Requirements( + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ColumnNames_Special("1.0") +) +def column_with_backticks(self): + """Check that the table is replicated when the source table has columns created with backticks.""" + clickhouse_node = self.context.clickhouse_node + table_name = "tb_" + getuid() + + with Given(f"I create the {table_name} table and populate it with data"): + create_table_with_is_deleted(table_name=table_name, backticks=True) + + with Then("I check that the data was inserted correctly into the ClickHouse table"): + for retry in retries(timeout=40, delay=1): + with retry: + clickhouse_data = clickhouse_node.query( + f"DESCRIBE TABLE test.{table_name} FORMAT CSV" + ) + assert ( + "__is_deleted" and "is_deleted" in clickhouse_data.output.strip() + ), error() + + +@Requirements( + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ColumnNames_Special("1.0") +) +@TestScenario +def column_with_is_deleted_backticks(self): + """Check that the table is replicated when the source table has columns created with is_deleted column having backticks.""" + clickhouse_node = self.context.clickhouse_node + table_name = "tb_" + getuid() + + with Given(f"I create the {table_name} table and populate it with data"): + create_table_with_is_deleted(table_name=table_name, column="\`is_deleted\`") + + with Then("I check that the data was inserted correctly into the ClickHouse table"): + for retry in retries(timeout=40, delay=1): + with retry: + clickhouse_data = clickhouse_node.query( + f"DESCRIBE TABLE test.{table_name} FORMAT CSV" + ) + assert ( + "__is_deleted" and "is_deleted" in clickhouse_data.output.strip() + ), error() + + @TestModule @Name("is deleted") @Requirements( diff --git a/sink-connector-lightweight/tests/integration/tests/multiple_databases.py b/sink-connector-lightweight/tests/integration/tests/multiple_databases.py index fe7402d5d..9fe2dc1ae 100644 --- a/sink-connector-lightweight/tests/integration/tests/multiple_databases.py +++ b/sink-connector-lightweight/tests/integration/tests/multiple_databases.py @@ -15,6 +15,7 @@ modify_column, drop_column, add_primary_key, + drop_primary_key, ) @@ -26,20 +27,35 @@ def get_n_random_items(lst, n): return random.sample(lst, n) -@TestStep(Given) -def replicate_all_databases(self): - """Set ClickHouse Sink Connector configuration to replicate all databases.""" +@TestOutline +def remove_configuration_and_restart_sink( + self, configuration, configuration_name="multiple_databases.yml" +): + """Remove the ClickHouse Sink Connector configuration and restart the sink connector.""" with By( - "creating a ClickHouse Sink Connector configuration without database.include.list values specified" + f"creating a ClickHouse Sink Connector configuration without {configuration} values specified" ): remove_sink_configuration( - key="database.include.list", - config_file=os.path.join( - self.context.config_file, "multiple_databases.yml" - ), + key=configuration, + config_file=os.path.join(self.context.config_file, configuration_name), ) +@TestStep(Given) +def replicate_all_databases(self): + """Set ClickHouse Sink Connector configuration to replicate all databases.""" + + remove_configuration_and_restart_sink(configuration="database.include.list") + + +@TestStep(Given) +def remove_database_map(self): + """Remove the database map from the ClickHouse Sink Connector configuration.""" + remove_configuration_and_restart_sink( + configuration="clickhouse.database.override.map" + ) + + @TestStep(Given) def replicate_all_databases_rrmt(self): """Set ClickHouse Sink Connector configuration to replicate all databases when tables have ReplicatedReplacingMergeTree engine.""" @@ -92,6 +108,101 @@ def set_list_of_databases_to_replicate(self, databases=None, config_name=None): replicate_all_databases() +@TestStep(Given) +def create_map_for_database_names(self, databases_map: dict = None, config_name=None): + """Create a map for the database names. + Example: + clickhouse.database.override.map: "employees:employees2, products:productsnew + """ + config_file = self.context.config_file + + new_database_map = [] + + for key, value in databases_map.items(): + key = key.strip(r"\`") + value = value.strip(r"\`") + new_database_map.append(rf"{key}:{value}") + + configuration_value = ", ".join(new_database_map) + + if config_name is None: + config_name = os.path.join(config_file, "override_multiple_database_names.yml") + else: + config_name = os.path.join(config_file, config_name) + try: + with By( + "setting the list of databases to replicate in the ClickHouse Sink Connector configuration", + description=f"MySQL to ClickHouse database map: {configuration_value}", + ): + change_sink_configuration( + values={"clickhouse.database.override.map": f"{configuration_value}"}, + config_file=config_name, + ) + yield + finally: + with Finally( + "removing the list of databases from the ClickHouse Sink Connector configuration" + ): + remove_database_map() + + +@TestOutline +def create_table_mapped( + self, + source_database=None, + destination_database=None, + exists=None, + validate_values=True, +): + """Create a sample table in MySQL and validate that it was replicated in ClickHouse.""" + + table_name = f"table_{getuid()}" + + if exists is None: + exists = "1" + + if source_database is None: + source_database = "test" + + if destination_database is None: + destination_database = "test" + + table_values = create_sample_values() + + with By("creating a sample table in MySQL"): + create_mysql_table( + table_name=rf"\`{table_name}\`", + columns=f"col1 varchar(255), col2 int", + database_name=source_database, + ) + + with And("inserting data into the table"): + insert( + table_name=table_name, database_name=source_database, values=table_values + ) + + if not validate_values: + table_values = "" + + with And("validating that the table was replicated in ClickHouse"): + for retry in retries(timeout=40): + with retry: + check_if_table_was_created( + table_name=table_name, + database_name=destination_database, + message=exists, + ) + if validate_values: + for retry in retries(timeout=10, delay=1): + with retry: + validate_data_in_clickhouse_table( + table_name=table_name, + expected_output=table_values.replace("'", ""), + database_name=destination_database, + statement="id, col1, col2", + ) + + def create_sample_values(): """Create sample values to insert into the table.""" return f'{generate_sample_mysql_value("INT")},{generate_sample_mysql_value("VARCHAR")},{generate_sample_mysql_value("INT")}' @@ -113,7 +224,7 @@ def create_table_and_insert_values( with By("creating a sample table in MySQL"): create_mysql_table( - table_name=f"\`{table_name}\`", + table_name=rf"\`{table_name}\`", columns=f"col1 varchar(255), col2 int", database_name=database_name, ) @@ -155,7 +266,23 @@ def create_source_and_destination_databases(self, database_name=None): @TestStep(Given) -def create_databases(self, databases=None): +def create_diff_name_dbs(self, databases): + """Create MySQL and ClickHouse databases with different names.""" + + with Pool(2) as pool: + for source, destination in databases.items(): + Step(test=create_mysql_database, parallel=True, executor=pool)( + database_name=source + ) + Step(test=create_clickhouse_database, parallel=True, executor=pool)( + name=destination + ) + + join() + + +@TestStep(Given) +def create_databases(self, databases: list = None): """Create MySQL and ClickHouse databases from a list of database names.""" if databases is None: databases = [] @@ -478,12 +605,86 @@ def add_primary_key_on_a_database(self, database): create_table_and_insert_values(table_name=table_name, database_name=database) with When("I add a primary key on the table"): + drop_primary_key(table_name=table_name, database=database) add_primary_key(table_name=table_name, database=database, column_name=column) with Then("I check that the primary key was added to the table"): check_column(table_name=table_name, database=database, column_name=column) +@TestOutline +def check_different_database_names(self, database_map): + """Check that the tables are replicated when we have source and destination databases with different names.""" + + with Given("I create the source and destination databases from a list"): + create_diff_name_dbs(databases=database_map) + + with When("I set the new map between source and destination database names"): + create_map_for_database_names(databases_map=database_map) + + with Then("I create table on each database and validate data"): + for source_database, destination_database in database_map.items(): + create_table_mapped( + source_database=source_database, + destination_database=destination_database, + ) + + +@TestScenario +def different_database_names(self): + """Check that the tables are replicated when we have source and destination databases with different names.""" + database_map = {"mysql1": "ch1", "mysql2": "ch2", "mysql3": "ch3", "mysql4": "ch4"} + check_different_database_names(database_map=database_map) + + +@TestScenario +def different_database_names_with_source_backticks(self): + """Check that the tables are replicated when we have source and destination databases with different names and source database name contains backticks.""" + database_map = { + "\`mysql1\`": "ch1", + "\`mysql2\`": "ch2", + "\`mysql3\`": "ch3", + "\`mysql4\`": "ch4", + } + check_different_database_names(database_map=database_map) + + +@TestScenario +def different_database_names_with_destination_backticks(self): + """Check that the tables are replicated when we have source and destination databases with different names and destination database name contains backticks.""" + database_map = { + "mysql1": "\`ch1\`", + "mysql2": "\`ch2\`", + "mysql3": "\`ch3\`", + "mysql4": "\`ch4\`", + } + check_different_database_names(database_map=database_map) + + +@TestScenario +def different_database_names_with_backticks(self): + """Check that the tables are replicated when we have source and destination databases with the same names and they contain backticks.""" + database_map = { + "\`mysql1\`": "\`ch1\`", + "\`mysql2\`": "\`ch2\`", + "\`mysql3\`": "\`ch3\`", + "\`mysql4\`": "\`ch4\`", + } + check_different_database_names(database_map=database_map) + + +@TestScenario +def same_database_names(self): + """Check that the tables are replicated when we have source and destination databases with the same names.""" + database_map = { + "mysql1": "mysql1", + "mysql2": "mysql2", + "mysql3": "mysql3", + "mysql4": "mysql4", + } + check_different_database_names(database_map=database_map) + + @TestCheck def check_alters(self, alter_1, alter_2, database_1, database_2): """Run multiple alter statements on different databases.""" @@ -627,6 +828,27 @@ def concurrent_actions(self, number_of_iterations=None): check_concurrent_actions(actions=actions) +@TestSuite +@Requirements( + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_ConfigValues_OverrideMap( + "1.0" + ), + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_MultipleDatabases_ConfigValues_OverrideMap_MultipleValues( + "1.0" + ), +) +def source_destination_overrides(self): + """Check that the tables are replicated when we have source and destination databases with different names. + For example, + mysql1:ch1; mysql2:ch2 + """ + Scenario(run=different_database_names) + Scenario(run=different_database_names_with_source_backticks) + Scenario(run=different_database_names_with_destination_backticks) + Scenario(run=different_database_names_with_backticks) + Scenario(run=same_database_names) + + @TestFeature @Name("multiple databases") @Requirements( @@ -683,8 +905,7 @@ def module( elif engine == "ReplicatedReplacingMergeTree": replicate_all_databases_rrmt() - with Pool(parallel_cases) as executor: - Feature(run=inserts, parallel=True, executor=executor) - Feature(run=alters, parallel=True, executor=executor) - Feature(run=concurrent_actions, parallel=True, executor=executor) - join() + Feature(run=inserts) + Feature(run=alters) + Feature(run=concurrent_actions) + Feature(run=source_destination_overrides) diff --git a/sink-connector-lightweight/tests/integration/tests/retry_on_fail.py b/sink-connector-lightweight/tests/integration/tests/retry_on_fail.py index cec1a6e5c..2746ae219 100644 --- a/sink-connector-lightweight/tests/integration/tests/retry_on_fail.py +++ b/sink-connector-lightweight/tests/integration/tests/retry_on_fail.py @@ -16,7 +16,7 @@ def retry_on_fail(self): with When("I creat a table in MySQL"): create_mysql_table( - table_name=f"\`{table_name}\`", + table_name=rf"\`{table_name}\`", columns=f"retry VARCHAR(16)", ) diff --git a/sink-connector-lightweight/tests/integration/tests/schema_only.py b/sink-connector-lightweight/tests/integration/tests/schema_only.py index 39e1f2052..89bb36e24 100644 --- a/sink-connector-lightweight/tests/integration/tests/schema_only.py +++ b/sink-connector-lightweight/tests/integration/tests/schema_only.py @@ -11,7 +11,7 @@ def create_table_structure(self, table_name): with By(f"creating a {table_name} table"): create_mysql_table( - table_name=f"\`{table_name}\`", + table_name=rf"\`{table_name}\`", columns=f"col1 varchar(255), col2 int", ) diff --git a/sink-connector-lightweight/tests/integration/tests/sink_cli_commands.py b/sink-connector-lightweight/tests/integration/tests/sink_cli_commands.py index 2ceb8f98b..ab7a071c3 100644 --- a/sink-connector-lightweight/tests/integration/tests/sink_cli_commands.py +++ b/sink-connector-lightweight/tests/integration/tests/sink_cli_commands.py @@ -47,7 +47,7 @@ def create_and_validate_table(self, table_name): "creating a table in MySQL and checking that it was also created in ClickHouse" ): create_mysql_table( - table_name=f"\`{table_name}\`", + table_name=rf"\`{table_name}\`", columns=f"col1 varchar(255), col2 int", ) diff --git a/sink-connector-lightweight/tests/integration/tests/steps/alter.py b/sink-connector-lightweight/tests/integration/tests/steps/alter.py index 33ba1ca56..3f8268794 100644 --- a/sink-connector-lightweight/tests/integration/tests/steps/alter.py +++ b/sink-connector-lightweight/tests/integration/tests/steps/alter.py @@ -20,7 +20,7 @@ def add_column( node = self.context.cluster.node("mysql-master") node.query( - f"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type};" + rf"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type};" ) @@ -41,7 +41,7 @@ def rename_column( node = self.context.cluster.node("mysql-master") node.query( - f"ALTER TABLE {database}.\`{table_name}\` RENAME COLUMN {column_name} to {new_column_name};" + rf"ALTER TABLE {database}.\`{table_name}\` RENAME COLUMN {column_name} to {new_column_name};" ) @@ -63,7 +63,7 @@ def change_column( node = self.context.cluster.node("mysql-master") node.query( - f"ALTER TABLE {database}.\`{table_name}\` CHANGE COLUMN {column_name} {new_column_name} {new_column_type};" + rf"ALTER TABLE {database}.\`{table_name}\` CHANGE COLUMN {column_name} {new_column_name} {new_column_type};" ) @@ -84,7 +84,7 @@ def modify_column( node = self.context.cluster.node("mysql-master") node.query( - f"ALTER TABLE {database}.\`{table_name}\` MODIFY COLUMN {column_name} {new_column_type};" + rf"ALTER TABLE {database}.\`{table_name}\` MODIFY COLUMN {column_name} {new_column_type};" ) @@ -97,7 +97,7 @@ def drop_column(self, table_name, column_name="new_col", node=None, database=Non if node is None: node = self.context.cluster.node("mysql-master") - node.query(f"ALTER TABLE {database}.\`{table_name}\` DROP COLUMN {column_name};") + node.query(rf"ALTER TABLE {database}.\`{table_name}\` DROP COLUMN {column_name};") @TestStep(When) @@ -149,7 +149,7 @@ def add_column_null_not_null( null_not_null = "NOT NULL" if not is_null else "NULL" node.query( - f"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type} {null_not_null};" + rf"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type} {null_not_null};" ) @@ -171,7 +171,7 @@ def add_column_default( node = self.context.cluster.node("mysql-master") node.query( - f"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type} DEFAULT {default_value};" + rf"ALTER TABLE {database}.\`{table_name}\` ADD COLUMN {column_name} {column_type} DEFAULT {default_value};" ) @@ -185,5 +185,17 @@ def add_primary_key(self, table_name, column_name, node=None, database=None): node = self.context.cluster.node("mysql-master") node.query( - f"ALTER TABLE {database}.\`{table_name}\` ADD PRIMARY KEY ({column_name});" + rf"ALTER TABLE {database}.\`{table_name}\` ADD PRIMARY KEY ({column_name});" ) + + +@TestStep(When) +def drop_primary_key(self, table_name, node=None, database=None): + """DROP PRIMARY KEY""" + if database is None: + database = "test" + + if node is None: + node = self.context.cluster.node("mysql-master") + + node.query(rf"ALTER TABLE {database}.\`{table_name}\` DROP PRIMARY KEY;") diff --git a/sink-connector-lightweight/tests/integration/tests/steps/clickhouse.py b/sink-connector-lightweight/tests/integration/tests/steps/clickhouse.py index 410087b14..96558e9d6 100644 --- a/sink-connector-lightweight/tests/integration/tests/steps/clickhouse.py +++ b/sink-connector-lightweight/tests/integration/tests/steps/clickhouse.py @@ -11,7 +11,7 @@ def drop_database(self, database_name=None, node=None): node = self.context.cluster.node("clickhouse") with By("executing drop database query"): node.query( - f"DROP DATABASE IF EXISTS {database_name} ON CLUSTER replicated_cluster;" + rf"DROP DATABASE IF EXISTS {database_name} ON CLUSTER replicated_cluster;" ) @@ -66,7 +66,7 @@ def create_clickhouse_database(self, name=None, node=None): drop_database(database_name=name) node.query( - f"CREATE DATABASE IF NOT EXISTS {name} ON CLUSTER replicated_cluster" + rf"CREATE DATABASE IF NOT EXISTS {name} ON CLUSTER replicated_cluster" ) yield finally: @@ -158,7 +158,13 @@ def check_if_table_was_created( @TestStep(Then) def validate_data_in_clickhouse_table( - self, table_name, expected_output, statement="*", node=None, database_name=None + self, + table_name, + expected_output, + statement="*", + node=None, + database_name=None, + timeout=40, ): """Validate data in ClickHouse table.""" @@ -170,22 +176,34 @@ def validate_data_in_clickhouse_table( if self.context.clickhouse_table_engine == "ReplicatedReplacingMergeTree": for node in self.context.cluster.nodes["clickhouse"]: - for retry in retries(timeout=40): + for retry in retries(timeout=timeout, delay=1): with retry: - data = self.context.cluster.node(node).query( - f"SELECT {statement} FROM {database_name}.{table_name} ORDER BY tuple(*) FORMAT CSV" + data = ( + self.context.cluster.node(node) + .query( + f"SELECT {statement} FROM {database_name}.{table_name} ORDER BY tuple(*) FORMAT CSV" + ) + .output.strip() + .replace('"', "") ) assert ( - data.output.strip().replace('"', "") == expected_output - ), error() + data == expected_output + ), f"Expected: {expected_output}, Actual: {data}" elif self.context.clickhouse_table_engine == "ReplacingMergeTree": - for retry in retries(timeout=40): + for retry in retries(timeout=timeout, delay=1): with retry: - data = node.query( - f"SELECT {statement} FROM {database_name}.{table_name} ORDER BY tuple(*) FORMAT CSV" + data = ( + node.query( + f"SELECT {statement} FROM {database_name}.{table_name} ORDER BY tuple(*) FORMAT CSV" + ) + .output.strip() + .replace('"', "") ) - assert data.output.strip().replace('"', "") == expected_output, error() + + assert ( + data == expected_output + ), f"Expected: {expected_output}, Actual: {data}" else: raise Exception("Unknown ClickHouse table engine") diff --git a/sink-connector-lightweight/tests/integration/tests/steps/datatypes.py b/sink-connector-lightweight/tests/integration/tests/steps/datatypes.py index b9e63c931..6aa9f85b4 100644 --- a/sink-connector-lightweight/tests/integration/tests/steps/datatypes.py +++ b/sink-connector-lightweight/tests/integration/tests/steps/datatypes.py @@ -80,10 +80,10 @@ "MIntmax": "MEDIUMINT NOT NULL", "UMIntmin": "MEDIUMINT UNSIGNED NOT NULL", "UMIntmax": "MEDIUMINT UNSIGNED NOT NULL", - "x_char": "CHAR NOT NULL", - "x_text": "TEXT NOT NULL", - "x_varchar": "VARCHAR(4) NOT NULL", - "x_Blob": "BLOB NOT NULL", + "x_char": "CHAR(255) NOT NULL", + "x_text": "TEXT(255) NOT NULL", + "x_varchar": "VARCHAR(255) NOT NULL", + "x_Blob": "BLOB(255) NOT NULL", "x_Mediumblob": "MEDIUMBLOB NOT NULL", "x_Longblob": "LONGBLOB NOT NULL", "x_binary": "BINARY NOT NULL", diff --git a/sink-connector-lightweight/tests/integration/tests/steps/mysql.py b/sink-connector-lightweight/tests/integration/tests/steps/mysql.py index dab4cc4e9..e451522e1 100644 --- a/sink-connector-lightweight/tests/integration/tests/steps/mysql.py +++ b/sink-connector-lightweight/tests/integration/tests/steps/mysql.py @@ -18,22 +18,16 @@ def generate_sample_mysql_value(data_type): return str(number) elif data_type.startswith("DOUBLE"): # Adjusting the range to avoid overflow, staying within a reasonable limit - return str(random.uniform(-1.7e308, 1.7e308)) + return f"'{str(random.uniform(-1.7e307, 1.7e307))}'" elif data_type == "DATE NOT NULL": - return (datetime.today() - timedelta(days=random.randint(1, 365))).strftime( - "%Y-%m-%d" - ) + return f'\'{(datetime.today() - timedelta(days=random.randint(1, 365))).strftime("%Y-%m-%d")}\'' elif data_type.startswith("DATETIME"): - return (datetime.now() - timedelta(days=random.randint(1, 365))).strftime( - "%Y-%m-%d %H:%M:%S.%f" - )[:19] + return f'\'{(datetime.now() - timedelta(days=random.randint(1, 365))).strftime("%Y-%m-%d %H:%M:%S.%f")[:19]}\'' elif data_type.startswith("TIME"): if "6" in data_type: - return (datetime.now()).strftime("%H:%M:%S.%f")[ - : 8 + 3 - ] # Including microseconds + return f'\'{(datetime.now()).strftime("%H:%M:%S.%f")[: 8 + 3]}\'' else: - return (datetime.now()).strftime("%H:%M:%S") + return f'\'{(datetime.now()).strftime("%H:%M:%S")}\'' elif "INT" in data_type: if "TINYINT" in data_type: return str( @@ -60,21 +54,19 @@ def generate_sample_mysql_value(data_type): else random.randint(-(2**63), 2**63 - 1) ) else: # INT - return str( - random.randint(0, 4294967295) - if "UNSIGNED" in data_type - else random.randint(-2147483648, 2147483647) - ) + return f'\'{str(random.randint(0, 4294967295) if "UNSIGNED" in data_type else random.randint(-2147483648, 2147483647))}\'' elif ( data_type.startswith("CHAR") or data_type.startswith("VARCHAR") - or data_type == "TEXT NOT NULL" + or data_type.startswith("TEXT") ): return "'SampleText'" + elif data_type.startswith("BLOB"): + return "'SampleBinaryData'" elif data_type.endswith("BLOB NOT NULL"): return "'SampleBinaryData'" elif data_type.startswith("BINARY") or data_type.startswith("VARBINARY"): - return "'BinaryData'" + return "'a'" else: return "UnknownType" @@ -90,12 +82,13 @@ def create_mysql_database(self, node=None, database_name=None): try: with Given(f"I create MySQL database {database_name}"): - node.query(f"DROP DATABASE IF EXISTS {database_name};") - node.query(f"CREATE DATABASE IF NOT EXISTS {database_name};") + node.query(rf"DROP DATABASE IF EXISTS {database_name};") + node.query(rf"CREATE DATABASE IF NOT EXISTS {database_name};") yield finally: with Finally(f"I delete MySQL database {database_name}"): - node.query(f"DROP DATABASE IF EXISTS {database_name};") + node.query(rf"DROP DATABASE IF EXISTS {database_name};") + @TestStep(Given) def create_mysql_table( diff --git a/sink-connector-lightweight/tests/integration/tests/table_names.py b/sink-connector-lightweight/tests/integration/tests/table_names.py index f74323602..09d1bacfd 100644 --- a/sink-connector-lightweight/tests/integration/tests/table_names.py +++ b/sink-connector-lightweight/tests/integration/tests/table_names.py @@ -52,23 +52,23 @@ def check_table_names(self, table_name): with Given(f"I create the {table_name} table"): create_mysql_table( - table_name=f"\`{table_name}\`", + table_name=rf"\`{table_name}\`", columns="x INT", ) with And("I insert data into the table"): - mysql_node.query(f"INSERT INTO \`{table_name}\` VALUES (1, 1);") + mysql_node.query(rf"INSERT INTO \`{table_name}\` VALUES (1, 1);") with Then(f"I check that the {table_name} was created in the ClickHouse side"): for retry in retries(timeout=40, delay=1): with retry: - clickhouse_node.query(f"EXISTS test.\`{table_name}\`", message="1") + clickhouse_node.query(rf"EXISTS test.\`{table_name}\`", message="1") with And("I check that the data was inserted correctly into the ClickHouse table"): for retry in retries(timeout=40, delay=1): with retry: clickhouse_data = clickhouse_node.query( - f"SELECT id,x FROM test.\`{table_name}\` FORMAT CSV" + rf"SELECT id,x FROM test.\`{table_name}\` FORMAT CSV" ) assert clickhouse_data.output.strip() == "1,1", error() diff --git a/sink-connector/deploy/helm/altinity-sink-connector/templates/sink-kafkaconnector.yaml b/sink-connector/deploy/helm/altinity-sink-connector/templates/sink-kafkaconnector.yaml index 58044fc64..6a39c8228 100644 --- a/sink-connector/deploy/helm/altinity-sink-connector/templates/sink-kafkaconnector.yaml +++ b/sink-connector/deploy/helm/altinity-sink-connector/templates/sink-kafkaconnector.yaml @@ -15,7 +15,6 @@ spec: clickhouse.server.url: {{ $.Values.clickhouseSinkConnector.clickhouse.server.url }} clickhouse.server.user: {{ $.Values.clickhouseSinkConnector.clickhouse.server.user }} clickhouse.server.password: {{ $.Values.clickhouseSinkConnector.clickhouse.server.password }} - clickhouse.server.database: {{ $.Values.clickhouseSinkConnector.clickhouse.server.url }} clickhouse.server.port: {{ $.Values.clickhouseSinkConnector.clickhouse.server.port }} clickhouse.topic2table.map: "employees:employee" store.kafka.metadata: "false" diff --git a/sink-connector/deploy/k8s/sink-connector-apicurio-avro.yaml b/sink-connector/deploy/k8s/sink-connector-apicurio-avro.yaml index e70f3f1ca..b3a32da93 100644 --- a/sink-connector/deploy/k8s/sink-connector-apicurio-avro.yaml +++ b/sink-connector/deploy/k8s/sink-connector-apicurio-avro.yaml @@ -18,7 +18,6 @@ spec: clickhouse.server.url: "${CLICKHOUSE_HOST}" clickhouse.server.user: "${CLICKHOUSE_USER}" clickhouse.server.password: "${CLICKHOUSE_PASSWORD}" - clickhouse.server.database: "${CLICKHOUSE_DATABASE}" clickhouse.server.port: "${CLICKHOUSE_PORT}" clickhouse.table.name: "${CLICKHOUSE_TABLE}" diff --git a/sink-connector/deploy/k8s/sink-connector-avro.yaml b/sink-connector/deploy/k8s/sink-connector-avro.yaml index 822eebae6..0f62d9bc9 100644 --- a/sink-connector/deploy/k8s/sink-connector-avro.yaml +++ b/sink-connector/deploy/k8s/sink-connector-avro.yaml @@ -18,7 +18,6 @@ spec: clickhouse.server.url: "${CLICKHOUSE_HOST}" clickhouse.server.user: "${CLICKHOUSE_USER}" clickhouse.server.password: "${CLICKHOUSE_PASSWORD}" - clickhouse.server.database: "${CLICKHOUSE_DATABASE}" clickhouse.server.port: "${CLICKHOUSE_PORT}" clickhouse.table.name: "${CLICKHOUSE_TABLE}" diff --git a/sink-connector/deploy/k8s/sink-connector-json.yaml b/sink-connector/deploy/k8s/sink-connector-json.yaml index 30cdcecfc..8e8f17e96 100644 --- a/sink-connector/deploy/k8s/sink-connector-json.yaml +++ b/sink-connector/deploy/k8s/sink-connector-json.yaml @@ -19,7 +19,6 @@ spec: clickhouse.server.url: "${CLICKHOUSE_HOST}" clickhouse.server.user: "${CLICKHOUSE_USER}" clickhouse.server.password: "${CLICKHOUSE_PASSWORD}" - clickhouse.server.database: "${CLICKHOUSE_DATABASE}" clickhouse.server.port: "${CLICKHOUSE_PORT}" clickhouse.table.name: "${CLICKHOUSE_TABLE}" key.converter: "io.apicurio.registry.utils.converter.AvroConverter" diff --git a/sink-connector/deploy/sink-connector-setup-database.sh b/sink-connector/deploy/sink-connector-setup-database.sh index 53c881e3f..43c46337b 100755 --- a/sink-connector/deploy/sink-connector-setup-database.sh +++ b/sink-connector/deploy/sink-connector-setup-database.sh @@ -32,7 +32,6 @@ if [[ $1 == "apicurio" ]]; then "clickhouse.server.url": "${CLICKHOUSE_HOST}", "clickhouse.server.user": "${CLICKHOUSE_USER}", "clickhouse.server.password": "${CLICKHOUSE_PASSWORD}", - "clickhouse.server.database": "${CLICKHOUSE_DATABASE}", "clickhouse.server.port": ${CLICKHOUSE_PORT}, # "clickhouse.table.name": "${CLICKHOUSE_TABLE}", "key.converter": "io.apicurio.registry.utils.converter.AvroConverter", @@ -80,7 +79,6 @@ else "clickhouse.server.url": "${CLICKHOUSE_HOST}", "clickhouse.server.user": "${CLICKHOUSE_USER}", "clickhouse.server.password": "${CLICKHOUSE_PASSWORD}", - "clickhouse.server.database": "${CLICKHOUSE_DATABASE}", "clickhouse.server.port": ${CLICKHOUSE_PORT}, "clickhouse.table.name": "${CLICKHOUSE_TABLE}", "key.converter": "io.confluent.connect.avro.AvroConverter", diff --git a/sink-connector/deploy/sink-connector-setup-schema-registry.sh b/sink-connector/deploy/sink-connector-setup-schema-registry.sh index 1d168dc6c..ad92eea9f 100755 --- a/sink-connector/deploy/sink-connector-setup-schema-registry.sh +++ b/sink-connector/deploy/sink-connector-setup-schema-registry.sh @@ -43,7 +43,6 @@ if [[ $2 == "apicurio" ]]; then "clickhouse.server.url": "${CLICKHOUSE_HOST}", "clickhouse.server.user": "${CLICKHOUSE_USER}", "clickhouse.server.password": "${CLICKHOUSE_PASSWORD}", - "clickhouse.server.database": "${CLICKHOUSE_DATABASE}", "clickhouse.server.port": ${CLICKHOUSE_PORT}, "clickhouse.table.name": "${CLICKHOUSE_TABLE}", "key.converter": "io.apicurio.registry.utils.converter.AvroConverter", @@ -90,7 +89,6 @@ else "clickhouse.server.url": "${CLICKHOUSE_HOST}", "clickhouse.server.user": "${CLICKHOUSE_USER}", "clickhouse.server.password": "${CLICKHOUSE_PASSWORD}", - "clickhouse.server.database": "${CLICKHOUSE_DATABASE}", "clickhouse.server.port": ${CLICKHOUSE_PORT}, "clickhouse.table.name": "${CLICKHOUSE_TABLE}", "key.converter": "io.confluent.connect.avro.AvroConverter", diff --git a/sink-connector/deploy/sink-connector-setup.sh b/sink-connector/deploy/sink-connector-setup.sh index b4350be43..5dd0e3c56 100755 --- a/sink-connector/deploy/sink-connector-setup.sh +++ b/sink-connector/deploy/sink-connector-setup.sh @@ -28,7 +28,6 @@ cat <com.altinity clickhouse-kafka-sink-connector - 0.0.8 + 0.0.9 jar ClickHouse Kafka Sink Connector Sinks data from Kafka into ClickHouse @@ -308,7 +308,7 @@ io.debezium debezium-core - 2.5.0.Beta1 + 2.7.0.Beta2 diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/ClickHouseSinkConnectorConfig.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/ClickHouseSinkConnectorConfig.java index ec64946c9..0335c1257 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/ClickHouseSinkConnectorConfig.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/ClickHouseSinkConnectorConfig.java @@ -3,6 +3,7 @@ import com.altinity.clickhouse.sink.connector.deduplicator.DeDuplicationPolicy; import com.altinity.clickhouse.sink.connector.deduplicator.DeDuplicationPolicyValidator; +import com.altinity.clickhouse.sink.connector.validators.DatabaseOverrideValidator; import com.altinity.clickhouse.sink.connector.validators.KafkaProviderValidator; import com.altinity.clickhouse.sink.connector.validators.TopicToTableValidator; import org.apache.kafka.common.config.AbstractConfig; @@ -103,6 +104,19 @@ static ConfigDef newConfigDef() { 0, ConfigDef.Width.NONE, ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_TOPICS_TABLES_MAP.toString()) + // Define overrides map for ClickHouse Database + .define( + ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_DATABASE_OVERRIDE_MAP.toString(), + Type.STRING, + "", + new DatabaseOverrideValidator(), + Importance.LOW, + "Map of source to destination database(override) (optional). Format : comma-separated tuples, e.g." + + " :,:,... ", + CONFIG_GROUP_CONNECTOR_CONFIG, + 0, + ConfigDef.Width.NONE, + ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_DATABASE_OVERRIDE_MAP.toString()) .define( ClickHouseSinkConnectorConfigVariables.BUFFER_COUNT.toString(), Type.LONG, @@ -181,17 +195,6 @@ static ConfigDef newConfigDef() { 2, ConfigDef.Width.NONE, ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_PASS.toString()) - .define( - ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_DATABASE.toString(), - Type.STRING, - null, - new ConfigDef.NonEmptyString(), - Importance.HIGH, - "ClickHouse database name", - CONFIG_GROUP_CLICKHOUSE_LOGIN_INFO, - 3, - ConfigDef.Width.NONE, - ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_DATABASE.toString()) .define( ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_PORT.toString(), Type.INT, @@ -447,6 +450,20 @@ static ConfigDef newConfigDef() { 6, ConfigDef.Width.NONE, ClickHouseSinkConnectorConfigVariables.MAX_QUEUE_SIZE.toString()) + .define( + ClickHouseSinkConnectorConfigVariables.REPLICA_STATUS_VIEW.toString(), + Type.STRING, + "CREATE VIEW IF NOT EXISTS %s.show_replica_status AS SELECT now() - " + + "fromUnixTimestamp(JSONExtractUInt(offset_val, 'ts_sec')) AS seconds_behind_source, " + + "toDateTime(fromUnixTimestamp(JSONExtractUInt(offset_val, 'ts_sec')), 'UTC') AS utc_time, " + + "fromUnixTimestamp(JSONExtractUInt(offset_val, 'ts_sec')) AS local_time," + + "* FROM %s FINAL", + Importance.HIGH, + "SQL query to get replica status, lag etc.", + CONFIG_GROUP_CONNECTOR_CONFIG, + 6, + ConfigDef.Width.NONE, + ClickHouseSinkConnectorConfigVariables.REPLICA_STATUS_VIEW.toString()) ; } diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/ClickHouseSinkConnectorConfigVariables.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/ClickHouseSinkConnectorConfigVariables.java index bdd6224f4..20001e092 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/ClickHouseSinkConnectorConfigVariables.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/ClickHouseSinkConnectorConfigVariables.java @@ -9,11 +9,11 @@ public enum ClickHouseSinkConnectorConfigVariables { DEDUPLICATION_POLICY("deduplication.policy"), CLICKHOUSE_TOPICS_TABLES_MAP("clickhouse.topic2table.map"), + CLICKHOUSE_DATABASE_OVERRIDE_MAP("clickhouse.database.override.map"), CLICKHOUSE_URL("clickhouse.server.url"), CLICKHOUSE_USER("clickhouse.server.user"), CLICKHOUSE_PASS("clickhouse.server.password"), - CLICKHOUSE_DATABASE("clickhouse.server.database"), CLICKHOUSE_PORT("clickhouse.server.port"), PROVIDER_CONFIG( "provider"), @@ -72,6 +72,7 @@ public enum ClickHouseSinkConnectorConfigVariables { RESTART_EVENT_LOOP_TIMEOUT_PERIOD("restart.event.loop.timeout.period.secs"), JDBC_PARAMETERS("clickhouse.jdbc.params"), + REPLICA_STATUS_VIEW("replica.status.view"), MAX_QUEUE_SIZE("sink.connector.max.queue.size"); private String label; diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/common/Utils.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/common/Utils.java index 46d29189d..5489c9503 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/common/Utils.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/common/Utils.java @@ -29,6 +29,68 @@ public class Utils { // Connector version, change every release public static final String VERSION = "1.0.0"; + /** + * Function to parse the topic to table configuration parameter + * + * @param input Delimiter separated list. + * @return key/value pair of configuration. + */ + public static Map parseSourceToDestinationDatabaseMap(String input) throws Exception { + Map srcToDestinationMap = new HashMap<>(); + boolean isInvalid = false; + + if(input == null || input.isEmpty()) { + return srcToDestinationMap; + } + + for (String str : input.split(",")) { + String[] tt = str.split(":"); + + if (tt.length != 2 || tt[0].trim().isEmpty() || tt[1].trim().isEmpty()) { + LOGGER.error( + Logging.logMessage( + "Invalid {} config format: {}", + ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_DATABASE_OVERRIDE_MAP.toString(), + input)); + return null; + } + + String srcDatabase = tt[0].trim(); + String dstDatabase = tt[1].trim(); + + if (!isValidDatabaseName(srcDatabase)) { + LOGGER.error( + Logging.logMessage( + "database name{} should have at least 2 " + + "characters, start with _a-zA-Z, and only contains " + + "_$a-zA-z0-9", + srcDatabase)); + isInvalid = true; + } + + if (!isValidDatabaseName(dstDatabase)) { + LOGGER.error( + Logging.logMessage( + "database name{} should have at least 2 " + + "characters, start with _a-zA-Z, and only contains " + + "_$a-zA-z0-9", + dstDatabase)); + isInvalid = true; + } + + if (srcToDestinationMap.containsKey(srcDatabase)) { + LOGGER.error(Logging.logMessage("source database name {} is duplicated", srcDatabase)); + isInvalid = true; + } + + srcToDestinationMap.put(tt[0].trim(), tt[1].trim()); + } + if (isInvalid) { + throw new Exception("Invalid clickhouse table"); + } + return srcToDestinationMap; + } + /** * Function to parse the topic to table configuration parameter * @@ -108,4 +170,28 @@ public static boolean isValidTable(String tableName) { return true; } + public static boolean isValidDatabaseName(String dbName) { + // Check if the name is empty or longer than 63 characters + if (dbName == null || dbName.isEmpty() || dbName.length() > 63) { + return false; + } + + // Check the first character: must be a letter or an underscore + char firstChar = dbName.charAt(0); + if (!(Character.isLetter(firstChar) || firstChar == '_')) { + return false; + } + + // Check the remaining characters + for (int i = 1; i < dbName.length(); i++) { + char ch = dbName.charAt(i); + if (!(Character.isLetterOrDigit(ch) || ch == '_' || ch == '.')) { + return false; + } + } + + return true; + } + + } \ No newline at end of file diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DBMetadata.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DBMetadata.java index d6e4e538f..fbccff1ea 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DBMetadata.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DBMetadata.java @@ -125,10 +125,10 @@ public MutablePair getTableEngineUsingShowTable(ClickHouse } rs.close(); stmt.close(); - log.info("ResultSet" + rs); + log.info("getTableEngineUsingShowTable ResultSet" + rs); } } catch(Exception e) { - log.error("getTableEngineUsingShowTable exception", e); + log.info("getTableEngineUsingShowTable exception", e); } return result; @@ -214,14 +214,14 @@ public MutablePair getTableEngineUsingSystemTables(final C String response = rs.getString(1); result = getEngineFromResponse(response); } else { - log.error("Error: Table not found in system tables:" + tableName + " Database:" + database); + log.debug("Error: Table not found in system tables:" + tableName + " Database:" + database); } rs.close(); stmt.close(); - log.info("ResultSet" + rs); + log.info("getTableEngineUsingSystemTables ResultSet" + rs); } } catch(Exception e) { - log.error("getTableEngineUsingSystemTables exception", e); + log.debug("getTableEngineUsingSystemTables exception", e); } return result; diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DbWriter.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DbWriter.java index bae2efd2a..3fd0a12ef 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DbWriter.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/DbWriter.java @@ -91,7 +91,7 @@ public DbWriter( new ClickHouseCreateDatabase().createNewDatabase(this.conn, database); } } catch(Exception e) { - log.error("Error creating Database", database); + log.error("Error creating Database: " + database); } MutablePair response = metadata.getTableEngine(this.conn, database, tableName); this.engine = response.getLeft(); @@ -108,7 +108,8 @@ public DbWriter( //ToDO: Is this a reliable way of checking if the table exists already. if (this.engine == null) { if (this.config.getBoolean(ClickHouseSinkConnectorConfigVariables.AUTO_CREATE_TABLES.toString())) { - log.info(String.format("**** Task(%s), AUTO CREATE TABLE (%s) *** ",taskId, tableName)); + log.info(String.format("**** Task(%s), AUTO CREATE TABLE (%s) Database(%s) *** ",taskId, tableName, + database)); ClickHouseAutoCreateTable act = new ClickHouseAutoCreateTable(); try { Field[] fields = null; @@ -119,10 +120,11 @@ public DbWriter( } boolean useReplicatedReplacingMergeTree = this.config.getBoolean( ClickHouseSinkConnectorConfigVariables.AUTO_CREATE_TABLES_REPLICATED.toString()); + String rmtDeleteColumn = this.config.getString(ClickHouseSinkConnectorConfigVariables.REPLACING_MERGE_TREE_DELETE_COLUMN.toString()); act.createNewTable(record.getPrimaryKey(), tableName, database, fields, this.conn, - isNewReplacingMergeTreeEngine, useReplicatedReplacingMergeTree); + isNewReplacingMergeTreeEngine, useReplicatedReplacingMergeTree, rmtDeleteColumn); } catch (Exception e) { - log.error("**** Error creating table ***" + tableName, e); + log.error(String.format("**** Error creating table(%s), database(%s) ***",tableName, database), e); } } else { log.error("********* AUTO CREATE DISABLED, Table does not exist, please enable it by setting auto.create.tables=true"); diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/QueryFormatter.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/QueryFormatter.java index 28f39c62c..75bd4f688 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/QueryFormatter.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/QueryFormatter.java @@ -93,7 +93,8 @@ public MutablePair> getInsertQueryUsingInputFunctio colNameToIndexMap.put(sourceColumnName, index++); } } else { - log.error(String.format("Table Name: %s, Column(%s) ignored", tableName, sourceColumnNameWithBackTicks)); + log.error(String.format("Table Name: %s, Database: %s, Column(%s) ignored", tableName, dbName, + sourceColumnNameWithBackTicks)); } } diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/batch/PreparedStatementExecutor.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/batch/PreparedStatementExecutor.java index 35ee1cdc0..a7b321d08 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/batch/PreparedStatementExecutor.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/batch/PreparedStatementExecutor.java @@ -6,7 +6,6 @@ import com.altinity.clickhouse.sink.connector.common.SnowFlakeId; import com.altinity.clickhouse.sink.connector.converters.ClickHouseConverter; import com.altinity.clickhouse.sink.connector.converters.ClickHouseDataTypeMapper; -import com.altinity.clickhouse.sink.connector.db.BaseDbWriter; import com.altinity.clickhouse.sink.connector.db.DBMetadata; import com.altinity.clickhouse.sink.connector.metadata.TableMetaDataWriter; import com.altinity.clickhouse.sink.connector.model.BlockMetaData; @@ -35,6 +34,7 @@ import static com.altinity.clickhouse.sink.connector.db.batch.CdcOperation.getCdcSectionBasedOnOperation; public class PreparedStatementExecutor { + private static final Logger log = LogManager.getLogger(PreparedStatementExecutor.class); private String replacingMergeTreeDeleteColumn; private boolean replacingMergeTreeWithIsDeletedColumn; @@ -45,23 +45,21 @@ public class PreparedStatementExecutor { private ZoneId serverTimeZone; + private String databaseName; + public PreparedStatementExecutor(String replacingMergeTreeDeleteColumn, boolean replacingMergeTreeWithIsDeletedColumn, String signColumn, String versionColumn, - ClickHouseConnection conn, ZoneId serverTimeZone) { + String databaseName, ZoneId serverTimeZone) { this.replacingMergeTreeDeleteColumn = replacingMergeTreeDeleteColumn; this.replacingMergeTreeWithIsDeletedColumn = replacingMergeTreeWithIsDeletedColumn; this.signColumn = signColumn; this.versionColumn = versionColumn; - this.serverTimeZone = serverTimeZone; - // serverTimeZone = BaseDbWriter. - //serverTimeZone = new DBMetadata().getServerTimeZone(conn); + this.databaseName = databaseName; } - private static final Logger log = LogManager.getLogger(PreparedStatementExecutor.class); - /** * Function to iterate through records and add it to JDBC prepared statement * batch @@ -81,12 +79,14 @@ public boolean addToPreparedStatementBatch(String topicName, Map>, List> entry = iter.next(); String insertQuery = entry.getKey().getKey(); - log.info("*** QUERY***" + insertQuery); + log.info(String.format("*** INSERT QUERY for Database(%s) ***: %s", databaseName, insertQuery)); // Create Hashmap of PreparedStatement(Query) -> Set of records // because the data will contain a mix of SQL statements(multiple columns) - if(false == executePreparedStatement(insertQuery, topicName, entry, bmd, config, conn, tableName, columnToDataTypeMap, engine)) { - log.error("**** ERROR: executing prepared statement"); + if(false == executePreparedStatement(insertQuery, topicName, entry, bmd, config, + conn, tableName, columnToDataTypeMap, engine)) { + log.error(String.format("**** ERROR: executing prepared statement for Database(%s), " + + "table(%s), Query(%s) ****", databaseName, tableName, insertQuery)); result = false; break; } else { @@ -138,17 +138,17 @@ private boolean executePreparedStatement(String insertQuery, String topicName, if (CdcRecordState.CDC_RECORD_STATE_BEFORE == getCdcSectionBasedOnOperation(record.getCdcOperation())) { insertPreparedStatement(entry.getKey().right, ps, record.getBeforeModifiedFields(), record, record.getBeforeStruct(), - true, config, columnToDataTypeMap, engine); + true, config, columnToDataTypeMap, engine, tableName); } else if (CdcRecordState.CDC_RECORD_STATE_AFTER == getCdcSectionBasedOnOperation(record.getCdcOperation())) { insertPreparedStatement(entry.getKey().right, ps, record.getAfterModifiedFields(), record, record.getAfterStruct(), - false, config, columnToDataTypeMap, engine); + false, config, columnToDataTypeMap, engine, tableName); } else if (CdcRecordState.CDC_RECORD_STATE_BOTH == getCdcSectionBasedOnOperation(record.getCdcOperation())) { if (engine != null && engine.getEngine().equalsIgnoreCase(DBMetadata.TABLE_ENGINE.COLLAPSING_MERGE_TREE.getEngine())) { insertPreparedStatement(entry.getKey().right, ps, record.getBeforeModifiedFields(), record, record.getBeforeStruct(), - true, config, columnToDataTypeMap, engine); + true, config, columnToDataTypeMap, engine, tableName); } insertPreparedStatement(entry.getKey().right, ps, record.getAfterModifiedFields(), record, record.getAfterStruct(), - false, config, columnToDataTypeMap, engine); + false, config, columnToDataTypeMap, engine, tableName); } else { log.error("INVALID CDC RECORD STATE"); } @@ -161,14 +161,17 @@ private boolean executePreparedStatement(String insertQuery, String topicName, long taskId = config.getLong(ClickHouseSinkConnectorConfigVariables.TASK_ID.toString()); log.info("*************** EXECUTED BATCH Successfully " + "Records: " + batch.size() + "************** " + - "task(" + taskId + ")" + " Thread ID: " + Thread.currentThread().getName() + " Result: " + - batchResult.toString()); + "task(" + taskId + ")" + " Thread ID: " + + Thread.currentThread().getName() + " Result: " + + batchResult.toString() + " Database: " + + databaseName + " Table: " + tableName); result.set(true); } catch (Exception e) { Metrics.updateErrorCounters(topicName, entry.getValue().size()); - log.error("******* ERROR inserting Batch *****************", e); + log.error(String.format("******* ERROR inserting Batch Database(%s), Table(%s) *****************", + databaseName, tableName), e); failedRecords.addAll(batch); throw new RuntimeException(e); } @@ -177,11 +180,13 @@ private boolean executePreparedStatement(String insertQuery, String topicName, try { ps = conn.prepareStatement("TRUNCATE TABLE " + databaseName + "." + tableName); } catch (SQLException e) { + log.error("*** Error: Truncate table statement error ****", e); throw new RuntimeException(e); } try { ps.execute(); } catch (SQLException e) { + log.error("*** Error: Truncate table statement execute error ****", e); throw new RuntimeException(e); } } @@ -200,7 +205,7 @@ public void insertPreparedStatement(Map columnNameToIndexMap, P ClickHouseStruct record, Struct struct, boolean beforeSection, ClickHouseSinkConnectorConfig config, Map columnNameToDataTypeMap, - DBMetadata.TABLE_ENGINE engine) throws Exception { + DBMetadata.TABLE_ENGINE engine, String tableName) throws Exception { // int index = 1; @@ -241,8 +246,15 @@ public void insertPreparedStatement(Map columnNameToIndexMap, P // Struct .get throws a DataException // if the field is not present. // If the record was not supplied, we need to set it as null. - ps.setNull(index, Types.OTHER); + // Ignore version and sign columns. + if(colName.equalsIgnoreCase(versionColumn) || colName.equalsIgnoreCase(signColumn) || + colName.equalsIgnoreCase(replacingMergeTreeDeleteColumn)) { + } else { + log.error(String.format("********** ERROR: Database(%s), Table(%s), ClickHouse column %s not present in source ************", databaseName, tableName, colName)); + log.error(String.format("********** ERROR: Database(%s), Table(%s), Setting column %s to NULL might fail for non-nullable columns ************", databaseName, tableName, colName)); + } + ps.setNull(index, Types.OTHER); continue; } diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAutoCreateTable.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAutoCreateTable.java index 41d3f63a6..0d8b906ad 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAutoCreateTable.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAutoCreateTable.java @@ -26,11 +26,11 @@ public class ClickHouseAutoCreateTable extends ClickHouseTableOperationsBase{ public void createNewTable(ArrayList primaryKey, String tableName, String databaseName, Field[] fields, ClickHouseConnection connection, boolean isNewReplacingMergeTree, - boolean useReplicatedReplacingMergeTree) throws SQLException { + boolean useReplicatedReplacingMergeTree, String rmtDeleteColumn) throws SQLException { Map colNameToDataTypeMap = this.getColumnNameToCHDataTypeMapping(fields); String createTableQuery = this.createTableSyntax(primaryKey, tableName, databaseName, fields, colNameToDataTypeMap, - isNewReplacingMergeTree, useReplicatedReplacingMergeTree); - log.info("**** AUTO CREATE TABLE " + createTableQuery); + isNewReplacingMergeTree, useReplicatedReplacingMergeTree, rmtDeleteColumn); + log.info(String.format("**** AUTO CREATE TABLE for database(%s), Query :%s)", databaseName, createTableQuery)); // ToDO: need to run it before a session is created. this.runQuery(createTableQuery, connection); } @@ -45,11 +45,17 @@ public void createNewTable(ArrayList primaryKey, String tableName, Strin public java.lang.String createTableSyntax(ArrayList primaryKey, String tableName, String databaseName, Field[] fields, Map columnToDataTypesMap, boolean isNewReplacingMergeTreeEngine, - boolean useReplicatedReplacingMergeTree) { + boolean useReplicatedReplacingMergeTree, + String rmtDeleteColumn) { StringBuilder createTableSyntax = new StringBuilder(); - createTableSyntax.append(CREATE_TABLE).append(" ").append(databaseName).append(".").append("`").append(tableName).append("`").append("("); + createTableSyntax.append(CREATE_TABLE).append(" ").append(databaseName).append(".").append("`").append(tableName).append("`"); + if(useReplicatedReplacingMergeTree == true) { + createTableSyntax.append(" ON CLUSTER `{cluster}` "); + } + + createTableSyntax.append("("); for(Field f: fields) { String colName = f.name(); @@ -78,6 +84,10 @@ public java.lang.String createTableSyntax(ArrayList primaryKey, String t String isDeletedColumn = IS_DELETED_COLUMN; + if(rmtDeleteColumn != null && !rmtDeleteColumn.isEmpty()) { + isDeletedColumn = rmtDeleteColumn; + } + if(isNewReplacingMergeTreeEngine == true) { createTableSyntax.append("`").append(VERSION_COLUMN).append("` ").append(VERSION_COLUMN_DATA_TYPE).append(","); createTableSyntax.append("`").append(isDeletedColumn).append("` ").append(IS_DELETED_COLUMN_DATA_TYPE); @@ -91,12 +101,12 @@ public java.lang.String createTableSyntax(ArrayList primaryKey, String t if(isNewReplacingMergeTreeEngine == true ){ if(useReplicatedReplacingMergeTree == true) { - createTableSyntax.append(String.format("Engine=ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/%s', '{replica}', %s, %s)", tableName, VERSION_COLUMN, isDeletedColumn)); + createTableSyntax.append(String.format("Engine=ReplicatedReplacingMergeTree(%s, %s)", VERSION_COLUMN, isDeletedColumn)); } else createTableSyntax.append(" Engine=ReplacingMergeTree(").append(VERSION_COLUMN).append(",").append(isDeletedColumn).append(")"); } else { if(useReplicatedReplacingMergeTree == true) { - createTableSyntax.append(String.format("Engine=ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/%s', '{replica}', %s)", tableName, VERSION_COLUMN)); + createTableSyntax.append(String.format("Engine=ReplicatedReplacingMergeTree(%s)", VERSION_COLUMN)); } else createTableSyntax.append("ENGINE = ReplacingMergeTree(").append(VERSION_COLUMN).append(")"); } diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseTableOperationsBase.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseTableOperationsBase.java index 108804044..082c1cc87 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseTableOperationsBase.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseTableOperationsBase.java @@ -80,10 +80,7 @@ public Map getColumnNameToCHDataTypeMapping(Field[] fields) { }else { log.error(" **** DATA TYPE MAPPING not found: " + "TYPE:" + type.getName() + "SCHEMA NAME:" + schemaName); } -// -// if(columnToDataTypesMap.isEmpty() == false) { -// String createTableQuery = this.createTableSyntax(primaryKey, tableName, columnToDataTypesMap); -// } + } return columnToDataTypesMap; diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/ClickHouseBatchRunnable.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/ClickHouseBatchRunnable.java index 0a5df9349..b74f4fe8e 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/ClickHouseBatchRunnable.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/ClickHouseBatchRunnable.java @@ -61,6 +61,9 @@ public class ClickHouseBatchRunnable implements Runnable { private List currentBatch = null; + + private Map databaseOverrideMap = new HashMap<>(); + public ClickHouseBatchRunnable(LinkedBlockingQueue> records, ClickHouseSinkConnectorConfig config, Map topic2TableMap) { @@ -78,6 +81,14 @@ public ClickHouseBatchRunnable(LinkedBlockingQueue> recor this.dbCredentials = parseDBConfiguration(); this.systemConnection = createConnection(); + + + try { + this.databaseOverrideMap = Utils.parseSourceToDestinationDatabaseMap(this.config. + getString(ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_DATABASE_OVERRIDE_MAP.toString())); + } catch (Exception e) { + log.error("Error parsing database override map" + e); + } } private ClickHouseConnection createConnection() { @@ -109,7 +120,6 @@ private DBCredentials parseDBConfiguration() { DBCredentials dbCredentials = new DBCredentials(); dbCredentials.setHostName(config.getString(ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_URL.toString())); - dbCredentials.setDatabase(config.getString(ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_DATABASE.toString())); dbCredentials.setPort(config.getInt(ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_PORT.toString())); dbCredentials.setUserName(config.getString(ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_USER.toString())); dbCredentials.setPassword(config.getString(ClickHouseSinkConnectorConfigVariables.CLICKHOUSE_PASS.toString())); @@ -139,10 +149,16 @@ public void run() { if(currentBatch == null) { currentBatch = records.poll(); + if(currentBatch == null) { + // No records in the queue. + continue; + } } else { log.debug("***** RETRYING the same batch again"); - } + + + ///// ***** START PROCESSING BATCH ************************** // Step 1: Add to Inflight batches. DebeziumOffsetManagement.addToBatchTimestamps(currentBatch); @@ -192,6 +208,7 @@ public void run() { try { Thread.sleep(10000); } catch (InterruptedException ex) { + log.error("******* ERROR **** Thread interrupted *********", ex); throw new RuntimeException(ex); } } @@ -247,7 +264,6 @@ public ZoneId getServerTimeZone(ClickHouseSinkConnectorConfig config) { try { if(!userProvidedTimeZone.isEmpty()) { userProvidedTimeZoneId = ZoneId.of(userProvidedTimeZone); - //log.info("**** OVERRIDE TIMEZONE for DateTime:" + userProvidedTimeZone); } } catch (Exception e){ log.error("**** Error parsing user provided timezone:"+ userProvidedTimeZone + e.toString()); @@ -271,25 +287,34 @@ private boolean processRecordsByTopic(String topicName, List r String tableName = getTableFromTopic(topicName); // Note: getting records.get(0) is safe as the topic name is same for all records. ClickHouseStruct firstRecord = records.get(0); - ClickHouseConnection databaseConn = getClickHouseConnection(firstRecord.getDatabase()); - DbWriter writer = getDbWriterForTable(topicName, tableName, firstRecord.getDatabase(), firstRecord, databaseConn); + String databaseName = firstRecord.getDatabase(); + + // Check if user has overridden the database name. + if(this.databaseOverrideMap.containsKey(firstRecord.getDatabase())) + databaseName = this.databaseOverrideMap.get(firstRecord.getDatabase()); + + ClickHouseConnection databaseConn = getClickHouseConnection(databaseName); + + DbWriter writer = getDbWriterForTable(topicName, tableName, databaseName, firstRecord, databaseConn); PreparedStatementExecutor preparedStatementExecutor = new PreparedStatementExecutor(writer.getReplacingMergeTreeDeleteColumn(), writer.isReplacingMergeTreeWithIsDeletedColumn(), writer.getSignColumn(), writer.getVersionColumn(), - writer.getConnection(), getServerTimeZone(this.config)); + writer.getDatabaseName(), getServerTimeZone(this.config)); if(writer == null || writer.wasTableMetaDataRetrieved() == false) { - log.error("*** TABLE METADATA not retrieved, retrying"); + log.error(String.format("*** TABLE METADATA not retrieved for Database(%), table(%s) retrying", + writer.getDatabaseName(), writer.getTableName())); if(writer == null) { - writer = getDbWriterForTable(topicName, tableName, firstRecord.getDatabase(), firstRecord, databaseConn); + writer = getDbWriterForTable(topicName, tableName, databaseName, firstRecord, databaseConn); } if(writer.wasTableMetaDataRetrieved() == false) writer.updateColumnNameToDataTypeMap(); if(writer == null || writer.wasTableMetaDataRetrieved() == false ) { - log.error("*** TABLE METADATA not retrieved, retrying on next attempt"); + log.error(String.format("*** TABLE METADATA not retrieved for Database(%s), table(%s), " + + "retrying on next attempt", writer.getDatabaseName(), writer.getTableName())); return false; } } @@ -317,8 +342,10 @@ private boolean processRecordsByTopic(String topicName, List r if (this.config.getBoolean(ClickHouseSinkConnectorConfigVariables.ENABLE_KAFKA_OFFSET.toString())) { log.info("***** KAFKA OFFSET MANAGEMENT ENABLED *****"); - DbKafkaOffsetWriter dbKafkaOffsetWriter = new DbKafkaOffsetWriter(dbCredentials.getHostName(), dbCredentials.getPort(), dbCredentials.getDatabase(), - "topic_offset_metadata", dbCredentials.getUserName(), dbCredentials.getPassword(), this.config, databaseConn); + DbKafkaOffsetWriter dbKafkaOffsetWriter = new DbKafkaOffsetWriter(dbCredentials.getHostName(), + dbCredentials.getPort(), dbCredentials.getDatabase(), + "topic_offset_metadata", dbCredentials.getUserName(), dbCredentials.getPassword(), + this.config, databaseConn); try { dbKafkaOffsetWriter.insertTopicOffsetMetadata(partitionToOffsetMap); } catch (SQLException e) { @@ -336,8 +363,10 @@ private boolean processRecordsByTopic(String topicName, List r * @param queryToRecordsMap * @return */ - private boolean flushRecordsToClickHouse(String topicName, DbWriter writer, Map>, - List> queryToRecordsMap, BlockMetaData bmd, long maxBufferSize, PreparedStatementExecutor preparedStatementExecutor) throws Exception { + private boolean flushRecordsToClickHouse(String topicName, DbWriter writer, + Map>, + List> queryToRecordsMap, BlockMetaData bmd, + long maxBufferSize, PreparedStatementExecutor preparedStatementExecutor) throws Exception { boolean result = false; diff --git a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/DebeziumOffsetManagement.java b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/DebeziumOffsetManagement.java index bcc6b7942..060b16d03 100644 --- a/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/DebeziumOffsetManagement.java +++ b/sink-connector/src/main/java/com/altinity/clickhouse/sink/connector/executor/DebeziumOffsetManagement.java @@ -116,6 +116,7 @@ static synchronized public boolean checkIfBatchCanBeCommitted(List:,:,..."); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + +} diff --git a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/UtilsTest.java b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/UtilsTest.java index 4b5ae01b7..d1af8621d 100644 --- a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/UtilsTest.java +++ b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/UtilsTest.java @@ -31,5 +31,27 @@ public void testGetTableNameFromTopic() { Assert.assertEquals(tableName, "employees"); } + + @Test + public void testIsValidDatabase() { + String database = "a".repeat(64); + boolean result = Utils.isValidDatabaseName(database); + Assert.assertFalse(result); + + String validDatabase = "test"; + boolean validResult = Utils.isValidDatabaseName(validDatabase); + Assert.assertTrue(validResult); + + // Database name with numbers. + String databaseWithNumbers = "test123"; + boolean resultWithNumbers = Utils.isValidDatabaseName(databaseWithNumbers); + Assert.assertTrue(resultWithNumbers); + + // Database name with special characters. + String databaseWithSpecialCharacters = "test_123"; + boolean resultWithSpecialCharacters = Utils.isValidDatabaseName(databaseWithSpecialCharacters); + Assert.assertTrue(resultWithSpecialCharacters); + + } } diff --git a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/DbWriterTest.java b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/DbWriterTest.java index 08da8d902..32d86013a 100644 --- a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/DbWriterTest.java +++ b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/DbWriterTest.java @@ -283,7 +283,7 @@ public void testGetClickHouseDataType() { ClickHouseConnection conn = DbWriter.createConnection(jdbcUrl, "client_1", userName, password, config); DbWriter dbWriter = new DbWriter(hostName, port, database, tableName, userName, password, config, null, conn); PreparedStatementExecutor preparedStatementExecutor = new PreparedStatementExecutor(null, - false, null, null, dbWriter.getConnection(), ZoneId.of("UTC")); + false, null, null, database, ZoneId.of("UTC")); ClickHouseDataType dt1 = preparedStatementExecutor.getClickHouseDataType("Min_Date", colNameToDataTypeMap); Assert.assertTrue(dt1 == ClickHouseDataType.Date); diff --git a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAutoCreateTableTest.java b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAutoCreateTableTest.java index 060741219..e8bbe1e9c 100644 --- a/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAutoCreateTableTest.java +++ b/sink-connector/src/test/java/com/altinity/clickhouse/sink/connector/db/operations/ClickHouseAutoCreateTableTest.java @@ -125,7 +125,7 @@ public void testCreateTableSyntax() { ClickHouseAutoCreateTable act = new ClickHouseAutoCreateTable(); String query = act.createTableSyntax(primaryKeys, "auto_create_table", "employees", - createFields(), this.columnToDataTypesMap, false, false); + createFields(), this.columnToDataTypesMap, false, false, null); System.out.println("QUERY" + query); Assert.assertTrue(query.equalsIgnoreCase("CREATE TABLE employees.`auto_create_table`(`customerName` String NOT NULL,`occupation` String NOT NULL,`quantity` Int32 NOT NULL,`amount_1` Float32 NOT NULL,`amount` Float64 NOT NULL,`employed` Bool NOT NULL,`blob_storage` String NOT NULL,`blob_storage_scale` Decimal NOT NULL,`json_output` JSON,`max_amount` Float64 NOT NULL,`_sign` Int8,`_version` UInt64) ENGINE = ReplacingMergeTree(_version) PRIMARY KEY(customerName) ORDER BY(customerName)")); //Assert.assertTrue(query.equalsIgnoreCase("CREATE TABLE auto_create_table(`customerName` String NOT NULL,`occupation` String NOT NULL,`quantity` Int32 NOT NULL,`amount_1` Float32 NOT NULL,`amount` Float64 NOT NULL,`employed` Bool NOT NULL,`blob_storage` String NOT NULL,`blob_storage_scale` Decimal NOT NULL,`json_output` JSON,`max_amount` Float64 NOT NULL,`_sign` Int8,`_version` UInt64) ENGINE = ReplacingMergeTree(_version) PRIMARY KEY(customerName) ORDER BY (customerName)")); @@ -137,7 +137,7 @@ public void testCreateTableEmptyPrimaryKey() { ClickHouseAutoCreateTable act = new ClickHouseAutoCreateTable(); String query = act.createTableSyntax(null, "auto_create_table", "employees", createFields(), - this.columnToDataTypesMap, false, false); + this.columnToDataTypesMap, false, false, null); String expectedQuery = "CREATE TABLE employees.`auto_create_table`(`customerName` String NOT NULL,`occupation` String NOT NULL,`quantity` Int32 NOT NULL,`amount_1` Float32 NOT NULL,`amount` Float64 NOT NULL,`employed` Bool NOT NULL,`blob_storage` String NOT NULL,`blob_storage_scale` Decimal NOT NULL,`json_output` JSON,`max_amount` Float64 NOT NULL,`_sign` Int8,`_version` UInt64) ENGINE = ReplacingMergeTree(_version) ORDER BY tuple()"; Assert.assertTrue(query.equalsIgnoreCase(expectedQuery)); @@ -151,7 +151,7 @@ public void testCreateTableMultiplePrimaryKeys() { ClickHouseAutoCreateTable act = new ClickHouseAutoCreateTable(); String query = act.createTableSyntax(primaryKeys, "auto_create_table", "customers", createFields(), - this.columnToDataTypesMap, false, false); + this.columnToDataTypesMap, false, false, null); String expectedQuery = "CREATE TABLE customers.`auto_create_table`(`customerName` String NOT NULL,`occupation` String NOT NULL,`quantity` Int32 NOT NULL,`amount_1` Float32 NOT NULL,`amount` Float64 NOT NULL,`employed` Bool NOT NULL,`blob_storage` String NOT NULL,`blob_storage_scale` Decimal NOT NULL,`json_output` JSON,`max_amount` Float64 NOT NULL,`_sign` Int8,`_version` UInt64) ENGINE = ReplacingMergeTree(_version) ORDER BY tuple()"; Assert.assertTrue(query.equalsIgnoreCase(expectedQuery)); @@ -181,7 +181,7 @@ public void testCreateNewTable() { try { act.createNewTable(primaryKeys, "auto_create_table", "default", this.createFields(), writer.getConnection(), - false, false); + false, false, null); } catch(SQLException se) { Assert.assertTrue(false); } diff --git a/sink-connector/tests/integration/env/docker-compose.yml b/sink-connector/tests/integration/env/docker-compose.yml index 969603d29..7c6733b67 100644 --- a/sink-connector/tests/integration/env/docker-compose.yml +++ b/sink-connector/tests/integration/env/docker-compose.yml @@ -4,7 +4,7 @@ version: "2.3" services: mysql-master: container_name: mysql-master - image: docker.io/bitnami/mysql:8.0 + image: docker.io/bitnami/mysql:8.0.36 restart: "no" expose: - "3306" diff --git a/sink-connector/tests/integration/requirements.txt b/sink-connector/tests/integration/requirements.txt index a89621f2e..2e9be8ba7 100644 --- a/sink-connector/tests/integration/requirements.txt +++ b/sink-connector/tests/integration/requirements.txt @@ -1,5 +1,10 @@ -docker-compose==1.29.2 testflows==2.1.5 -docker==6.1.3 +python-dateutil==2.9.0 +numpy==1.26.4 +pyarrow==16.1.0 +pandas==2.2.0 +PyYAML==5.3.1 +docker-compose==1.29.2 awscli==1.27.36 -requests==2.31.0 +docker==6.1.3 +requests==2.31.0 \ No newline at end of file diff --git a/sink-connector/tests/integration/tests/manual_scripts/sink-connector-setup-schema-registry-local.sh b/sink-connector/tests/integration/tests/manual_scripts/sink-connector-setup-schema-registry-local.sh index 8faeebbe4..062b51483 100755 --- a/sink-connector/tests/integration/tests/manual_scripts/sink-connector-setup-schema-registry-local.sh +++ b/sink-connector/tests/integration/tests/manual_scripts/sink-connector-setup-schema-registry-local.sh @@ -36,7 +36,6 @@ cat <