Merge pull request #684 from Altinity/new_tests

New tests for TestFlows that incldue coverage for the new release

Author: subkanthi

sink-connector-lightweight/tests/integration/env/auto/configs/logs/log4j2.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="INFO">
+    <Appenders>
+        <Console name="console" target="SYSTEM_OUT">
+<!--             <JSONLayout compact="true" eventEol="true" properties="true" stacktraceAsString="true" includeTimeMillis="true" />-->
+
+            <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level - %msg%n"/>
+        </Console>
+    </Appenders>
+    <Loggers>
+        <!-- <Logger name="org.slf4j" level="DEBUG"
+                additivity="false">
+        <AppenderRef ref="console"/>
+        </Logger>-->
+            <Logger name="com.clickhouse" level="ERROR"
+                additivity="false">
+            <AppenderRef ref="console"/>
+        </Logger>
+
+    <Root level="info" additivity="false">
+            <AppenderRef ref="console" />
+        </Root>
+    </Loggers>
+</Configuration>

sink-connector-lightweight/tests/integration/env/auto_replicated/configs/logs/log4j2.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="INFO">
+    <Appenders>
+        <Console name="console" target="SYSTEM_OUT">
+<!--             <JSONLayout compact="true" eventEol="true" properties="true" stacktraceAsString="true" includeTimeMillis="true" />-->
+
+            <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level - %msg%n"/>
+        </Console>
+    </Appenders>
+    <Loggers>
+        <!-- <Logger name="org.slf4j" level="DEBUG"
+                additivity="false">
+        <AppenderRef ref="console"/>
+        </Logger>-->
+            <Logger name="com.clickhouse" level="ERROR"
+                additivity="false">
+            <AppenderRef ref="console"/>
+        </Logger>
+
+    <Root level="info" additivity="false">
+            <AppenderRef ref="console" />
+        </Root>
+    </Loggers>
+</Configuration>

sink-connector-lightweight/tests/integration/helpers/cluster.py

@@ -596,15 +596,14 @@ def up(self, timeout=30 * 60):
                     "IMAGE_DEPENDENCY_PROXY", ""
                 )
                 self.environ["COMPOSE_HTTP_TIMEOUT"] = "300"
-                self.environ["CLICKHOUSE_TESTS_SERVER_BIN_PATH"] = (
-                    self.clickhouse_binary_path
-                )
-                self.environ["CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH"] = (
-                    self.clickhouse_odbc_bridge_binary_path
-                    or os.path.join(
-                        os.path.dirname(self.clickhouse_binary_path),
-                        "clickhouse-odbc-bridge",
-                    )
+                self.environ[
+                    "CLICKHOUSE_TESTS_SERVER_BIN_PATH"
+                ] = self.clickhouse_binary_path
+                self.environ[
+                    "CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH"
+                ] = self.clickhouse_odbc_bridge_binary_path or os.path.join(
+                    os.path.dirname(self.clickhouse_binary_path),
+                    "clickhouse-odbc-bridge",
                 )
                 self.environ["CLICKHOUSE_TESTS_DIR"] = self.configs_dir
 
@@ -890,6 +889,10 @@ def query(
                     if raise_on_exception:
                         raise QueryRuntimeException(r.output)
                     assert False, error(r.output)
+                elif "ERROR" in r.output:
+                    if raise_on_exception:
+                        raise QueryRuntimeException(r.output)
+                    assert False, error(r.output)
 
         return r
 
@@ -970,16 +973,23 @@ def parse_value(input_string):
         else:
             raise ValueError(f"Failed to parse value from string: {input_string}")
 
-    def start_sink_connector(self, timeout=300, config_file=None):
+    def start_sink_connector(self, timeout=300, config_file=None, log_file=None):
         """Start ClickHouse Sink Connector."""
         if config_file is None:
             config_file = "config.yml"
 
         config_file = config_file.rsplit("/", 1)[-1]
 
         with Given("I start ClickHouse Sink Connector"):
+            command = f"java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Xms4g -Xmx4g "
+
+            if log_file is None:
+                log_file = "log4j2.xml"
+
+            command += f"-Dlog4j2.configurationFile={log_file} -jar /app.jar /configs/{config_file} com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication >> /logs/sink-connector-lt.log 2>&1 &"
+
             start_command = self.command(
-                command=f"java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -jar /app.jar /configs/{config_file} com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication > /logs/sink-connector-lt.log 2>&1 &",
+                command=command,
                 exitcode=0,
                 timeout=timeout,
             )

sink-connector-lightweight/tests/integration/helpers/create_config.py

@@ -26,7 +26,8 @@ def update(self, new_data):
 
     def remove(self, key):
         """Remove the ClickHouse Sink Connector configuration key."""
-        self.data.pop(key)
+        if key in self.data:
+            self.data.pop(key)
 
     def display_config(self):
         """Print out the ClickHouse Sink Connector configuration."""

sink-connector-lightweight/tests/integration/helpers/default_config.py

@@ -7,12 +7,10 @@
     "database.user": "root",
     "database.password": "root",
     "database.server.name": "ER54",
-    "database.include.list": "test",
     "clickhouse.server.url": "clickhouse",
     "clickhouse.server.user": "root",
     "clickhouse.server.password": "root",
     "clickhouse.server.port": "8123",
-    "clickhouse.server.database": "test",
     "database.allowPublicKeyRetrieval": "true",
     "snapshot.mode": "initial",
     "offset.flush.interval.ms": "5000",

sink-connector-lightweight/tests/integration/regression_auto.py

@@ -123,11 +123,15 @@
     ),
     "/mysql to clickhouse replication/auto table creation/schema only/*": (
         Skip,
-        "Seems to be broken in CI/CD. need oto fix.",
+        "Seems to be broken in CI/CD. need to fix.",
     ),
     "/mysql to clickhouse replication/auto table creation/cli/*": (
         Skip,
-        "Seems to be broken in CI/CD. need oto fix.",
+        "Seems to be broken in CI/CD. need to fix.",
+    ),
+    "/mysql to clickhouse replication/auto table creation/parallel alters/multiple parallel add modify drop column": (
+        Skip,
+        "Test requires fixing.",
     ),
 }
 

sink-connector-lightweight/tests/integration/regression_auto_replicated.py

@@ -127,11 +127,15 @@
     ),
     "/mysql to clickhouse replication/auto replicated table creation/schema only/*": (
         Skip,
-        "Seems to be broken in CI/CD. need oto fix.",
+        "Seems to be broken in CI/CD. need to fix.",
     ),
     "/mysql to clickhouse replication/auto replicated table creation/cli/*": (
         Skip,
-        "Seems to be broken in CI/CD. need oto fix.",
+        "Seems to be broken in CI/CD. need to fix.",
+    ),
+    "/mysql to clickhouse replication/auto replicated table creation/parallel alters/multiple parallel add modify drop column": (
+        Skip,
+        "Test requires fixing.",
     ),
 }
 
@@ -303,6 +307,7 @@ def regression(
 
         join()
 
+    Feature(run=load("tests.databases", "module"))
     Feature(
         run=load("tests.schema_only", "module"),
     )

sink-connector-lightweight/tests/integration/requirements/requirements.md

@@ -190,6 +190,8 @@
 * 28 [Column Names](#column-names)
     * 28.1 [Replicate Tables With Special Column Names](#replicate-tables-with-special-column-names)
         * 28.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ColumnNames.Special](#rqsrs-030clickhousemysqltoclickhousereplicationcolumnnamesspecial)
+    * 28.2 [Replicate Tables With Backticks in Column Names](#replicate-tables-with-backticks-in-column-names)
+        * 28.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ColumnNames.Backticks](#rqsrs-030clickhousemysqltoclickhousereplicationcolumnnamesbackticks)
 * 29 [Replication Interruption](#replication-interruption)
     * 29.1 [Retry Replication When ClickHouse Instance Is Not Active](#retry-replication-when-clickhouse-instance-is-not-active)
         * 29.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Interruption.ClickHouse.Instance.Stopped](#rqsrs-030clickhousemysqltoclickhousereplicationinterruptionclickhouseinstancestopped)
@@ -273,17 +275,21 @@
             * 34.4.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.IncludeList](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesincludelist)
         * 34.4.2 [Replicate All Databases](#replicate-all-databases)
             * 34.4.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.ReplicateAll](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesreplicateall)
-    * 34.5 [Table Operations](#table-operations)
-        * 34.5.1 [Specify Database Name in Table Operations](#specify-database-name-in-table-operations)
-            * 34.5.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.SpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsspecifydatabasename)
-        * 34.5.2 [Table Operations Without Specifying Database Name](#table-operations-without-specifying-database-name)
-            * 34.5.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.NoSpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsnospecifydatabasename)
-    * 34.6 [Error Handling](#error-handling)
-        * 34.6.1 [When Replicated Database Does Not Exist on the Destination](#when-replicated-database-does-not-exist-on-the-destination)
-            * 34.6.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ErrorHandling.DatabaseNotExist](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabaseserrorhandlingdatabasenotexist)
-    * 34.7 [Concurrent Actions](#concurrent-actions)
-        * 34.7.1 [Perform Table Operations on Each Database Concurrently](#perform-table-operations-on-each-database-concurrently)
-            * 34.7.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConcurrentActions](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconcurrentactions)
+    * 34.5 [Overriding Source To Destination Database Name Mapping](#overriding-source-to-destination-database-name-mapping)
+        * 34.5.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.OverrideMap](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesoverridemap)
+        * 34.5.2 [Multiple Database Names](#multiple-database-names)
+            * 34.5.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.OverrideMap.MultipleValues](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconfigvaluesoverridemapmultiplevalues)
+    * 34.6 [Table Operations](#table-operations)
+        * 34.6.1 [Specify Database Name in Table Operations](#specify-database-name-in-table-operations)
+            * 34.6.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.SpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsspecifydatabasename)
+        * 34.6.2 [Table Operations Without Specifying Database Name](#table-operations-without-specifying-database-name)
+            * 34.6.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.TableOperations.NoSpecifyDatabaseName](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasestableoperationsnospecifydatabasename)
+    * 34.7 [Error Handling](#error-handling)
+        * 34.7.1 [When Replicated Database Does Not Exist on the Destination](#when-replicated-database-does-not-exist-on-the-destination)
+            * 34.7.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ErrorHandling.DatabaseNotExist](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabaseserrorhandlingdatabasenotexist)
+    * 34.8 [Concurrent Actions](#concurrent-actions)
+        * 34.8.1 [Perform Table Operations on Each Database Concurrently](#perform-table-operations-on-each-database-concurrently)
+            * 34.8.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConcurrentActions](#rqsrs-030clickhousemysqltoclickhousereplicationmultipledatabasesconcurrentactions)
 
 ## Introduction
 
@@ -1458,6 +1464,21 @@ CREATE TABLE new_table(col1 VARCHAR(255), col2 INT, is_deleted INT)
 
 The `ReplacingMergeTree` table created on ClickHouse side SHALL be updated and the `is_deleted` column should be renamed to  `_is_deleted` so there are no column name conflicts between ClickHouse and source table.
 
+### Replicate Tables With Backticks in Column Names
+
+#### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ColumnNames.Backticks
+version: 1.0
+
+[Altinity Sink Connector] SHALL support replication from the source tables that have backticks in column names.
+
+For example,
+
+If we create a source table that contains the column with the `is_deleted` name,
+
+```sql
+CREATE TABLE new_table(col1 VARCHAR(255), `col2` INT, `is_deleted` INT)
+```
+
 ## Replication Interruption
 
 ### Retry Replication When ClickHouse Instance Is Not Active
@@ -1796,10 +1817,12 @@ Multiple Databases:
       actions:
           - Perform table operations on each database sequentially
           - Perform table operations on all databases simultaneously
+          - Create database on source and map it to the database with different name on destination
           - Remove database
       configValues: 
-        - database.include.list: database1, database2, ... , databaseN
         - Don't specify database.include.list
+        - database.include.list: database1, database2, ... , databaseN
+        - clickhouse.database.override.map: "test:test2, products:products2"
       TableOperations:
         - types:
             - With database name
@@ -1937,6 +1960,53 @@ version: 1.0
 
 [Altinity Sink Connector] SHALL support the ability to monitor all databases from the source and replicate them to the destination without specifying the `database.include.list` configuration value.
 
+### Overriding Source To Destination Database Name Mapping
+
+#### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.OverrideMap
+version: 1.0
+
+[Altinity Sink Connector] SHALL support the usage of the `clickhouse.database.override.map` configuration value to allow the user to replicate the data from the source database to the destination database with a different name.
+
+For example, when using the following value in configuration,
+
+```yaml
+clickhouse.database.override.map: "mysql1:ch1"
+```
+
+The source database `mysql1` SHALL be mapped to the destination database `ch1`, and the data from the source `mysql1` SHALL only be replicated to the destination database `ch1`.
+
+```mermaid
+flowchart TD
+    B[Read clickhouse.database.override.map] --> D[Identify Source Database mysql1]
+    D --> E[Map to Destination Database ch1]
+    E --> F[Replicate Data to ch1]
+```
+
+#### Multiple Database Names
+
+##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDatabases.ConfigValues.OverrideMap.MultipleValues
+version: 1.0
+
+[Altinity Sink Connector] SHALL support the usage of the `clickhouse.database.override.map` configuration value to map multiple source databases to different databases on the destination.
+
+For example, when using the following value in configuration,
+
+```yaml
+clickhouse.database.override.map: "mysql1:ch1, mysql2:ch2"
+```
+
+The source databases `mysql1` and `mysql2` SHALL be mapped to the destination databases `ch1` and `ch2`, and the data from these source databases SHALL only be replicated to the destination databases `ch1` and `ch2`.
+
+```mermaid
+flowchart TD
+    B[Read clickhouse.database.override.map]
+    B --> C[Parse Override Map]
+    C --> E[Map mysql1 to ch1]
+    C --> F[Map mysql2 to ch2]
+    E --> G[Replicate Data from mysql1 to ch1]
+    F --> H[Replicate Data from mysql2 to ch2]
+```
+
 ### Table Operations
 
 #### Specify Database Name in Table Operations