From 1ccdb8eb3a71fea391a9c4ac07b09e5f9ee1a566 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Mon, 6 Jan 2025 18:47:18 +0530 Subject: [PATCH 01/31] Fix: MAP value converter for debezium --- .../datatypes/export_data_status-report.json | 5 +++ .../datatypes/import_data_status-report.json | 7 ++++ .../tests/pg/datatypes/pg_datatypes_data.sql | 13 +++++++ .../pg/datatypes/pg_datatypes_schema.sql | 6 +++- migtests/tests/pg/datatypes/validate | 3 +- .../tests/pg/datatypes/validateAfterChanges | 2 ++ .../src/tgtdb/suites/yugabytedbSuite.go | 34 +++++++++++++------ .../src/tgtdb/suites/yugabytedbSuite_test.go | 19 +++++++++++ 8 files changed, 77 insertions(+), 12 deletions(-) diff --git a/migtests/tests/pg/datatypes/export_data_status-report.json b/migtests/tests/pg/datatypes/export_data_status-report.json index beca513dfb..859d2956d4 100644 --- a/migtests/tests/pg/datatypes/export_data_status-report.json +++ b/migtests/tests/pg/datatypes/export_data_status-report.json @@ -33,5 +33,10 @@ "table_name": "null_and_default", "status": "DONE", "exported_count": 2 + }, + { + "exported_count": 7, + "status": "DONE", + "table_name": "hstore_example" } ] diff --git a/migtests/tests/pg/datatypes/import_data_status-report.json b/migtests/tests/pg/datatypes/import_data_status-report.json index c90c3670b7..b075ea90d5 100644 --- a/migtests/tests/pg/datatypes/import_data_status-report.json +++ b/migtests/tests/pg/datatypes/import_data_status-report.json @@ -34,6 +34,13 @@ "imported_count": 3, "percentage_complete": 100 }, + { + "table_name": "public.\"hstore_example\"", + "status": "DONE", + "total_count": 7, + "imported_count": 7, + "percentage_complete": 100 + }, { "table_name": "public.\"null_and_default\"", "status": "DONE", diff --git a/migtests/tests/pg/datatypes/pg_datatypes_data.sql b/migtests/tests/pg/datatypes/pg_datatypes_data.sql index d9b41cd6e1..097d3f1175 100644 --- a/migtests/tests/pg/datatypes/pg_datatypes_data.sql +++ b/migtests/tests/pg/datatypes/pg_datatypes_data.sql @@ -34,3 +34,16 @@ select * from datatypes2; insert into null_and_default (id) VALUES (1); insert into null_and_default VALUES(2, NULL, NULL, NULL); +INSERT INTO hstore_example (data) +VALUES + ('"key1"=>"value1", "key2"=>"value2"'), + (NULL), + (''), + ('key1 => value1, key2 => value2'), + (hstore(ARRAY['key1', 'key2'], ARRAY['value1', 'value2'])), + ('key7 => value7, key8 => 123, key9 => true'), + ('"paperback" => "243", + "publisher" => "postgresqltutorial.com", + "language" => "English", + "ISBN-13" => "978-1449370000", + "weight" => "11.2 ounces"'); \ No newline at end of file diff --git a/migtests/tests/pg/datatypes/pg_datatypes_schema.sql b/migtests/tests/pg/datatypes/pg_datatypes_schema.sql index ed26faa3ba..35ea49c61f 100644 --- a/migtests/tests/pg/datatypes/pg_datatypes_schema.sql +++ b/migtests/tests/pg/datatypes/pg_datatypes_schema.sql @@ -41,5 +41,9 @@ create table datatypes2(id serial primary key, v1 json, v2 BIT(10), v3 int ARRAY drop table if exists null_and_default; create table null_and_default(id int PRIMARY KEY, b boolean default false, i int default 10, val varchar default 'testdefault'); +create EXTENSION hstore; - +CREATE TABLE hstore_example ( + id SERIAL PRIMARY KEY, + data hstore +); diff --git a/migtests/tests/pg/datatypes/validate b/migtests/tests/pg/datatypes/validate index 5eba851246..988eb2989d 100755 --- a/migtests/tests/pg/datatypes/validate +++ b/migtests/tests/pg/datatypes/validate @@ -26,6 +26,7 @@ EXPECTED_ROW_COUNT = { 'datetime_type2': 2, 'null_and_default' :2, 'decimal_types': 3, + 'hstore_example': 7, } EXPECTED_SUM_OF_COLUMN = { @@ -73,7 +74,7 @@ def migration_completed_checks_fb(): def migration_completed_checks(tgt): table_list = tgt.get_table_names("public") print("table_list:", table_list) - assert len(table_list) == 7 + assert len(table_list) == 8 got_row_count = tgt.row_count_of_all_tables("public") for table_name, row_count in EXPECTED_ROW_COUNT.items(): diff --git a/migtests/tests/pg/datatypes/validateAfterChanges b/migtests/tests/pg/datatypes/validateAfterChanges index f5aa16c726..38fa259b8a 100755 --- a/migtests/tests/pg/datatypes/validateAfterChanges +++ b/migtests/tests/pg/datatypes/validateAfterChanges @@ -29,6 +29,7 @@ EXPECTED_ROW_COUNT = { 'datetime_type2': 2, 'null_and_default' :2, 'decimal_types': 4, + 'hstore_example': 7, } EXPECTED_SUM_OF_COLUMN = { @@ -65,6 +66,7 @@ EXPECTED_ROW_COUNT_FF = { 'datetime_type2': 3, 'null_and_default' :2, 'decimal_types': 4, + 'hstore_example': 7, } EXPECTED_SUM_OF_COLUMN_FF = { diff --git a/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go b/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go index 5ca6291479..fb5a344bdd 100644 --- a/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go +++ b/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go @@ -19,13 +19,13 @@ package tgtdbsuite import ( "encoding/base64" "encoding/binary" - "encoding/json" "fmt" "strconv" "strings" "time" "github.com/samber/lo" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/schemareg" ) @@ -173,17 +173,31 @@ var YBValueConverterSuite = map[string]ConverterFn{ } return string(hexValue), nil }, - "MAP": func(columnValue string, _ bool, _ *schemareg.ColumnSchema) (string, error) { - mapValue := make(map[string]interface{}) - err := json.Unmarshal([]byte(columnValue), &mapValue) - if err != nil { - return columnValue, fmt.Errorf("parsing map: %v", err) - } + "MAP": func(columnValue string, formatIfRequired bool, colDbzmSchema *schemareg.ColumnSchema) (string, error) { + //e.g. val - {key1=value1, key2=value2} + // Remove curly braces from the string + input := strings.Trim(columnValue, "{}") + + // Split the string by comma to get key-value pairs + pairs := strings.Split(input, ", ") + + // Create a map to store the parsed key-value pairs var transformedMapValue string - for key, value := range mapValue { - transformedMapValue = transformedMapValue + fmt.Sprintf("\"%s\"=>\"%s\",", key, value) + + // Iterate over each pair + for _, pair := range pairs { + // Split the pair by '=' to separate key and value + kv := strings.SplitN(pair, "=", 2) + if len(kv) == 2 { + key := kv[0] + value := kv[1] + transformedMapValue = transformedMapValue + fmt.Sprintf("\"%s\"=>\"%s\",", key, value) + } + } + if len(transformedMapValue) > 1 { + transformedMapValue = transformedMapValue[:len(transformedMapValue)-1]//remove last comma and } - return fmt.Sprintf("'%s'", transformedMapValue[:len(transformedMapValue)-1]), nil //remove last comma and add quotes + return quoteValueIfRequired(transformedMapValue, formatIfRequired, colDbzmSchema) // add quotes if required }, "STRING": quoteValueIfRequiredWithEscaping, } diff --git a/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go b/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go index 8f51d621b2..f3e4eb572f 100644 --- a/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go +++ b/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go @@ -93,3 +93,22 @@ func TestUUIDConversionWithFormatting(t *testing.T) { // Then assert.Equal(t, `'123e4567-e89b-12d3-a456-426614174000'`, result) } + +func TestMapConversion(t *testing.T) { + value := "{key1=value1, key2=value2}" + result, err := YBValueConverterSuite["MAP"](value, false, nil) + assert.NoError(t, err) + assert.Equal(t, `"key1"=>"value1","key2"=>"value2"`, result) + + result, err = YBValueConverterSuite["MAP"](value, true, nil) + assert.NoError(t, err) + assert.Equal(t, `'"key1"=>"value1","key2"=>"value2"'`, result) + + result, err = YBValueConverterSuite["MAP"]("", false, nil) + assert.NoError(t, err) + assert.Equal(t, "", result) + + result, err = YBValueConverterSuite["MAP"]("", true, nil) + assert.NoError(t, err) + assert.Equal(t, "''", result) +} \ No newline at end of file From 78ba6c555dd5b17649ea93b0f817c853411db913 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Mon, 6 Jan 2025 21:18:50 +0530 Subject: [PATCH 02/31] add cdc events for live --- migtests/tests/pg/datatypes/source_delta.sql | 9 +++++++++ migtests/tests/pg/datatypes/target_delta.sql | 10 ++++++++++ migtests/tests/pg/datatypes/validateAfterChanges | 4 ++-- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/migtests/tests/pg/datatypes/source_delta.sql b/migtests/tests/pg/datatypes/source_delta.sql index 8a54829cde..8ed7dcae5f 100644 --- a/migtests/tests/pg/datatypes/source_delta.sql +++ b/migtests/tests/pg/datatypes/source_delta.sql @@ -51,5 +51,14 @@ UPDATE datatypes2 SET v1 = '{"updated": true}', v2 = B'0101010101', v5 = B'101010101010101010101010101010', v3 = ARRAY[5, 6, 7, 8], v4 = '{{"e", "f"}, {"g", "h"}}' WHERE v1 IS NULL; +UPDATE hstore_example +SET data = data || 'key3 => value3' +WHERE id = 1; +UPDATE hstore_example +SET data = delete(data, 'key2') +WHERE id = 3; +INSERT INTO hstore_example (data) +VALUES + ('key5 => value5, key6 => value6'); diff --git a/migtests/tests/pg/datatypes/target_delta.sql b/migtests/tests/pg/datatypes/target_delta.sql index 5b277ff516..ab488e1fda 100644 --- a/migtests/tests/pg/datatypes/target_delta.sql +++ b/migtests/tests/pg/datatypes/target_delta.sql @@ -61,3 +61,13 @@ SET v1 = '{"new": "data"}', v2 = B'1111000011', v5=B'001010100101010101010101010 DELETE FROM datatypes2 WHERE 5 = ANY(v3); + +INSERT INTO hstore_example (data) +VALUES + ('key7 => value7, key8 => value8'); + +UPDATE hstore_example +SET data = delete(data, 'key2') +WHERE id = 8; + +DELETE FROM hstore_example WHERE data ? 'key5'; \ No newline at end of file diff --git a/migtests/tests/pg/datatypes/validateAfterChanges b/migtests/tests/pg/datatypes/validateAfterChanges index 38fa259b8a..3574f2e7cc 100755 --- a/migtests/tests/pg/datatypes/validateAfterChanges +++ b/migtests/tests/pg/datatypes/validateAfterChanges @@ -29,7 +29,7 @@ EXPECTED_ROW_COUNT = { 'datetime_type2': 2, 'null_and_default' :2, 'decimal_types': 4, - 'hstore_example': 7, + 'hstore_example': 8, } EXPECTED_SUM_OF_COLUMN = { @@ -66,7 +66,7 @@ EXPECTED_ROW_COUNT_FF = { 'datetime_type2': 3, 'null_and_default' :2, 'decimal_types': 4, - 'hstore_example': 7, + 'hstore_example': 8, } EXPECTED_SUM_OF_COLUMN_FF = { From 2f5b4112fb28f759b34d73ea52adbf0be4846c18 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Tue, 7 Jan 2025 17:27:26 +0530 Subject: [PATCH 03/31] move hstore to PostgreToYBConverter --- .../debezium/server/ybexporter/PostgresToYbValueConverter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/PostgresToYbValueConverter.java b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/PostgresToYbValueConverter.java index 8d1e79a6cc..185716a75d 100644 --- a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/PostgresToYbValueConverter.java +++ b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/PostgresToYbValueConverter.java @@ -39,6 +39,7 @@ public void converterFor(RelationalColumn column, switch (column.typeName()) { case "tsvector": case "tsquery": + case "hstore": registration.register(SchemaBuilder.string(), this::stringify); break; } From a952d7427a5446f523129b8763ba6ae6916038a0 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Tue, 7 Jan 2025 23:04:16 +0530 Subject: [PATCH 04/31] try --- .github/workflows/pg-13-migtests.yml | 20 +++++++++---------- .../PostgresToYbValueConverter.java | 2 ++ migtests/scripts/run-test.sh | 3 +++ 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pg-13-migtests.yml b/.github/workflows/pg-13-migtests.yml index d6b03ec165..08e2f828a4 100644 --- a/.github/workflows/pg-13-migtests.yml +++ b/.github/workflows/pg-13-migtests.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: version: [2024.2.0.0-b145, 2.20.8.0-b53, 2024.1.3.1-b8, 2.23.1.0-b220] - BETA_FAST_DATA_EXPORT: [0, 1] + BETA_FAST_DATA_EXPORT: [1] test_group: - offline - live_basic @@ -109,17 +109,17 @@ jobs: echo "127.0.0.1 yb-master-n1" | sudo tee -a /etc/hosts psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();" - - name: "TEST: pg-table-list-flags-test (table-list and exclude-table-list)" - if: ${{ !cancelled() && matrix.test_group == 'offline' }} - run: migtests/scripts/run-test.sh pg/table-list-flags-tests + # - name: "TEST: pg-table-list-flags-test (table-list and exclude-table-list)" + # if: ${{ !cancelled() && matrix.test_group == 'offline' }} + # run: migtests/scripts/run-test.sh pg/table-list-flags-tests - - name: "TEST: pg-table-list-file-path-test (table-list-file-path and exclude-table-list-file-path)" - if: ${{ !cancelled() && matrix.test_group == 'offline' }} - run: migtests/scripts/run-test.sh pg/table-list-flags-tests env-file-path-flags.sh + # - name: "TEST: pg-table-list-file-path-test (table-list-file-path and exclude-table-list-file-path)" + # if: ${{ !cancelled() && matrix.test_group == 'offline' }} + # run: migtests/scripts/run-test.sh pg/table-list-flags-tests env-file-path-flags.sh - - name: "TEST: pg-case-sensitivity-single-table" - if: ${{ !cancelled() && matrix.test_group == 'offline' }} - run: migtests/scripts/run-test-export-data.sh pg/case-sensitivity-single-table + # - name: "TEST: pg-case-sensitivity-single-table" + # if: ${{ !cancelled() && matrix.test_group == 'offline' }} + # run: migtests/scripts/run-test-export-data.sh pg/case-sensitivity-single-table - name: "TEST: pg-datatypes" if: ${{ !cancelled() && matrix.test_group == 'offline' }} diff --git a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/PostgresToYbValueConverter.java b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/PostgresToYbValueConverter.java index 185716a75d..566fc2e89a 100644 --- a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/PostgresToYbValueConverter.java +++ b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/PostgresToYbValueConverter.java @@ -36,6 +36,8 @@ public void converterFor(RelationalColumn column, break; } + LOGGER.info("column={}", column); + LOGGER.info("registration={}", registration); switch (column.typeName()) { case "tsvector": case "tsquery": diff --git a/migtests/scripts/run-test.sh b/migtests/scripts/run-test.sh index f0d9408cf8..b9941fe43f 100755 --- a/migtests/scripts/run-test.sh +++ b/migtests/scripts/run-test.sh @@ -120,6 +120,9 @@ main() { fi fi + cat ${EXPORT_DIR}/data/hstore_example_data.sql + + step "Fix data." if [ -x "${TEST_DIR}/fix-data" ] then From 7ed45824a5f0315bf70606d67925357a6ab218d3 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Tue, 7 Jan 2025 23:21:31 +0530 Subject: [PATCH 05/31] schema log --- migtests/scripts/run-test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/migtests/scripts/run-test.sh b/migtests/scripts/run-test.sh index b9941fe43f..1725eaab40 100755 --- a/migtests/scripts/run-test.sh +++ b/migtests/scripts/run-test.sh @@ -121,7 +121,8 @@ main() { fi cat ${EXPORT_DIR}/data/hstore_example_data.sql - + cat ${EXPORT_DIR}/data/schemas/source_db_exporter/hstore_example_schema.json + cat ${EXPORT_DIR}/logs/debezium-source_db_exporter.log step "Fix data." if [ -x "${TEST_DIR}/fix-data" ] From 1a565401f969c8e110a242a9521a2de2c44f864e Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Wed, 8 Jan 2025 11:15:14 +0530 Subject: [PATCH 06/31] default hstore mode --- yb-voyager/src/dbzm/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yb-voyager/src/dbzm/config.go b/yb-voyager/src/dbzm/config.go index b8b2909e6f..521947186a 100644 --- a/yb-voyager/src/dbzm/config.go +++ b/yb-voyager/src/dbzm/config.go @@ -27,6 +27,7 @@ import ( "github.com/google/uuid" log "github.com/sirupsen/logrus" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" ) @@ -122,7 +123,6 @@ debezium.source.connector.class=io.debezium.connector.postgresql.PostgresConnect debezium.source.database.dbname=%s debezium.source.schema.include.list=%s debezium.source.plugin.name=pgoutput -debezium.source.hstore.handling.mode=map debezium.source.converters=postgres_to_yb_converter debezium.source.postgres_to_yb_converter.type=io.debezium.server.ybexporter.PostgresToYbValueConverter debezium.source.provide.transaction.metadata=true From 16399c4a04efb970a12be944680aa94cacfb7710 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Wed, 8 Jan 2025 11:41:18 +0530 Subject: [PATCH 07/31] json hstore mode --- yb-voyager/src/dbzm/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/yb-voyager/src/dbzm/config.go b/yb-voyager/src/dbzm/config.go index 521947186a..aabc9598f9 100644 --- a/yb-voyager/src/dbzm/config.go +++ b/yb-voyager/src/dbzm/config.go @@ -123,6 +123,7 @@ debezium.source.connector.class=io.debezium.connector.postgresql.PostgresConnect debezium.source.database.dbname=%s debezium.source.schema.include.list=%s debezium.source.plugin.name=pgoutput +debezium.source.hstore.handling.mode=json debezium.source.converters=postgres_to_yb_converter debezium.source.postgres_to_yb_converter.type=io.debezium.server.ybexporter.PostgresToYbValueConverter debezium.source.provide.transaction.metadata=true From 891cddff4520cd18a53f525f35d895a1bce1cda2 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Wed, 8 Jan 2025 12:56:38 +0530 Subject: [PATCH 08/31] cleanup --- .github/workflows/pg-13-migtests.yml | 2 +- .../server/ybexporter/PostgresToYbValueConverter.java | 3 --- migtests/scripts/run-test.sh | 4 ---- yb-voyager/src/dbzm/config.go | 2 +- 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pg-13-migtests.yml b/.github/workflows/pg-13-migtests.yml index 08e2f828a4..9b2d428cdc 100644 --- a/.github/workflows/pg-13-migtests.yml +++ b/.github/workflows/pg-13-migtests.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: version: [2024.2.0.0-b145, 2.20.8.0-b53, 2024.1.3.1-b8, 2.23.1.0-b220] - BETA_FAST_DATA_EXPORT: [1] + BETA_FAST_DATA_EXPORT: [0, 1] test_group: - offline - live_basic diff --git a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/PostgresToYbValueConverter.java b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/PostgresToYbValueConverter.java index 566fc2e89a..8d1e79a6cc 100644 --- a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/PostgresToYbValueConverter.java +++ b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/PostgresToYbValueConverter.java @@ -36,12 +36,9 @@ public void converterFor(RelationalColumn column, break; } - LOGGER.info("column={}", column); - LOGGER.info("registration={}", registration); switch (column.typeName()) { case "tsvector": case "tsquery": - case "hstore": registration.register(SchemaBuilder.string(), this::stringify); break; } diff --git a/migtests/scripts/run-test.sh b/migtests/scripts/run-test.sh index 1725eaab40..b7edc5a623 100755 --- a/migtests/scripts/run-test.sh +++ b/migtests/scripts/run-test.sh @@ -119,10 +119,6 @@ main() { echo "Error: pg_dump version not found in the log file." >&2 fi fi - - cat ${EXPORT_DIR}/data/hstore_example_data.sql - cat ${EXPORT_DIR}/data/schemas/source_db_exporter/hstore_example_schema.json - cat ${EXPORT_DIR}/logs/debezium-source_db_exporter.log step "Fix data." if [ -x "${TEST_DIR}/fix-data" ] diff --git a/yb-voyager/src/dbzm/config.go b/yb-voyager/src/dbzm/config.go index aabc9598f9..48739fee5d 100644 --- a/yb-voyager/src/dbzm/config.go +++ b/yb-voyager/src/dbzm/config.go @@ -123,7 +123,7 @@ debezium.source.connector.class=io.debezium.connector.postgresql.PostgresConnect debezium.source.database.dbname=%s debezium.source.schema.include.list=%s debezium.source.plugin.name=pgoutput -debezium.source.hstore.handling.mode=json +debezium.source.hstore.handling.mode=map debezium.source.converters=postgres_to_yb_converter debezium.source.postgres_to_yb_converter.type=io.debezium.server.ybexporter.PostgresToYbValueConverter debezium.source.provide.transaction.metadata=true From d2433c35f38a6fc5549785fe4c94ed4a7ecf4e7e Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Wed, 8 Jan 2025 12:57:11 +0530 Subject: [PATCH 09/31] cleanup --- .github/workflows/pg-13-migtests.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pg-13-migtests.yml b/.github/workflows/pg-13-migtests.yml index 9b2d428cdc..d6b03ec165 100644 --- a/.github/workflows/pg-13-migtests.yml +++ b/.github/workflows/pg-13-migtests.yml @@ -109,17 +109,17 @@ jobs: echo "127.0.0.1 yb-master-n1" | sudo tee -a /etc/hosts psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();" - # - name: "TEST: pg-table-list-flags-test (table-list and exclude-table-list)" - # if: ${{ !cancelled() && matrix.test_group == 'offline' }} - # run: migtests/scripts/run-test.sh pg/table-list-flags-tests + - name: "TEST: pg-table-list-flags-test (table-list and exclude-table-list)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/table-list-flags-tests - # - name: "TEST: pg-table-list-file-path-test (table-list-file-path and exclude-table-list-file-path)" - # if: ${{ !cancelled() && matrix.test_group == 'offline' }} - # run: migtests/scripts/run-test.sh pg/table-list-flags-tests env-file-path-flags.sh + - name: "TEST: pg-table-list-file-path-test (table-list-file-path and exclude-table-list-file-path)" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test.sh pg/table-list-flags-tests env-file-path-flags.sh - # - name: "TEST: pg-case-sensitivity-single-table" - # if: ${{ !cancelled() && matrix.test_group == 'offline' }} - # run: migtests/scripts/run-test-export-data.sh pg/case-sensitivity-single-table + - name: "TEST: pg-case-sensitivity-single-table" + if: ${{ !cancelled() && matrix.test_group == 'offline' }} + run: migtests/scripts/run-test-export-data.sh pg/case-sensitivity-single-table - name: "TEST: pg-datatypes" if: ${{ !cancelled() && matrix.test_group == 'offline' }} From a72b633f1aa69eea34a678f10dc6a088e203e74d Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Wed, 8 Jan 2025 12:57:46 +0530 Subject: [PATCH 10/31] cleanup --- migtests/scripts/run-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migtests/scripts/run-test.sh b/migtests/scripts/run-test.sh index b7edc5a623..f0d9408cf8 100755 --- a/migtests/scripts/run-test.sh +++ b/migtests/scripts/run-test.sh @@ -119,7 +119,7 @@ main() { echo "Error: pg_dump version not found in the log file." >&2 fi fi - + step "Fix data." if [ -x "${TEST_DIR}/fix-data" ] then From 2849530695c6941c16e0de40f4116aa43b7f6544 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Wed, 8 Jan 2025 21:20:42 +0530 Subject: [PATCH 11/31] changed the hstore handling mode to json and handling it in io.debezium.data.Json converter to convert it properly for complex cases as well --- .../tests/pg/datatypes/pg_datatypes_data.sql | 6 +- migtests/tests/pg/datatypes/source_delta.sql | 6 +- migtests/tests/pg/datatypes/target_delta.sql | 6 +- migtests/tests/pg/datatypes/validate | 2 +- .../tests/pg/datatypes/validateAfterChanges | 4 +- yb-voyager/src/dbzm/config.go | 4 +- .../src/tgtdb/suites/yugabytedbSuite.go | 78 ++++++++++++------- .../src/tgtdb/suites/yugabytedbSuite_test.go | 30 +++++-- 8 files changed, 94 insertions(+), 42 deletions(-) diff --git a/migtests/tests/pg/datatypes/pg_datatypes_data.sql b/migtests/tests/pg/datatypes/pg_datatypes_data.sql index 097d3f1175..9c17dc1ffa 100644 --- a/migtests/tests/pg/datatypes/pg_datatypes_data.sql +++ b/migtests/tests/pg/datatypes/pg_datatypes_data.sql @@ -46,4 +46,8 @@ VALUES "publisher" => "postgresqltutorial.com", "language" => "English", "ISBN-13" => "978-1449370000", - "weight" => "11.2 ounces"'); \ No newline at end of file + "weight" => "11.2 ounces"'), + (hstore(ROW(1,'{\"key1=value1, key2=value2\"}'))), + (hstore('json_field', '{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}')), + (hstore('{"key1=value1, key2=value2"}', '{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}')), + (hstore('"{""key1"":""value1"",""key2"":""value2""}"', '{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}')); diff --git a/migtests/tests/pg/datatypes/source_delta.sql b/migtests/tests/pg/datatypes/source_delta.sql index 8ed7dcae5f..fdf99c796b 100644 --- a/migtests/tests/pg/datatypes/source_delta.sql +++ b/migtests/tests/pg/datatypes/source_delta.sql @@ -56,9 +56,13 @@ SET data = data || 'key3 => value3' WHERE id = 1; UPDATE hstore_example -SET data = delete(data, 'key2') +SET data = hstore('{"key1=value1, key2=value2"}', '{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}') WHERE id = 3; INSERT INTO hstore_example (data) VALUES ('key5 => value5, key6 => value6'); + +INSERT INTO hstore_example (data) +VALUES + (hstore('{"key1=value1, key2=value2"}', '{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}')); diff --git a/migtests/tests/pg/datatypes/target_delta.sql b/migtests/tests/pg/datatypes/target_delta.sql index ab488e1fda..f235f4567b 100644 --- a/migtests/tests/pg/datatypes/target_delta.sql +++ b/migtests/tests/pg/datatypes/target_delta.sql @@ -70,4 +70,8 @@ UPDATE hstore_example SET data = delete(data, 'key2') WHERE id = 8; -DELETE FROM hstore_example WHERE data ? 'key5'; \ No newline at end of file +DELETE FROM hstore_example WHERE data ? 'key5'; + +INSERT INTO hstore_example (data) +VALUES +(hstore('"{""key1"":""value1"",""key2"":""value2""}"', '{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}')); \ No newline at end of file diff --git a/migtests/tests/pg/datatypes/validate b/migtests/tests/pg/datatypes/validate index 988eb2989d..d597aa6c21 100755 --- a/migtests/tests/pg/datatypes/validate +++ b/migtests/tests/pg/datatypes/validate @@ -26,7 +26,7 @@ EXPECTED_ROW_COUNT = { 'datetime_type2': 2, 'null_and_default' :2, 'decimal_types': 3, - 'hstore_example': 7, + 'hstore_example': 11, } EXPECTED_SUM_OF_COLUMN = { diff --git a/migtests/tests/pg/datatypes/validateAfterChanges b/migtests/tests/pg/datatypes/validateAfterChanges index 3574f2e7cc..8deb780f1c 100755 --- a/migtests/tests/pg/datatypes/validateAfterChanges +++ b/migtests/tests/pg/datatypes/validateAfterChanges @@ -29,7 +29,7 @@ EXPECTED_ROW_COUNT = { 'datetime_type2': 2, 'null_and_default' :2, 'decimal_types': 4, - 'hstore_example': 8, + 'hstore_example': 13, } EXPECTED_SUM_OF_COLUMN = { @@ -66,7 +66,7 @@ EXPECTED_ROW_COUNT_FF = { 'datetime_type2': 3, 'null_and_default' :2, 'decimal_types': 4, - 'hstore_example': 8, + 'hstore_example': 13, } EXPECTED_SUM_OF_COLUMN_FF = { diff --git a/yb-voyager/src/dbzm/config.go b/yb-voyager/src/dbzm/config.go index 48739fee5d..776b4cb826 100644 --- a/yb-voyager/src/dbzm/config.go +++ b/yb-voyager/src/dbzm/config.go @@ -97,7 +97,7 @@ debezium.source.offset.flush.interval.ms=0 debezium.source.table.include.list=%s debezium.source.interval.handling.mode=string debezium.source.include.unknown.datatypes=true -debezium.source.datatype.propagate.source.type=.*BOX.*,.*LINE.*,.*LSEG.*,.*PATH.*,.*POLYGON.*,.*CIRCLE.*,.*DATE.*,.*INTERVAL.*,.*CHAR.*,.*TIMESTAMP.*,.*LONG.* +debezium.source.datatype.propagate.source.type=.*BOX.*,.*LINE.*,.*LSEG.*,.*PATH.*,.*POLYGON.*,.*CIRCLE.*,.*DATE.*,.*INTERVAL.*,.*CHAR.*,.*TIMESTAMP.*,.*LONG.*,.*HSTORE.* debezium.source.tombstones.on.delete=false debezium.source.topic.naming.strategy=io.debezium.server.ybexporter.DummyTopicNamingStrategy @@ -123,7 +123,7 @@ debezium.source.connector.class=io.debezium.connector.postgresql.PostgresConnect debezium.source.database.dbname=%s debezium.source.schema.include.list=%s debezium.source.plugin.name=pgoutput -debezium.source.hstore.handling.mode=map +debezium.source.hstore.handling.mode=json debezium.source.converters=postgres_to_yb_converter debezium.source.postgres_to_yb_converter.type=io.debezium.server.ybexporter.PostgresToYbValueConverter debezium.source.provide.transaction.metadata=true diff --git a/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go b/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go index fb5a344bdd..210860953c 100644 --- a/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go +++ b/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go @@ -19,7 +19,9 @@ package tgtdbsuite import ( "encoding/base64" "encoding/binary" + "encoding/json" "fmt" + "slices" "strconv" "strings" "time" @@ -50,8 +52,56 @@ func quoteValueIfRequiredWithEscaping(value string, formatIfRequired bool, _ *sc } } +func hstoreValueConverter(columnValue string, formatIfRequired bool, dbzmSchema *schemareg.ColumnSchema) (string, error) { + fmt.Println(columnValue) + //e.g. val - "{""key1"":""value1"",""key2"":""value2""}" and for empty string val - {} + columnValue = fmt.Sprintf(`%s`, columnValue) + + // for the cases where value "{""{\""key1=value1, key2=value2\""}"":""{\\\""key1=value1, key2={\\\""key1=value1, key2=value2\\\""}\\\""}""}" + // escaping the \ -> \\ to preserve the escaping while unmarshalling + columnValue = strings.Replace(columnValue, `\"`, `\\\"`, -1) + + // unescaping the cases which are already escaped {\\\\\" -> {\\\" + columnValue = strings.Replace(columnValue, `\\\\\"`, `\\\"`, -1) + + // Initialize a map to hold the parsed data + var result map[string]interface{} + + // Parse the JSON string into the map + err := json.Unmarshal([]byte(columnValue), &result) + if err != nil { + return "", fmt.Errorf("error converting the value to map: %v", err) + } + // Create a map to store the parsed key-value pairs + var transformedMapValue string + + //sorting the keys so that result always have keys in an order + keys := lo.Keys(result) + slices.Sort(keys) + + // Access key-value pairs to format the string as - "\"{\"\"key1\"\":\"\"value1\"\",\"\"key2\"\":\"\"value2\"\"}\""=>"{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}" + for _, key := range keys { + value := result[key] + transformedMapValue = transformedMapValue + fmt.Sprintf(`"%s"=>"%s",`, key, value) + } + + if len(transformedMapValue) > 1 { + transformedMapValue = transformedMapValue[:len(transformedMapValue)-1] //remove last comma and + } + return quoteValueIfRequired(transformedMapValue, formatIfRequired, dbzmSchema) // add quotes if required +} + var YBValueConverterSuite = map[string]ConverterFn{ - "io.debezium.data.Json": quoteValueIfRequiredWithEscaping, + "io.debezium.data.Json": func(columnValue string, formatIfRequired bool, dbzmSchema *schemareg.ColumnSchema) (string, error) { + if dbzmSchema != nil { + colType, ok := dbzmSchema.Parameters["__debezium.source.column.type"] + if !ok || colType != "HSTORE" { + return quoteValueIfRequiredWithEscaping(columnValue, formatIfRequired, dbzmSchema) + } + return hstoreValueConverter(columnValue, formatIfRequired, dbzmSchema) + } + return quoteValueIfRequiredWithEscaping(columnValue, formatIfRequired, dbzmSchema) + }, "io.debezium.data.Enum": quoteValueIfRequiredWithEscaping, "io.debezium.time.Interval": quoteValueIfRequired, "io.debezium.data.Uuid": quoteValueIfRequired, @@ -173,31 +223,5 @@ var YBValueConverterSuite = map[string]ConverterFn{ } return string(hexValue), nil }, - "MAP": func(columnValue string, formatIfRequired bool, colDbzmSchema *schemareg.ColumnSchema) (string, error) { - //e.g. val - {key1=value1, key2=value2} - // Remove curly braces from the string - input := strings.Trim(columnValue, "{}") - - // Split the string by comma to get key-value pairs - pairs := strings.Split(input, ", ") - - // Create a map to store the parsed key-value pairs - var transformedMapValue string - - // Iterate over each pair - for _, pair := range pairs { - // Split the pair by '=' to separate key and value - kv := strings.SplitN(pair, "=", 2) - if len(kv) == 2 { - key := kv[0] - value := kv[1] - transformedMapValue = transformedMapValue + fmt.Sprintf("\"%s\"=>\"%s\",", key, value) - } - } - if len(transformedMapValue) > 1 { - transformedMapValue = transformedMapValue[:len(transformedMapValue)-1]//remove last comma and - } - return quoteValueIfRequired(transformedMapValue, formatIfRequired, colDbzmSchema) // add quotes if required - }, "STRING": quoteValueIfRequiredWithEscaping, } diff --git a/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go b/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go index f3e4eb572f..4c657efe14 100644 --- a/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go +++ b/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go @@ -22,6 +22,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/schemareg" ) func TestStringConversionWithFormattingWithDoubleQuotes(t *testing.T) { @@ -94,21 +96,35 @@ func TestUUIDConversionWithFormatting(t *testing.T) { assert.Equal(t, `'123e4567-e89b-12d3-a456-426614174000'`, result) } -func TestMapConversion(t *testing.T) { - value := "{key1=value1, key2=value2}" - result, err := YBValueConverterSuite["MAP"](value, false, nil) +func TestHstoreValueConversion(t *testing.T) { + colSchema := &schemareg.ColumnSchema{ + Parameters: map[string]string{ + "__debezium.source.column.type": "HSTORE", + }, + } + value := `{"key1":"value1","key2":"value2"}` + result, err := YBValueConverterSuite["io.debezium.data.Json"](value, false, colSchema) assert.NoError(t, err) assert.Equal(t, `"key1"=>"value1","key2"=>"value2"`, result) - result, err = YBValueConverterSuite["MAP"](value, true, nil) + result, err = YBValueConverterSuite["io.debezium.data.Json"](value, true, colSchema) assert.NoError(t, err) assert.Equal(t, `'"key1"=>"value1","key2"=>"value2"'`, result) - result, err = YBValueConverterSuite["MAP"]("", false, nil) + result, err = YBValueConverterSuite["io.debezium.data.Json"]("{}", false, colSchema) assert.NoError(t, err) assert.Equal(t, "", result) - result, err = YBValueConverterSuite["MAP"]("", true, nil) + result, err = YBValueConverterSuite["io.debezium.data.Json"]("{}", true, colSchema) assert.NoError(t, err) assert.Equal(t, "''", result) -} \ No newline at end of file + + value = `{"\"{\"\"key1\"\":\"\"value1\"\",\"\"key2\"\":\"\"value2\"\"}\"":"{\\\"key1=value1, key2={\\\"key1=value1, key2=value2\\\"}\\\"}"}` + result, err = YBValueConverterSuite["io.debezium.data.Json"](value, false, colSchema) + assert.NoError(t, err) + assert.Equal(t, `"\"{\"\"key1\"\":\"\"value1\"\",\"\"key2\"\":\"\"value2\"\"}\""=>"{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}"`, result) + + result, err = YBValueConverterSuite["io.debezium.data.Json"](value, true, colSchema) + assert.NoError(t, err) + assert.Equal(t, `'"\"{\"\"key1\"\":\"\"value1\"\",\"\"key2\"\":\"\"value2\"\"}\""=>"{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}"'`, result) +} From 0a228642d60b89a935fc57dc922e31fd449b0ee2 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Wed, 8 Jan 2025 21:43:14 +0530 Subject: [PATCH 12/31] try java map --- .../ybexporter/DebeziumRecordTransformer.java | 20 +++++++++++++++++++ yb-voyager/src/dbzm/config.go | 2 +- .../src/tgtdb/suites/yugabytedbSuite.go | 4 +++- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java index 86ff13efbe..8af80bb2f8 100644 --- a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java +++ b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java @@ -16,6 +16,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Map; +import java.util.HashMap; /** * This class ensures of doing any transformation of the record received from debezium @@ -70,6 +71,25 @@ private String makeFieldValueSerializable(Object fieldValue, Field field){ case BYTES: case STRUCT: return toKafkaConnectJsonConverted(fieldValue, field); + case MAP: + StringBuilder mapString = new StringBuilder(); + for (Map.Entry entry : ((HashMap) fieldValue).entrySet()) { + String key = entry.getKey(); + String val = entry.getValue(); + mapString.append("\""); + mapString.append(key); + mapString.append("\""); + mapString.append(" => "); + mapString.append("\""); + mapString.append(val); + mapString.append("\""); + mapString.append(","); + } + if(mapString.length() == 0) { + return ""; + } + return mapString.toString().substring(0, mapString.length() - 1); + } return fieldValue.toString(); } diff --git a/yb-voyager/src/dbzm/config.go b/yb-voyager/src/dbzm/config.go index 776b4cb826..936934977c 100644 --- a/yb-voyager/src/dbzm/config.go +++ b/yb-voyager/src/dbzm/config.go @@ -123,7 +123,7 @@ debezium.source.connector.class=io.debezium.connector.postgresql.PostgresConnect debezium.source.database.dbname=%s debezium.source.schema.include.list=%s debezium.source.plugin.name=pgoutput -debezium.source.hstore.handling.mode=json +debezium.source.hstore.handling.mode=map debezium.source.converters=postgres_to_yb_converter debezium.source.postgres_to_yb_converter.type=io.debezium.server.ybexporter.PostgresToYbValueConverter debezium.source.provide.transaction.metadata=true diff --git a/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go b/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go index 210860953c..f971b26a6d 100644 --- a/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go +++ b/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go @@ -53,7 +53,6 @@ func quoteValueIfRequiredWithEscaping(value string, formatIfRequired bool, _ *sc } func hstoreValueConverter(columnValue string, formatIfRequired bool, dbzmSchema *schemareg.ColumnSchema) (string, error) { - fmt.Println(columnValue) //e.g. val - "{""key1"":""value1"",""key2"":""value2""}" and for empty string val - {} columnValue = fmt.Sprintf(`%s`, columnValue) @@ -223,5 +222,8 @@ var YBValueConverterSuite = map[string]ConverterFn{ } return string(hexValue), nil }, + "MAP": func(columnValue string, formatIfRequired bool, _ *schemareg.ColumnSchema) (string, error) { + return columnValue, nil //handled in exporter plugin + }, "STRING": quoteValueIfRequiredWithEscaping, } From 2c9e33b81587722f838e8cfdb51e261c4a3f2d70 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Wed, 8 Jan 2025 22:14:35 +0530 Subject: [PATCH 13/31] fix reports --- .../debezium/server/ybexporter/DebeziumRecordTransformer.java | 4 +++- migtests/scripts/run-test.sh | 4 ++++ migtests/tests/pg/datatypes/export_data_status-report.json | 2 +- migtests/tests/pg/datatypes/import_data_status-report.json | 4 ++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java index 8af80bb2f8..f83a58c759 100644 --- a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java +++ b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java @@ -17,6 +17,7 @@ import java.util.Collections; import java.util.Map; import java.util.HashMap; +import org.slf4j.Logger; /** * This class ensures of doing any transformation of the record received from debezium @@ -72,7 +73,7 @@ private String makeFieldValueSerializable(Object fieldValue, Field field){ case STRUCT: return toKafkaConnectJsonConverted(fieldValue, field); case MAP: - StringBuilder mapString = new StringBuilder(); + StringBuilder mapString = new StringBuilder(); for (Map.Entry entry : ((HashMap) fieldValue).entrySet()) { String key = entry.getKey(); String val = entry.getValue(); @@ -88,6 +89,7 @@ private String makeFieldValueSerializable(Object fieldValue, Field field){ if(mapString.length() == 0) { return ""; } + LOGGER.info("map value = {}", mapString); return mapString.toString().substring(0, mapString.length() - 1); } diff --git a/migtests/scripts/run-test.sh b/migtests/scripts/run-test.sh index f0d9408cf8..892b23048a 100755 --- a/migtests/scripts/run-test.sh +++ b/migtests/scripts/run-test.sh @@ -120,6 +120,10 @@ main() { fi fi + cat ${EXPORT_DIR}/data/hstore_example_data.sql + cat ${EXPORT_DIR}/data/schemas/source_db_exporter/hstore_example_schema.json + cat ${EXPORT_DIR}/logs/debezium-source_db_exporter.log + step "Fix data." if [ -x "${TEST_DIR}/fix-data" ] then diff --git a/migtests/tests/pg/datatypes/export_data_status-report.json b/migtests/tests/pg/datatypes/export_data_status-report.json index 859d2956d4..f8d5321c1b 100644 --- a/migtests/tests/pg/datatypes/export_data_status-report.json +++ b/migtests/tests/pg/datatypes/export_data_status-report.json @@ -35,7 +35,7 @@ "exported_count": 2 }, { - "exported_count": 7, + "exported_count": 11, "status": "DONE", "table_name": "hstore_example" } diff --git a/migtests/tests/pg/datatypes/import_data_status-report.json b/migtests/tests/pg/datatypes/import_data_status-report.json index b075ea90d5..dcfb034892 100644 --- a/migtests/tests/pg/datatypes/import_data_status-report.json +++ b/migtests/tests/pg/datatypes/import_data_status-report.json @@ -37,8 +37,8 @@ { "table_name": "public.\"hstore_example\"", "status": "DONE", - "total_count": 7, - "imported_count": 7, + "total_count": 11, + "imported_count": 11, "percentage_complete": 100 }, { From 73bffd000322801f1d6a7ea3605751120f4eae50 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Wed, 8 Jan 2025 22:19:11 +0530 Subject: [PATCH 14/31] add all datatype tests --- .github/workflows/pg-17-migtests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/pg-17-migtests.yml b/.github/workflows/pg-17-migtests.yml index fc2aba70b3..f55219f51d 100644 --- a/.github/workflows/pg-17-migtests.yml +++ b/.github/workflows/pg-17-migtests.yml @@ -226,6 +226,14 @@ jobs: # - name: "TEST: pg-basic-non-public-fall-back-test" # run: migtests/scripts/live-migration-fallb-run-test.sh pg/basic-non-public-live-test + - name: "TEST: pg-datatypes-live-test" + if: ${{ !cancelled() && matrix.test_group == 'live_basic' }} + run: migtests/scripts/live-migration-run-test.sh pg/datatypes + + - name: "TEST: pg-datatypes-fall-forward-test" + if: ${{ !cancelled() && matrix.test_group == 'live_basic' }} + run: migtests/scripts/live-migration-fallf-run-test.sh pg/datatypes + - name: "TEST: pg-datatypes-fall-back-test" if: ${{ !cancelled() && matrix.test_group == 'live_basic' }} run: migtests/scripts/live-migration-fallb-run-test.sh pg/datatypes From b01386427e540234237ccc65c0814d72feac5e9f Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Wed, 8 Jan 2025 23:00:44 +0530 Subject: [PATCH 15/31] fix compilation --- .../debezium/server/ybexporter/DebeziumRecordTransformer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java index f83a58c759..23410f0bdf 100644 --- a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java +++ b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java @@ -18,12 +18,14 @@ import java.util.Map; import java.util.HashMap; import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class ensures of doing any transformation of the record received from debezium * before actually writing that record. */ public class DebeziumRecordTransformer implements RecordTransformer { + private static final Logger LOGGER = LoggerFactory.getLogger(DebeziumRecordTransformer.class); private JsonConverter jsonConverter; public DebeziumRecordTransformer(){ From 69c393f63684982c7a9d65baf547373e826a76ce Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Thu, 9 Jan 2025 00:07:59 +0530 Subject: [PATCH 16/31] fix run test to cat only in specific case --- migtests/scripts/run-test.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/migtests/scripts/run-test.sh b/migtests/scripts/run-test.sh index 892b23048a..f41df17302 100755 --- a/migtests/scripts/run-test.sh +++ b/migtests/scripts/run-test.sh @@ -120,9 +120,13 @@ main() { fi fi - cat ${EXPORT_DIR}/data/hstore_example_data.sql - cat ${EXPORT_DIR}/data/schemas/source_db_exporter/hstore_example_schema.json - cat ${EXPORT_DIR}/logs/debezium-source_db_exporter.log + if [ "${TEST_DIR}" = "${TESTS_DIR}/pg/datatypes" ]; then + cat ${EXPORT_DIR}/data/hstore_example_data.sql + if [ "${BETA_FAST_DATA_EXPORT}" = "1" ]; then + cat ${EXPORT_DIR}/data/schemas/source_db_exporter/hstore_example_schema.json + cat ${EXPORT_DIR}/logs/debezium-source_db_exporter.log + fi + fi step "Fix data." if [ -x "${TEST_DIR}/fix-data" ] From dd4e212f11795c39cf66e6f24b78a62e81ea29ce Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Thu, 9 Jan 2025 00:49:59 +0530 Subject: [PATCH 17/31] fix --- yb-voyager/src/tgtdb/suites/yugabytedbSuite.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go b/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go index f971b26a6d..c40b38ef11 100644 --- a/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go +++ b/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go @@ -222,8 +222,9 @@ var YBValueConverterSuite = map[string]ConverterFn{ } return string(hexValue), nil }, - "MAP": func(columnValue string, formatIfRequired bool, _ *schemareg.ColumnSchema) (string, error) { - return columnValue, nil //handled in exporter plugin + "MAP": func(columnValue string, formatIfRequired bool, dbzmSchema *schemareg.ColumnSchema) (string, error) { + fmt.Println(columnValue) + return quoteValueIfRequiredWithEscaping(columnValue, formatIfRequired, dbzmSchema) //handled in exporter plugin }, "STRING": quoteValueIfRequiredWithEscaping, } From b894c51566eb4cb4a06470571ac296667351bf1b Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Thu, 9 Jan 2025 14:51:26 +0000 Subject: [PATCH 18/31] Fix java hashmap approach to escape " -> \" --- .../debezium/server/ybexporter/DebeziumRecordTransformer.java | 4 ++++ migtests/tests/pg/datatypes/pg_datatypes_data.sql | 4 +++- migtests/tests/pg/datatypes/validate | 2 +- migtests/tests/pg/datatypes/validateAfterChanges | 4 ++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java index 23410f0bdf..fb88f4d061 100644 --- a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java +++ b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java @@ -79,6 +79,10 @@ private String makeFieldValueSerializable(Object fieldValue, Field field){ for (Map.Entry entry : ((HashMap) fieldValue).entrySet()) { String key = entry.getKey(); String val = entry.getValue(); + key = key.replace("\"", "\\\""); // escaping double quotes " -> \" ( "\"a"b\"" -> "\\"a\"b\\"" ) + val = val.replace("\"", "\\\""); + key = key.replace("\\\\", "\\"); // fixing \\-> \ ( "\\"a\"b\\"" -> "\"a\\"b\"" ) + val = val.replace("\\\\", "\\"); mapString.append("\""); mapString.append(key); mapString.append("\""); diff --git a/migtests/tests/pg/datatypes/pg_datatypes_data.sql b/migtests/tests/pg/datatypes/pg_datatypes_data.sql index 9c17dc1ffa..dd06c6b872 100644 --- a/migtests/tests/pg/datatypes/pg_datatypes_data.sql +++ b/migtests/tests/pg/datatypes/pg_datatypes_data.sql @@ -37,6 +37,7 @@ insert into null_and_default VALUES(2, NULL, NULL, NULL); INSERT INTO hstore_example (data) VALUES ('"key1"=>"value1", "key2"=>"value2"'), + (hstore('a"b', 'd"a')), (NULL), (''), ('key1 => value1, key2 => value2'), @@ -50,4 +51,5 @@ VALUES (hstore(ROW(1,'{\"key1=value1, key2=value2\"}'))), (hstore('json_field', '{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}')), (hstore('{"key1=value1, key2=value2"}', '{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}')), - (hstore('"{""key1"":""value1"",""key2"":""value2""}"', '{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}')); + (hstore('"{""key1"":""value1"",""key2"":""value2""}"', '{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}')), + (hstore('"{key1:value1,key2:value2}"', '{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}')); diff --git a/migtests/tests/pg/datatypes/validate b/migtests/tests/pg/datatypes/validate index d597aa6c21..ff35262cda 100755 --- a/migtests/tests/pg/datatypes/validate +++ b/migtests/tests/pg/datatypes/validate @@ -26,7 +26,7 @@ EXPECTED_ROW_COUNT = { 'datetime_type2': 2, 'null_and_default' :2, 'decimal_types': 3, - 'hstore_example': 11, + 'hstore_example': 13, } EXPECTED_SUM_OF_COLUMN = { diff --git a/migtests/tests/pg/datatypes/validateAfterChanges b/migtests/tests/pg/datatypes/validateAfterChanges index 8deb780f1c..cd8b55c541 100755 --- a/migtests/tests/pg/datatypes/validateAfterChanges +++ b/migtests/tests/pg/datatypes/validateAfterChanges @@ -29,7 +29,7 @@ EXPECTED_ROW_COUNT = { 'datetime_type2': 2, 'null_and_default' :2, 'decimal_types': 4, - 'hstore_example': 13, + 'hstore_example': 15, } EXPECTED_SUM_OF_COLUMN = { @@ -66,7 +66,7 @@ EXPECTED_ROW_COUNT_FF = { 'datetime_type2': 3, 'null_and_default' :2, 'decimal_types': 4, - 'hstore_example': 13, + 'hstore_example': 15, } EXPECTED_SUM_OF_COLUMN_FF = { From c996e46198936ddfbf777795795d692a0b76deed Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Thu, 9 Jan 2025 20:58:11 +0530 Subject: [PATCH 19/31] fix --- migtests/tests/pg/datatypes/export_data_status-report.json | 2 +- migtests/tests/pg/datatypes/import_data_status-report.json | 4 ++-- migtests/tests/pg/datatypes/validateAfterChanges | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/migtests/tests/pg/datatypes/export_data_status-report.json b/migtests/tests/pg/datatypes/export_data_status-report.json index f8d5321c1b..dc91417c35 100644 --- a/migtests/tests/pg/datatypes/export_data_status-report.json +++ b/migtests/tests/pg/datatypes/export_data_status-report.json @@ -35,7 +35,7 @@ "exported_count": 2 }, { - "exported_count": 11, + "exported_count": 13, "status": "DONE", "table_name": "hstore_example" } diff --git a/migtests/tests/pg/datatypes/import_data_status-report.json b/migtests/tests/pg/datatypes/import_data_status-report.json index dcfb034892..d7453e7876 100644 --- a/migtests/tests/pg/datatypes/import_data_status-report.json +++ b/migtests/tests/pg/datatypes/import_data_status-report.json @@ -37,8 +37,8 @@ { "table_name": "public.\"hstore_example\"", "status": "DONE", - "total_count": 11, - "imported_count": 11, + "total_count": 13, + "imported_count": 13, "percentage_complete": 100 }, { diff --git a/migtests/tests/pg/datatypes/validateAfterChanges b/migtests/tests/pg/datatypes/validateAfterChanges index cd8b55c541..a05073ec09 100755 --- a/migtests/tests/pg/datatypes/validateAfterChanges +++ b/migtests/tests/pg/datatypes/validateAfterChanges @@ -29,7 +29,7 @@ EXPECTED_ROW_COUNT = { 'datetime_type2': 2, 'null_and_default' :2, 'decimal_types': 4, - 'hstore_example': 15, + 'hstore_example': 16, } EXPECTED_SUM_OF_COLUMN = { @@ -66,7 +66,7 @@ EXPECTED_ROW_COUNT_FF = { 'datetime_type2': 3, 'null_and_default' :2, 'decimal_types': 4, - 'hstore_example': 15, + 'hstore_example': 16, } EXPECTED_SUM_OF_COLUMN_FF = { From ffbe2910cfac9254897787da11eb44653b02ca4f Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Fri, 10 Jan 2025 21:34:09 +0530 Subject: [PATCH 20/31] change the inserts to be proper --- ...migration-report-live-migration-fallf.json | 39 +++++++++++++++++++ .../tests/pg/datatypes/pg_datatypes_data.sql | 10 ++--- migtests/tests/pg/datatypes/source_delta.sql | 8 +++- migtests/tests/pg/datatypes/target_delta.sql | 28 ++++++++----- .../tests/pg/datatypes/validateAfterChanges | 4 +- 5 files changed, 70 insertions(+), 19 deletions(-) diff --git a/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json b/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json index 5eee6a3d6a..9e367e3aa9 100644 --- a/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json +++ b/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json @@ -271,5 +271,44 @@ "exported_updates": 0, "exported_deletes": 0, "final_row_count": 2 + }, + { + "table_name": "public.\"null_and_default\"", + "db_type": "source", + "exported_snapshot_rows": 13, + "imported_snapshot_rows": 0, + "imported_inserts": 0, + "imported_updates": 0, + "imported_deletes": 0, + "exported_inserts": 2, + "exported_updates": 3, + "exported_deletes": 0, + "final_row_count": 15 + }, + { + "table_name": "public.\"null_and_default\"", + "db_type": "source-replica", + "exported_snapshot_rows": 0, + "imported_snapshot_rows": 13, + "imported_inserts": 2, + "imported_updates": 3, + "imported_deletes": 0, + "exported_inserts": 0, + "exported_updates": 0, + "exported_deletes": 0, + "final_row_count": 15 + }, + { + "table_name": "public.\"null_and_default\"", + "db_type": "target", + "exported_snapshot_rows": 13, + "imported_snapshot_rows": 0, + "imported_inserts": 2, + "imported_updates": 3, + "imported_deletes": 0, + "exported_inserts": 0, + "exported_updates": 0, + "exported_deletes": 0, + "final_row_count": 15 } ] \ No newline at end of file diff --git a/migtests/tests/pg/datatypes/pg_datatypes_data.sql b/migtests/tests/pg/datatypes/pg_datatypes_data.sql index dd06c6b872..b569357d91 100644 --- a/migtests/tests/pg/datatypes/pg_datatypes_data.sql +++ b/migtests/tests/pg/datatypes/pg_datatypes_data.sql @@ -48,8 +48,8 @@ VALUES "language" => "English", "ISBN-13" => "978-1449370000", "weight" => "11.2 ounces"'), - (hstore(ROW(1,'{\"key1=value1, key2=value2\"}'))), - (hstore('json_field', '{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}')), - (hstore('{"key1=value1, key2=value2"}', '{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}')), - (hstore('"{""key1"":""value1"",""key2"":""value2""}"', '{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}')), - (hstore('"{key1:value1,key2:value2}"', '{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}')); + (hstore(ROW(1,'{"key1=value1, key2=value2"}'))), + (hstore('json_field', '{"key1=value1, key2={"key1=value1, key2=value2"}"}')), --hstore() key and values need no extra processing + ('"{\"key1=value1, key2=value2\"}"=>"{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}"'), --single quotes string need to escaped properly + (hstore('"{""key1"":""value1"",""key2"":""value2""}"', '{"key1=value1, key2={"key1=value1, key2=value2"}"}')), + (hstore('"{key1:value1,key2:value2}"', '{"key1=value1, key2={"key1=value1, key2=value2"}"}')); diff --git a/migtests/tests/pg/datatypes/source_delta.sql b/migtests/tests/pg/datatypes/source_delta.sql index fdf99c796b..a98a32d92b 100644 --- a/migtests/tests/pg/datatypes/source_delta.sql +++ b/migtests/tests/pg/datatypes/source_delta.sql @@ -56,13 +56,17 @@ SET data = data || 'key3 => value3' WHERE id = 1; UPDATE hstore_example -SET data = hstore('{"key1=value1, key2=value2"}', '{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}') +SET data = hstore('{"key1=value1, key2=value2"}', '{"key1=value1, key2={"key1=value1, key2=value2"}"}') WHERE id = 3; +UPDATE hstore_example +SET data = '"{\"key1=value1, key2=value2\"}"=>"{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}"' +WHERE id = 7; + INSERT INTO hstore_example (data) VALUES ('key5 => value5, key6 => value6'); INSERT INTO hstore_example (data) VALUES - (hstore('{"key1=value1, key2=value2"}', '{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}')); + (hstore('{"key1=value1, key2=value2"}', '{"key1=value1, key2={"key1=value1, key2=value2"}"}')); diff --git a/migtests/tests/pg/datatypes/target_delta.sql b/migtests/tests/pg/datatypes/target_delta.sql index f235f4567b..7eed5ed9be 100644 --- a/migtests/tests/pg/datatypes/target_delta.sql +++ b/migtests/tests/pg/datatypes/target_delta.sql @@ -62,16 +62,24 @@ SET v1 = '{"new": "data"}', v2 = B'1111000011', v5=B'001010100101010101010101010 DELETE FROM datatypes2 WHERE 5 = ANY(v3); -INSERT INTO hstore_example (data) -VALUES - ('key7 => value7, key8 => value8'); +-- INSERT INTO hstore_example (data) +-- VALUES +-- ('key7 => value7, key8 => value8'); -UPDATE hstore_example -SET data = delete(data, 'key2') -WHERE id = 8; +-- UPDATE hstore_example +-- SET data = delete(data, 'key2') +-- WHERE id = 8; -DELETE FROM hstore_example WHERE data ? 'key5'; +-- DELETE FROM hstore_example WHERE data ? 'key5'; -INSERT INTO hstore_example (data) -VALUES -(hstore('"{""key1"":""value1"",""key2"":""value2""}"', '{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}')); \ No newline at end of file +-- INSERT INTO hstore_example (data) +-- VALUES +-- (hstore('"{""key1"":""value1"",""key2"":""value2""}"', '{"key1=value1, key2={"key1=value1, key2=value2"}"}')); + +-- UPDATE hstore_example +-- SET data = hstore('{"key1=value1, key2=value2"}', '{"key1=value1, key2={"key1=value1, key2=value2"}"}') +-- WHERE id = 15; + +-- UPDATE hstore_example +-- SET data = '"{\"key1=value1, key2=value2\"}"=>"{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}"' +-- WHERE id = 14; diff --git a/migtests/tests/pg/datatypes/validateAfterChanges b/migtests/tests/pg/datatypes/validateAfterChanges index a05073ec09..cd8b55c541 100755 --- a/migtests/tests/pg/datatypes/validateAfterChanges +++ b/migtests/tests/pg/datatypes/validateAfterChanges @@ -29,7 +29,7 @@ EXPECTED_ROW_COUNT = { 'datetime_type2': 2, 'null_and_default' :2, 'decimal_types': 4, - 'hstore_example': 16, + 'hstore_example': 15, } EXPECTED_SUM_OF_COLUMN = { @@ -66,7 +66,7 @@ EXPECTED_ROW_COUNT_FF = { 'datetime_type2': 3, 'null_and_default' :2, 'decimal_types': 4, - 'hstore_example': 16, + 'hstore_example': 15, } EXPECTED_SUM_OF_COLUMN_FF = { From 24d9530bb8b9dcf0107ad3a9ebc9042f32fc35f5 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Sun, 12 Jan 2025 18:15:48 +0530 Subject: [PATCH 21/31] fix expected file --- .../data-migration-report-live-migration-fallf.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json b/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json index 9e367e3aa9..58075e8eac 100644 --- a/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json +++ b/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json @@ -273,7 +273,7 @@ "final_row_count": 2 }, { - "table_name": "public.\"null_and_default\"", + "table_name": "public.\"hstore_example\"", "db_type": "source", "exported_snapshot_rows": 13, "imported_snapshot_rows": 0, @@ -286,10 +286,10 @@ "final_row_count": 15 }, { - "table_name": "public.\"null_and_default\"", - "db_type": "source-replica", - "exported_snapshot_rows": 0, - "imported_snapshot_rows": 13, + "table_name": "public.\"hstore_example\"", + "db_type": "target", + "exported_snapshot_rows": 13, + "imported_snapshot_rows": 0, "imported_inserts": 2, "imported_updates": 3, "imported_deletes": 0, @@ -299,7 +299,7 @@ "final_row_count": 15 }, { - "table_name": "public.\"null_and_default\"", + "table_name": "public.\"hstore_example\"", "db_type": "target", "exported_snapshot_rows": 13, "imported_snapshot_rows": 0, From 099f1c1ebc0f0df81f3adace086ac9bfdd9620d7 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Wed, 15 Jan 2025 20:09:29 +0530 Subject: [PATCH 22/31] review comments --- .../ybexporter/DebeziumRecordTransformer.java | 20 ++++--- ...migration-report-live-migration-fallf.json | 18 +++---- .../tests/pg/datatypes/pg_datatypes_data.sql | 2 +- migtests/tests/pg/datatypes/source_delta.sql | 12 +++++ migtests/tests/pg/datatypes/target_delta.sql | 1 + .../tests/pg/datatypes/validateAfterChanges | 4 +- yb-voyager/src/dbzm/config.go | 2 +- .../src/tgtdb/suites/yugabytedbSuite.go | 52 +------------------ 8 files changed, 40 insertions(+), 71 deletions(-) diff --git a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java index fb88f4d061..dee077b2cb 100644 --- a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java +++ b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java @@ -75,14 +75,21 @@ private String makeFieldValueSerializable(Object fieldValue, Field field){ case STRUCT: return toKafkaConnectJsonConverted(fieldValue, field); case MAP: - StringBuilder mapString = new StringBuilder(); + StringBuilder mapString = new StringBuilder(); for (Map.Entry entry : ((HashMap) fieldValue).entrySet()) { String key = entry.getKey(); String val = entry.getValue(); - key = key.replace("\"", "\\\""); // escaping double quotes " -> \" ( "\"a"b\"" -> "\\"a\"b\\"" ) - val = val.replace("\"", "\\\""); - key = key.replace("\\\\", "\\"); // fixing \\-> \ ( "\\"a\"b\\"" -> "\"a\\"b\"" ) - val = val.replace("\\\\", "\\"); + /* + Escaping the key and value here for the double quote (")" and backslash char (\) with a backslash character as mentioned here + https://www.postgresql.org/docs/9/hstore.html#:~:text=To%20include%20a%20double%20quote%20or%20a%20backslash%20in%20a%20key%20or%20value%2C%20escape%20it%20with%20a%20backslash. + + Following the order of escaping the backslash first and then the double quote becasue first escape the backslashes in the string and adding the backslash for escaping to handle case like + e.g. key - "a\"b" -> (first escaping) -> "a\\"b" -> (second escaping) -> "a\\\"b" + */ + key = key.replace("\\", "\\\\"); // escaping backslash \ -> \\ ( "a\b" -> "a\\b" ) " + val = val.replace("\\", "\\\\"); + key = key.replace("\"", "\\\""); // escaping double quotes " -> \" ( "a"b" -> "a\"b" ) " + val = val.replace("\"", "\\\""); mapString.append("\""); mapString.append(key); mapString.append("\""); @@ -92,10 +99,9 @@ private String makeFieldValueSerializable(Object fieldValue, Field field){ mapString.append("\""); mapString.append(","); } - if(mapString.length() == 0) { + if(mapString.length() == 0) { return ""; } - LOGGER.info("map value = {}", mapString); return mapString.toString().substring(0, mapString.length() - 1); } diff --git a/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json b/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json index 58075e8eac..29a90e2156 100644 --- a/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json +++ b/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json @@ -280,35 +280,35 @@ "imported_inserts": 0, "imported_updates": 0, "imported_deletes": 0, - "exported_inserts": 2, - "exported_updates": 3, + "exported_inserts": 3, + "exported_updates": 5, "exported_deletes": 0, - "final_row_count": 15 + "final_row_count": 16 }, { "table_name": "public.\"hstore_example\"", "db_type": "target", "exported_snapshot_rows": 13, "imported_snapshot_rows": 0, - "imported_inserts": 2, - "imported_updates": 3, + "imported_inserts": 3, + "imported_updates": 5, "imported_deletes": 0, "exported_inserts": 0, "exported_updates": 0, "exported_deletes": 0, - "final_row_count": 15 + "final_row_count": 16 }, { "table_name": "public.\"hstore_example\"", "db_type": "target", "exported_snapshot_rows": 13, "imported_snapshot_rows": 0, - "imported_inserts": 2, - "imported_updates": 3, + "imported_inserts": 3, + "imported_updates": 5, "imported_deletes": 0, "exported_inserts": 0, "exported_updates": 0, "exported_deletes": 0, - "final_row_count": 15 + "final_row_count": 16 } ] \ No newline at end of file diff --git a/migtests/tests/pg/datatypes/pg_datatypes_data.sql b/migtests/tests/pg/datatypes/pg_datatypes_data.sql index b569357d91..230b9aae68 100644 --- a/migtests/tests/pg/datatypes/pg_datatypes_data.sql +++ b/migtests/tests/pg/datatypes/pg_datatypes_data.sql @@ -37,7 +37,7 @@ insert into null_and_default VALUES(2, NULL, NULL, NULL); INSERT INTO hstore_example (data) VALUES ('"key1"=>"value1", "key2"=>"value2"'), - (hstore('a"b', 'd"a')), + (hstore('a"b', 'd\"a')), (NULL), (''), ('key1 => value1, key2 => value2'), diff --git a/migtests/tests/pg/datatypes/source_delta.sql b/migtests/tests/pg/datatypes/source_delta.sql index a98a32d92b..eefaa46681 100644 --- a/migtests/tests/pg/datatypes/source_delta.sql +++ b/migtests/tests/pg/datatypes/source_delta.sql @@ -70,3 +70,15 @@ VALUES INSERT INTO hstore_example (data) VALUES (hstore('{"key1=value1, key2=value2"}', '{"key1=value1, key2={"key1=value1, key2=value2"}"}')); + +INSERT INTO hstore_example (data) +VALUES + (''); + +UPDATE hstore_example +SET data = NULL +WHERE id = 5; + +UPDATE hstore_example +SET data = '' +WHERE id = 6; \ No newline at end of file diff --git a/migtests/tests/pg/datatypes/target_delta.sql b/migtests/tests/pg/datatypes/target_delta.sql index 7eed5ed9be..d5502ae47d 100644 --- a/migtests/tests/pg/datatypes/target_delta.sql +++ b/migtests/tests/pg/datatypes/target_delta.sql @@ -62,6 +62,7 @@ SET v1 = '{"new": "data"}', v2 = B'1111000011', v5=B'001010100101010101010101010 DELETE FROM datatypes2 WHERE 5 = ANY(v3); +-- NOT WORKING WIT H YB CDC GRPC connector as of now -- INSERT INTO hstore_example (data) -- VALUES -- ('key7 => value7, key8 => value8'); diff --git a/migtests/tests/pg/datatypes/validateAfterChanges b/migtests/tests/pg/datatypes/validateAfterChanges index cd8b55c541..a05073ec09 100755 --- a/migtests/tests/pg/datatypes/validateAfterChanges +++ b/migtests/tests/pg/datatypes/validateAfterChanges @@ -29,7 +29,7 @@ EXPECTED_ROW_COUNT = { 'datetime_type2': 2, 'null_and_default' :2, 'decimal_types': 4, - 'hstore_example': 15, + 'hstore_example': 16, } EXPECTED_SUM_OF_COLUMN = { @@ -66,7 +66,7 @@ EXPECTED_ROW_COUNT_FF = { 'datetime_type2': 3, 'null_and_default' :2, 'decimal_types': 4, - 'hstore_example': 15, + 'hstore_example': 16, } EXPECTED_SUM_OF_COLUMN_FF = { diff --git a/yb-voyager/src/dbzm/config.go b/yb-voyager/src/dbzm/config.go index 936934977c..48739fee5d 100644 --- a/yb-voyager/src/dbzm/config.go +++ b/yb-voyager/src/dbzm/config.go @@ -97,7 +97,7 @@ debezium.source.offset.flush.interval.ms=0 debezium.source.table.include.list=%s debezium.source.interval.handling.mode=string debezium.source.include.unknown.datatypes=true -debezium.source.datatype.propagate.source.type=.*BOX.*,.*LINE.*,.*LSEG.*,.*PATH.*,.*POLYGON.*,.*CIRCLE.*,.*DATE.*,.*INTERVAL.*,.*CHAR.*,.*TIMESTAMP.*,.*LONG.*,.*HSTORE.* +debezium.source.datatype.propagate.source.type=.*BOX.*,.*LINE.*,.*LSEG.*,.*PATH.*,.*POLYGON.*,.*CIRCLE.*,.*DATE.*,.*INTERVAL.*,.*CHAR.*,.*TIMESTAMP.*,.*LONG.* debezium.source.tombstones.on.delete=false debezium.source.topic.naming.strategy=io.debezium.server.ybexporter.DummyTopicNamingStrategy diff --git a/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go b/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go index c40b38ef11..e71ee948e8 100644 --- a/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go +++ b/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go @@ -19,9 +19,7 @@ package tgtdbsuite import ( "encoding/base64" "encoding/binary" - "encoding/json" "fmt" - "slices" "strconv" "strings" "time" @@ -52,55 +50,8 @@ func quoteValueIfRequiredWithEscaping(value string, formatIfRequired bool, _ *sc } } -func hstoreValueConverter(columnValue string, formatIfRequired bool, dbzmSchema *schemareg.ColumnSchema) (string, error) { - //e.g. val - "{""key1"":""value1"",""key2"":""value2""}" and for empty string val - {} - columnValue = fmt.Sprintf(`%s`, columnValue) - - // for the cases where value "{""{\""key1=value1, key2=value2\""}"":""{\\\""key1=value1, key2={\\\""key1=value1, key2=value2\\\""}\\\""}""}" - // escaping the \ -> \\ to preserve the escaping while unmarshalling - columnValue = strings.Replace(columnValue, `\"`, `\\\"`, -1) - - // unescaping the cases which are already escaped {\\\\\" -> {\\\" - columnValue = strings.Replace(columnValue, `\\\\\"`, `\\\"`, -1) - - // Initialize a map to hold the parsed data - var result map[string]interface{} - - // Parse the JSON string into the map - err := json.Unmarshal([]byte(columnValue), &result) - if err != nil { - return "", fmt.Errorf("error converting the value to map: %v", err) - } - // Create a map to store the parsed key-value pairs - var transformedMapValue string - - //sorting the keys so that result always have keys in an order - keys := lo.Keys(result) - slices.Sort(keys) - - // Access key-value pairs to format the string as - "\"{\"\"key1\"\":\"\"value1\"\",\"\"key2\"\":\"\"value2\"\"}\""=>"{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}" - for _, key := range keys { - value := result[key] - transformedMapValue = transformedMapValue + fmt.Sprintf(`"%s"=>"%s",`, key, value) - } - - if len(transformedMapValue) > 1 { - transformedMapValue = transformedMapValue[:len(transformedMapValue)-1] //remove last comma and - } - return quoteValueIfRequired(transformedMapValue, formatIfRequired, dbzmSchema) // add quotes if required -} - var YBValueConverterSuite = map[string]ConverterFn{ - "io.debezium.data.Json": func(columnValue string, formatIfRequired bool, dbzmSchema *schemareg.ColumnSchema) (string, error) { - if dbzmSchema != nil { - colType, ok := dbzmSchema.Parameters["__debezium.source.column.type"] - if !ok || colType != "HSTORE" { - return quoteValueIfRequiredWithEscaping(columnValue, formatIfRequired, dbzmSchema) - } - return hstoreValueConverter(columnValue, formatIfRequired, dbzmSchema) - } - return quoteValueIfRequiredWithEscaping(columnValue, formatIfRequired, dbzmSchema) - }, + "io.debezium.data.Json": quoteValueIfRequiredWithEscaping, "io.debezium.data.Enum": quoteValueIfRequiredWithEscaping, "io.debezium.time.Interval": quoteValueIfRequired, "io.debezium.data.Uuid": quoteValueIfRequired, @@ -223,7 +174,6 @@ var YBValueConverterSuite = map[string]ConverterFn{ return string(hexValue), nil }, "MAP": func(columnValue string, formatIfRequired bool, dbzmSchema *schemareg.ColumnSchema) (string, error) { - fmt.Println(columnValue) return quoteValueIfRequiredWithEscaping(columnValue, formatIfRequired, dbzmSchema) //handled in exporter plugin }, "STRING": quoteValueIfRequiredWithEscaping, From 15f04d6df9557f053d78cded190fe041fc62f442 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Wed, 15 Jan 2025 22:06:23 +0530 Subject: [PATCH 23/31] fix unit test/expected file --- ...migration-report-live-migration-fallf.json | 10 +++--- .../src/tgtdb/suites/yugabytedbSuite_test.go | 35 ------------------- 2 files changed, 5 insertions(+), 40 deletions(-) diff --git a/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json b/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json index 29a90e2156..f3aaa0379d 100644 --- a/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json +++ b/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json @@ -288,8 +288,8 @@ { "table_name": "public.\"hstore_example\"", "db_type": "target", - "exported_snapshot_rows": 13, - "imported_snapshot_rows": 0, + "exported_snapshot_rows": 0, + "imported_snapshot_rows": 13, "imported_inserts": 3, "imported_updates": 5, "imported_deletes": 0, @@ -300,9 +300,9 @@ }, { "table_name": "public.\"hstore_example\"", - "db_type": "target", - "exported_snapshot_rows": 13, - "imported_snapshot_rows": 0, + "db_type": "source-replica", + "exported_snapshot_rows": 0, + "imported_snapshot_rows": 13, "imported_inserts": 3, "imported_updates": 5, "imported_deletes": 0, diff --git a/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go b/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go index 4c657efe14..8f51d621b2 100644 --- a/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go +++ b/yb-voyager/src/tgtdb/suites/yugabytedbSuite_test.go @@ -22,8 +22,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - - "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/schemareg" ) func TestStringConversionWithFormattingWithDoubleQuotes(t *testing.T) { @@ -95,36 +93,3 @@ func TestUUIDConversionWithFormatting(t *testing.T) { // Then assert.Equal(t, `'123e4567-e89b-12d3-a456-426614174000'`, result) } - -func TestHstoreValueConversion(t *testing.T) { - colSchema := &schemareg.ColumnSchema{ - Parameters: map[string]string{ - "__debezium.source.column.type": "HSTORE", - }, - } - value := `{"key1":"value1","key2":"value2"}` - result, err := YBValueConverterSuite["io.debezium.data.Json"](value, false, colSchema) - assert.NoError(t, err) - assert.Equal(t, `"key1"=>"value1","key2"=>"value2"`, result) - - result, err = YBValueConverterSuite["io.debezium.data.Json"](value, true, colSchema) - assert.NoError(t, err) - assert.Equal(t, `'"key1"=>"value1","key2"=>"value2"'`, result) - - result, err = YBValueConverterSuite["io.debezium.data.Json"]("{}", false, colSchema) - assert.NoError(t, err) - assert.Equal(t, "", result) - - result, err = YBValueConverterSuite["io.debezium.data.Json"]("{}", true, colSchema) - assert.NoError(t, err) - assert.Equal(t, "''", result) - - value = `{"\"{\"\"key1\"\":\"\"value1\"\",\"\"key2\"\":\"\"value2\"\"}\"":"{\\\"key1=value1, key2={\\\"key1=value1, key2=value2\\\"}\\\"}"}` - result, err = YBValueConverterSuite["io.debezium.data.Json"](value, false, colSchema) - assert.NoError(t, err) - assert.Equal(t, `"\"{\"\"key1\"\":\"\"value1\"\",\"\"key2\"\":\"\"value2\"\"}\""=>"{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}"`, result) - - result, err = YBValueConverterSuite["io.debezium.data.Json"](value, true, colSchema) - assert.NoError(t, err) - assert.Equal(t, `'"\"{\"\"key1\"\":\"\"value1\"\",\"\"key2\"\":\"\"value2\"\"}\""=>"{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}"'`, result) -} From d12e8ed4b6c1ae47e52869b0011ea97f7054ca5c Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Thu, 16 Jan 2025 13:41:59 +0530 Subject: [PATCH 24/31] add tests of reporting --- .../server/ybexporter/DebeziumRecordTransformer.java | 2 +- .../dummy-export-dir/schema/tables/table.sql | 2 +- migtests/tests/analyze-schema/expected_issues.json | 11 +++++++++++ migtests/tests/analyze-schema/summary.json | 2 +- .../expectedAssessmentReport.json | 10 +++++++--- .../assessment-report-test/pg_assessment_report.sql | 3 +++ yb-voyager/src/srcdb/yugabytedb.go | 2 +- 7 files changed, 25 insertions(+), 7 deletions(-) diff --git a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java index dee077b2cb..f5080d73e0 100644 --- a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java +++ b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java @@ -99,7 +99,7 @@ Escaping the key and value here for the double quote (")" and backslash char (\ mapString.append("\""); mapString.append(","); } - if(mapString.length() == 0) { + if(mapString.length() == 0) { return ""; } return mapString.toString().substring(0, mapString.length() - 1); diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql index 4da2e5f444..58281c9ee1 100755 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql @@ -386,7 +386,7 @@ CREATE TABLE public.locations ( CREATE TABLE image (title text, raster lo); -CREATE TABLE employees (id INT PRIMARY KEY, salary INT); +CREATE TABLE employees (id INT PRIMARY KEY, salary INT, data hstore); -- create table with multirange data types -- Create tables with primary keys directly diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 0bfd66b0e5..24718d7f03 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -2000,5 +2000,16 @@ "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", "GH": "", "MinimumVersionsFixedIn": null + }, + { + "IssueType": "migration_caveats", + "ObjectType": "TABLE", + "ObjectName": "employees", + "Reason": "Unsupported datatype for Live migration with fall-forward/fallback - hstore on column - data", + "SqlStatement": "CREATE TABLE employees (id INT PRIMARY KEY, salary INT, data hstore);", + "Suggestion": "", + "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null } ] diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index 5e9353c7ac..8e1ccdb641 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -27,7 +27,7 @@ { "ObjectType": "TABLE", "TotalCount": 58, - "InvalidCount": 49, + "InvalidCount": 50, "ObjectNames": "employees, image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table" }, { diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 76e5ffb549..a057bd9938 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -20,9 +20,9 @@ }, { "ObjectType": "EXTENSION", - "TotalCount": 4, + "TotalCount": 5, "InvalidCount": 0, - "ObjectNames": "citext, pgcrypto, pg_stat_statements, lo" + "ObjectNames": "citext, pgcrypto, pg_stat_statements, lo, hstore" }, { "ObjectType": "TYPE", @@ -801,7 +801,7 @@ "SchemaName": "public", "ObjectName": "combined_tbl", "RowCount": 0, - "ColumnCount": 12, + "ColumnCount": 13, "Reads": 0, "Writes": 0, "ReadsPerSecond": 0, @@ -2403,6 +2403,10 @@ { "ObjectName": "schema2.products.item (schema2.item_details)", "SqlStatement": "" + }, + { + "ObjectName": "public.combined_tbl.data (public.hstore)", + "SqlStatement": "" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index 7e3c798ead..112431f654 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -204,6 +204,8 @@ CREATE TYPE public.address_type AS ( ); CREATE EXTENSION lo; + +CREATE EXTENSION hstore; --other misc types create table public.combined_tbl ( id int, @@ -218,6 +220,7 @@ create table public.combined_tbl ( address address_type, raster lo, arr_enum enum_kind[], + data hstore, PRIMARY KEY (id, arr_enum) ); diff --git a/yb-voyager/src/srcdb/yugabytedb.go b/yb-voyager/src/srcdb/yugabytedb.go index c2c7ef07e5..add154b83e 100644 --- a/yb-voyager/src/srcdb/yugabytedb.go +++ b/yb-voyager/src/srcdb/yugabytedb.go @@ -38,7 +38,7 @@ import ( ) // Apart from these we also skip UDT columns and error out for array of enums as unsupported tables. -var YugabyteUnsupportedDataTypesForDbzm = []string{"BOX", "CIRCLE", "LINE", "LSEG", "PATH", "PG_LSN", "POINT", "POLYGON", "TSQUERY", "TSVECTOR", "TXID_SNAPSHOT", "GEOMETRY", "GEOGRAPHY", "RASTER"} +var YugabyteUnsupportedDataTypesForDbzm = []string{"BOX", "CIRCLE", "LINE", "LSEG", "PATH", "PG_LSN", "POINT", "POLYGON", "TSQUERY", "TSVECTOR", "TXID_SNAPSHOT", "GEOMETRY", "GEOGRAPHY", "RASTER", "HSTORE"} type YugabyteDB struct { source *Source From d746bd10089f2b774babe449c8178570f347c959 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Thu, 16 Jan 2025 17:03:53 +0530 Subject: [PATCH 25/31] review comments --- .../server/ybexporter/DebeziumRecordTransformer.java | 6 ++++++ migtests/tests/pg/datatypes/validate | 8 ++++++++ migtests/tests/pg/datatypes/validateAfterChanges | 7 +++++++ yb-voyager/src/tgtdb/suites/oracleSuite.go | 12 ------------ 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java index f5080d73e0..c33c54f660 100644 --- a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java +++ b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java @@ -79,6 +79,8 @@ private String makeFieldValueSerializable(Object fieldValue, Field field){ for (Map.Entry entry : ((HashMap) fieldValue).entrySet()) { String key = entry.getKey(); String val = entry.getValue(); + LOGGER.debug("[MAP] before transforming key - {}", key); + LOGGER.debug("[MAP] before transforming value - {}", val); /* Escaping the key and value here for the double quote (")" and backslash char (\) with a backslash character as mentioned here https://www.postgresql.org/docs/9/hstore.html#:~:text=To%20include%20a%20double%20quote%20or%20a%20backslash%20in%20a%20key%20or%20value%2C%20escape%20it%20with%20a%20backslash. @@ -90,6 +92,10 @@ Escaping the key and value here for the double quote (")" and backslash char (\ val = val.replace("\\", "\\\\"); key = key.replace("\"", "\\\""); // escaping double quotes " -> \" ( "a"b" -> "a\"b" ) " val = val.replace("\"", "\\\""); + + LOGGER.debug("[MAP] after transforming key - {}", key); + LOGGER.debug("[MAP] after transforming value - {}", val); + mapString.append("\""); mapString.append(key); mapString.append("\""); diff --git a/migtests/tests/pg/datatypes/validate b/migtests/tests/pg/datatypes/validate index ff35262cda..923037216a 100755 --- a/migtests/tests/pg/datatypes/validate +++ b/migtests/tests/pg/datatypes/validate @@ -147,6 +147,14 @@ def migration_completed_checks(tgt): tgt.assert_all_values_of_col("null_and_default", "i", "public", expected_values=[10, None]) tgt.assert_all_values_of_col("null_and_default", "b", "public", expected_values=[False, None]) + print("hstore_example:") + expected_hstore_values=['"f1"=>"1", "f2"=>"{\\"key1=value1, key2=value2\\"}"', None, '"json_field"=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', + '"weight"=>"11.2 ounces", "ISBN-13"=>"978-1449370000", "language"=>"English", "paperback"=>"243", "publisher"=>"postgresqltutorial.com"', '"key1"=>"value1", "key2"=>"value2"', + '"\\"{key1:value1,key2:value2}\\""=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', '"a\\"b"=>"d\\\\\\"a"', '"{\\"key1=value1, key2=value2\\"}"=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', + '"key7"=>"value7", "key8"=>"123", "key9"=>"true"', '"\\"{\\"\\"key1\\"\\":\\"\\"value1\\"\\",\\"\\"key2\\"\\":\\"\\"value2\\"\\"}\\""=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', ''] + tgt.assert_distinct_values_of_col("hstore_example", "data", "public", expected_distinct_values=expected_hstore_values) + + def YB_specific_checks(tgt): yb.verify_colocation(tgt, "postgresql") diff --git a/migtests/tests/pg/datatypes/validateAfterChanges b/migtests/tests/pg/datatypes/validateAfterChanges index a05073ec09..6d329580ba 100755 --- a/migtests/tests/pg/datatypes/validateAfterChanges +++ b/migtests/tests/pg/datatypes/validateAfterChanges @@ -130,6 +130,13 @@ def migration_completed_checks(tgt): expected_distinct_values = EXPECTED_DISNTICT_VALUES['v5'] tgt.assert_distinct_values_of_col("datatypes2", "v5", "public", None, expected_distinct_values = expected_distinct_values) + + print("hstore_example:") + expected_hstore_values=['"f1"=>"1", "f2"=>"{\\"key1=value1, key2=value2\\"}"', None, '"json_field"=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', + '"weight"=>"11.2 ounces", "ISBN-13"=>"978-1449370000", "language"=>"English", "paperback"=>"243", "publisher"=>"postgresqltutorial.com"', '"key1"=>"value1", "key2"=>"value2"', + '"\\"{key1:value1,key2:value2}\\""=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', '"a\\"b"=>"d\\\\\\"a"', '"{\\"key1=value1, key2=value2\\"}"=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', + '"key7"=>"value7", "key8"=>"123", "key9"=>"true"', '"\\"{\\"\\"key1\\"\\":\\"\\"value1\\"\\",\\"\\"key2\\"\\":\\"\\"value2\\"\\"}\\""=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', ''] + tgt.assert_distinct_values_of_col("hstore_example", "data", "public", expected_distinct_values=expected_hstore_values) if __name__ == "__main__": main() \ No newline at end of file diff --git a/yb-voyager/src/tgtdb/suites/oracleSuite.go b/yb-voyager/src/tgtdb/suites/oracleSuite.go index 12ee3dec96..d8856e2369 100644 --- a/yb-voyager/src/tgtdb/suites/oracleSuite.go +++ b/yb-voyager/src/tgtdb/suites/oracleSuite.go @@ -137,18 +137,6 @@ var OraValueConverterSuite = map[string]ConverterFn{ } return string(hexValue), nil }, - "MAP": func(columnValue string, _ bool, dbzmSchema *schemareg.ColumnSchema) (string, error) { - mapValue := make(map[string]interface{}) - err := json.Unmarshal([]byte(columnValue), &mapValue) - if err != nil { - return columnValue, fmt.Errorf("parsing map: %v", err) - } - var transformedMapValue string - for key, value := range mapValue { - transformedMapValue = transformedMapValue + fmt.Sprintf("\"%s\"=>\"%s\",", key, value) - } - return fmt.Sprintf("'%s'", transformedMapValue[:len(transformedMapValue)-1]), nil //remove last comma and add quotes - }, "STRING": func(columnValue string, formatIfRequired bool, dbzmSchema *schemareg.ColumnSchema) (string, error) { if formatIfRequired { formattedColumnValue := strings.Replace(columnValue, "'", "''", -1) From 4ce41f22b589626fc15f44d1f99b783be55bfcb3 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Thu, 16 Jan 2025 17:33:28 +0530 Subject: [PATCH 26/31] fix tests --- .../expected_files/expectedAssessmentReport.json | 6 +++++- .../expected_schema_analysis_report.json | 14 +++++++++++++- .../expected_files/expectedAssessmentReport.json | 15 ++++++++++++++- .../expected_schema_analysis_report.json | 12 ++++++++++++ .../expected_files/expectedAssessmentReport.json | 6 +++++- .../expected_schema_analysis_report.json | 12 ++++++++++++ 6 files changed, 61 insertions(+), 4 deletions(-) diff --git a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json index 6bc3e3653b..2a7d6f823f 100755 --- a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json @@ -65,7 +65,7 @@ { "ObjectType": "TABLE", "TotalCount": 339, - "InvalidCount": 8, + "InvalidCount": 9, "ObjectNames": "fn_examples.ordinary_table, agg_ex.my_table, am_examples.am_partitioned, am_examples.fast_emp4000, am_examples.heaptable, am_examples.tableam_parted_heapx, am_examples.tableam_parted_1_heapx, am_examples.tableam_parted_2_heapx, am_examples.tableam_parted_heap2, am_examples.tableam_parted_c_heap2, am_examples.tableam_parted_d_heap2, am_examples.tableam_tbl_heap2, am_examples.tableam_tbl_heapx, am_examples.tableam_tblas_heap2, am_examples.tableam_tblas_heapx, base_type_examples.default_test, composite_type_examples.ordinary_table, composite_type_examples.equivalent_rowtype, composite_type_examples.i_0, composite_type_examples.i_1, composite_type_examples.i_2, composite_type_examples.i_3, composite_type_examples.i_4, composite_type_examples.i_5, composite_type_examples.i_6, composite_type_examples.i_7, composite_type_examples.i_8, composite_type_examples.i_9, composite_type_examples.i_10, composite_type_examples.i_11, composite_type_examples.i_12, composite_type_examples.i_13, composite_type_examples.i_14, composite_type_examples.i_15, composite_type_examples.i_16, composite_type_examples.i_17, composite_type_examples.i_18, composite_type_examples.i_19, composite_type_examples.i_20, composite_type_examples.i_21, composite_type_examples.i_22, composite_type_examples.i_23, composite_type_examples.i_24, composite_type_examples.i_25, composite_type_examples.i_26, composite_type_examples.i_27, composite_type_examples.i_28, composite_type_examples.i_29, composite_type_examples.i_30, composite_type_examples.i_31, composite_type_examples.i_32, composite_type_examples.i_33, composite_type_examples.i_34, composite_type_examples.i_35, composite_type_examples.i_36, composite_type_examples.i_37, composite_type_examples.i_38, composite_type_examples.i_39, composite_type_examples.i_40, composite_type_examples.i_41, composite_type_examples.i_42, composite_type_examples.i_43, composite_type_examples.i_44, composite_type_examples.i_45, composite_type_examples.i_46, composite_type_examples.i_47, composite_type_examples.i_48, composite_type_examples.i_49, composite_type_examples.i_50, composite_type_examples.i_51, composite_type_examples.i_52, composite_type_examples.i_53, composite_type_examples.i_54, composite_type_examples.i_55, composite_type_examples.i_56, composite_type_examples.i_57, composite_type_examples.i_58, composite_type_examples.i_59, composite_type_examples.i_60, composite_type_examples.i_61, composite_type_examples.i_62, composite_type_examples.i_63, composite_type_examples.i_64, composite_type_examples.i_65, composite_type_examples.i_66, composite_type_examples.i_67, composite_type_examples.i_68, composite_type_examples.i_69, composite_type_examples.i_70, composite_type_examples.i_71, composite_type_examples.i_72, composite_type_examples.i_73, composite_type_examples.i_74, composite_type_examples.i_75, composite_type_examples.i_76, composite_type_examples.i_77, composite_type_examples.i_78, composite_type_examples.i_79, composite_type_examples.i_80, composite_type_examples.i_81, composite_type_examples.i_82, composite_type_examples.i_83, composite_type_examples.i_84, composite_type_examples.i_85, composite_type_examples.i_86, composite_type_examples.i_87, composite_type_examples.i_88, composite_type_examples.i_89, composite_type_examples.i_90, composite_type_examples.i_91, composite_type_examples.i_92, composite_type_examples.i_93, composite_type_examples.i_94, composite_type_examples.i_95, composite_type_examples.i_96, composite_type_examples.i_97, composite_type_examples.i_98, composite_type_examples.i_99, composite_type_examples.i_100, composite_type_examples.i_101, composite_type_examples.i_102, composite_type_examples.i_103, composite_type_examples.i_104, composite_type_examples.i_105, composite_type_examples.i_106, composite_type_examples.i_107, composite_type_examples.i_108, composite_type_examples.i_109, composite_type_examples.i_110, composite_type_examples.i_111, composite_type_examples.i_112, composite_type_examples.i_113, composite_type_examples.i_114, composite_type_examples.i_115, composite_type_examples.i_116, composite_type_examples.i_117, composite_type_examples.i_118, composite_type_examples.i_119, composite_type_examples.i_120, composite_type_examples.i_121, composite_type_examples.i_122, composite_type_examples.i_123, composite_type_examples.i_124, composite_type_examples.i_125, composite_type_examples.i_126, composite_type_examples.i_127, composite_type_examples.i_128, composite_type_examples.i_129, composite_type_examples.i_130, composite_type_examples.i_131, composite_type_examples.i_132, composite_type_examples.i_133, composite_type_examples.i_134, composite_type_examples.i_135, composite_type_examples.i_136, composite_type_examples.i_137, composite_type_examples.i_138, composite_type_examples.i_139, composite_type_examples.i_140, composite_type_examples.i_141, composite_type_examples.i_142, composite_type_examples.i_143, composite_type_examples.i_144, composite_type_examples.i_145, composite_type_examples.i_146, composite_type_examples.i_147, composite_type_examples.i_148, composite_type_examples.i_149, composite_type_examples.i_150, composite_type_examples.i_151, composite_type_examples.i_152, composite_type_examples.i_153, composite_type_examples.i_154, composite_type_examples.i_155, composite_type_examples.i_156, composite_type_examples.i_157, composite_type_examples.i_158, composite_type_examples.i_159, composite_type_examples.i_160, composite_type_examples.i_161, composite_type_examples.i_162, composite_type_examples.i_163, composite_type_examples.i_164, composite_type_examples.i_165, composite_type_examples.i_166, composite_type_examples.i_167, composite_type_examples.i_168, composite_type_examples.i_169, composite_type_examples.i_170, composite_type_examples.i_171, composite_type_examples.i_172, composite_type_examples.i_173, composite_type_examples.i_174, composite_type_examples.i_175, composite_type_examples.i_176, composite_type_examples.i_177, composite_type_examples.i_178, composite_type_examples.i_179, composite_type_examples.i_180, composite_type_examples.i_181, composite_type_examples.i_182, composite_type_examples.i_183, composite_type_examples.i_184, composite_type_examples.i_185, composite_type_examples.i_186, composite_type_examples.i_187, composite_type_examples.i_188, composite_type_examples.i_189, composite_type_examples.i_190, composite_type_examples.i_191, composite_type_examples.i_192, composite_type_examples.i_193, composite_type_examples.i_194, composite_type_examples.i_195, composite_type_examples.i_196, composite_type_examples.i_197, composite_type_examples.i_198, composite_type_examples.i_199, composite_type_examples.i_200, composite_type_examples.i_201, composite_type_examples.i_202, composite_type_examples.i_203, composite_type_examples.i_204, composite_type_examples.i_205, composite_type_examples.i_206, composite_type_examples.i_207, composite_type_examples.i_208, composite_type_examples.i_209, composite_type_examples.i_210, composite_type_examples.i_211, composite_type_examples.i_212, composite_type_examples.i_213, composite_type_examples.i_214, composite_type_examples.i_215, composite_type_examples.i_216, composite_type_examples.i_217, composite_type_examples.i_218, composite_type_examples.i_219, composite_type_examples.i_220, composite_type_examples.i_221, composite_type_examples.i_222, composite_type_examples.i_223, composite_type_examples.i_224, composite_type_examples.i_225, composite_type_examples.i_226, composite_type_examples.i_227, composite_type_examples.i_228, composite_type_examples.i_229, composite_type_examples.i_230, composite_type_examples.i_231, composite_type_examples.i_232, composite_type_examples.i_233, composite_type_examples.i_234, composite_type_examples.i_235, composite_type_examples.i_236, composite_type_examples.i_237, composite_type_examples.i_238, composite_type_examples.i_239, composite_type_examples.i_240, composite_type_examples.i_241, composite_type_examples.i_242, composite_type_examples.i_243, composite_type_examples.i_244, composite_type_examples.i_245, composite_type_examples.i_246, composite_type_examples.i_247, composite_type_examples.i_248, composite_type_examples.i_249, composite_type_examples.i_250, composite_type_examples.i_251, composite_type_examples.i_252, composite_type_examples.i_253, composite_type_examples.i_254, composite_type_examples.i_255, composite_type_examples.i_256, composite_type_examples.inherited_table, domain_examples.even_numbers, domain_examples.us_snail_addy, enum_example._bug_severity, enum_example.bugs, enum_example.bugs_clone, extension_example.testhstore, idx_ex.films, ordinary_tables.binary_examples, ordinary_tables.bit_string_examples, ordinary_tables.boolean_examples, ordinary_tables.character_examples, ordinary_tables.geometric_examples, ordinary_tables.money_example, ordinary_tables.network_addr_examples, ordinary_tables.numeric_type_examples, ordinary_tables.\"time\", range_type_example.example_tbl, regress_rls_schema.b1, regress_rls_schema.category, regress_rls_schema.dependee, regress_rls_schema.dependent, regress_rls_schema.dob_t1, regress_rls_schema.dob_t2, regress_rls_schema.document, regress_rls_schema.part_document, regress_rls_schema.part_document_fiction, regress_rls_schema.part_document_nonfiction, regress_rls_schema.part_document_satire, regress_rls_schema.r1, regress_rls_schema.r1_2, regress_rls_schema.r1_3, regress_rls_schema.r1_4, regress_rls_schema.r1_5, regress_rls_schema.r2, regress_rls_schema.r2_3, regress_rls_schema.r2_4, regress_rls_schema.r2_5, regress_rls_schema.rec1, regress_rls_schema.rec2, regress_rls_schema.ref_tbl, regress_rls_schema.y1, regress_rls_schema.rls_tbl, regress_rls_schema.rls_tbl_2, regress_rls_schema.rls_tbl_3, regress_rls_schema.z1, regress_rls_schema.s1, regress_rls_schema.s2, regress_rls_schema.t, regress_rls_schema.t1, regress_rls_schema.t1_2, regress_rls_schema.t1_3, regress_rls_schema.t2, regress_rls_schema.t2_3, regress_rls_schema.t3_3, regress_rls_schema.tbl1, regress_rls_schema.test_qual_pushdown, regress_rls_schema.uaccount, regress_rls_schema.x1, regress_rls_schema.y2, regress_rls_schema.z1_blacklist, regress_rls_schema.z2, trigger_test.accounts, trigger_test.update_log" }, { @@ -5644,6 +5644,10 @@ { "ObjectName": "composite_type_examples.basic_view._nested (composite_type_examples.nested)", "SqlStatement": "" + }, + { + "ObjectName": "extension_example.testhstore.h (extension_example.hstore)", + "SqlStatement": "" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", diff --git a/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json index bde78722cb..0d75055022 100755 --- a/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json @@ -67,7 +67,7 @@ { "ObjectType": "TABLE", "TotalCount": 339, - "InvalidCount": 8, + "InvalidCount": 9, "ObjectNames": "fn_examples.ordinary_table, agg_ex.my_table, am_examples.am_partitioned, am_examples.fast_emp4000, am_examples.heaptable, am_examples.tableam_parted_heapx, am_examples.tableam_parted_1_heapx, am_examples.tableam_parted_2_heapx, am_examples.tableam_parted_heap2, am_examples.tableam_parted_c_heap2, am_examples.tableam_parted_d_heap2, am_examples.tableam_tbl_heap2, am_examples.tableam_tbl_heapx, am_examples.tableam_tblas_heap2, am_examples.tableam_tblas_heapx, base_type_examples.default_test, composite_type_examples.ordinary_table, composite_type_examples.equivalent_rowtype, composite_type_examples.i_0, composite_type_examples.i_1, composite_type_examples.i_2, composite_type_examples.i_3, composite_type_examples.i_4, composite_type_examples.i_5, composite_type_examples.i_6, composite_type_examples.i_7, composite_type_examples.i_8, composite_type_examples.i_9, composite_type_examples.i_10, composite_type_examples.i_11, composite_type_examples.i_12, composite_type_examples.i_13, composite_type_examples.i_14, composite_type_examples.i_15, composite_type_examples.i_16, composite_type_examples.i_17, composite_type_examples.i_18, composite_type_examples.i_19, composite_type_examples.i_20, composite_type_examples.i_21, composite_type_examples.i_22, composite_type_examples.i_23, composite_type_examples.i_24, composite_type_examples.i_25, composite_type_examples.i_26, composite_type_examples.i_27, composite_type_examples.i_28, composite_type_examples.i_29, composite_type_examples.i_30, composite_type_examples.i_31, composite_type_examples.i_32, composite_type_examples.i_33, composite_type_examples.i_34, composite_type_examples.i_35, composite_type_examples.i_36, composite_type_examples.i_37, composite_type_examples.i_38, composite_type_examples.i_39, composite_type_examples.i_40, composite_type_examples.i_41, composite_type_examples.i_42, composite_type_examples.i_43, composite_type_examples.i_44, composite_type_examples.i_45, composite_type_examples.i_46, composite_type_examples.i_47, composite_type_examples.i_48, composite_type_examples.i_49, composite_type_examples.i_50, composite_type_examples.i_51, composite_type_examples.i_52, composite_type_examples.i_53, composite_type_examples.i_54, composite_type_examples.i_55, composite_type_examples.i_56, composite_type_examples.i_57, composite_type_examples.i_58, composite_type_examples.i_59, composite_type_examples.i_60, composite_type_examples.i_61, composite_type_examples.i_62, composite_type_examples.i_63, composite_type_examples.i_64, composite_type_examples.i_65, composite_type_examples.i_66, composite_type_examples.i_67, composite_type_examples.i_68, composite_type_examples.i_69, composite_type_examples.i_70, composite_type_examples.i_71, composite_type_examples.i_72, composite_type_examples.i_73, composite_type_examples.i_74, composite_type_examples.i_75, composite_type_examples.i_76, composite_type_examples.i_77, composite_type_examples.i_78, composite_type_examples.i_79, composite_type_examples.i_80, composite_type_examples.i_81, composite_type_examples.i_82, composite_type_examples.i_83, composite_type_examples.i_84, composite_type_examples.i_85, composite_type_examples.i_86, composite_type_examples.i_87, composite_type_examples.i_88, composite_type_examples.i_89, composite_type_examples.i_90, composite_type_examples.i_91, composite_type_examples.i_92, composite_type_examples.i_93, composite_type_examples.i_94, composite_type_examples.i_95, composite_type_examples.i_96, composite_type_examples.i_97, composite_type_examples.i_98, composite_type_examples.i_99, composite_type_examples.i_100, composite_type_examples.i_101, composite_type_examples.i_102, composite_type_examples.i_103, composite_type_examples.i_104, composite_type_examples.i_105, composite_type_examples.i_106, composite_type_examples.i_107, composite_type_examples.i_108, composite_type_examples.i_109, composite_type_examples.i_110, composite_type_examples.i_111, composite_type_examples.i_112, composite_type_examples.i_113, composite_type_examples.i_114, composite_type_examples.i_115, composite_type_examples.i_116, composite_type_examples.i_117, composite_type_examples.i_118, composite_type_examples.i_119, composite_type_examples.i_120, composite_type_examples.i_121, composite_type_examples.i_122, composite_type_examples.i_123, composite_type_examples.i_124, composite_type_examples.i_125, composite_type_examples.i_126, composite_type_examples.i_127, composite_type_examples.i_128, composite_type_examples.i_129, composite_type_examples.i_130, composite_type_examples.i_131, composite_type_examples.i_132, composite_type_examples.i_133, composite_type_examples.i_134, composite_type_examples.i_135, composite_type_examples.i_136, composite_type_examples.i_137, composite_type_examples.i_138, composite_type_examples.i_139, composite_type_examples.i_140, composite_type_examples.i_141, composite_type_examples.i_142, composite_type_examples.i_143, composite_type_examples.i_144, composite_type_examples.i_145, composite_type_examples.i_146, composite_type_examples.i_147, composite_type_examples.i_148, composite_type_examples.i_149, composite_type_examples.i_150, composite_type_examples.i_151, composite_type_examples.i_152, composite_type_examples.i_153, composite_type_examples.i_154, composite_type_examples.i_155, composite_type_examples.i_156, composite_type_examples.i_157, composite_type_examples.i_158, composite_type_examples.i_159, composite_type_examples.i_160, composite_type_examples.i_161, composite_type_examples.i_162, composite_type_examples.i_163, composite_type_examples.i_164, composite_type_examples.i_165, composite_type_examples.i_166, composite_type_examples.i_167, composite_type_examples.i_168, composite_type_examples.i_169, composite_type_examples.i_170, composite_type_examples.i_171, composite_type_examples.i_172, composite_type_examples.i_173, composite_type_examples.i_174, composite_type_examples.i_175, composite_type_examples.i_176, composite_type_examples.i_177, composite_type_examples.i_178, composite_type_examples.i_179, composite_type_examples.i_180, composite_type_examples.i_181, composite_type_examples.i_182, composite_type_examples.i_183, composite_type_examples.i_184, composite_type_examples.i_185, composite_type_examples.i_186, composite_type_examples.i_187, composite_type_examples.i_188, composite_type_examples.i_189, composite_type_examples.i_190, composite_type_examples.i_191, composite_type_examples.i_192, composite_type_examples.i_193, composite_type_examples.i_194, composite_type_examples.i_195, composite_type_examples.i_196, composite_type_examples.i_197, composite_type_examples.i_198, composite_type_examples.i_199, composite_type_examples.i_200, composite_type_examples.i_201, composite_type_examples.i_202, composite_type_examples.i_203, composite_type_examples.i_204, composite_type_examples.i_205, composite_type_examples.i_206, composite_type_examples.i_207, composite_type_examples.i_208, composite_type_examples.i_209, composite_type_examples.i_210, composite_type_examples.i_211, composite_type_examples.i_212, composite_type_examples.i_213, composite_type_examples.i_214, composite_type_examples.i_215, composite_type_examples.i_216, composite_type_examples.i_217, composite_type_examples.i_218, composite_type_examples.i_219, composite_type_examples.i_220, composite_type_examples.i_221, composite_type_examples.i_222, composite_type_examples.i_223, composite_type_examples.i_224, composite_type_examples.i_225, composite_type_examples.i_226, composite_type_examples.i_227, composite_type_examples.i_228, composite_type_examples.i_229, composite_type_examples.i_230, composite_type_examples.i_231, composite_type_examples.i_232, composite_type_examples.i_233, composite_type_examples.i_234, composite_type_examples.i_235, composite_type_examples.i_236, composite_type_examples.i_237, composite_type_examples.i_238, composite_type_examples.i_239, composite_type_examples.i_240, composite_type_examples.i_241, composite_type_examples.i_242, composite_type_examples.i_243, composite_type_examples.i_244, composite_type_examples.i_245, composite_type_examples.i_246, composite_type_examples.i_247, composite_type_examples.i_248, composite_type_examples.i_249, composite_type_examples.i_250, composite_type_examples.i_251, composite_type_examples.i_252, composite_type_examples.i_253, composite_type_examples.i_254, composite_type_examples.i_255, composite_type_examples.i_256, composite_type_examples.inherited_table, domain_examples.even_numbers, domain_examples.us_snail_addy, enum_example._bug_severity, enum_example.bugs, enum_example.bugs_clone, extension_example.testhstore, idx_ex.films, ordinary_tables.binary_examples, ordinary_tables.bit_string_examples, ordinary_tables.boolean_examples, ordinary_tables.character_examples, ordinary_tables.geometric_examples, ordinary_tables.money_example, ordinary_tables.network_addr_examples, ordinary_tables.numeric_type_examples, ordinary_tables.\"time\", range_type_example.example_tbl, regress_rls_schema.b1, regress_rls_schema.category, regress_rls_schema.dependee, regress_rls_schema.dependent, regress_rls_schema.dob_t1, regress_rls_schema.dob_t2, regress_rls_schema.document, regress_rls_schema.part_document, regress_rls_schema.part_document_fiction, regress_rls_schema.part_document_nonfiction, regress_rls_schema.part_document_satire, regress_rls_schema.r1, regress_rls_schema.r1_2, regress_rls_schema.r1_3, regress_rls_schema.r1_4, regress_rls_schema.r1_5, regress_rls_schema.r2, regress_rls_schema.r2_3, regress_rls_schema.r2_4, regress_rls_schema.r2_5, regress_rls_schema.rec1, regress_rls_schema.rec2, regress_rls_schema.ref_tbl, regress_rls_schema.y1, regress_rls_schema.rls_tbl, regress_rls_schema.rls_tbl_2, regress_rls_schema.rls_tbl_3, regress_rls_schema.z1, regress_rls_schema.s1, regress_rls_schema.s2, regress_rls_schema.t, regress_rls_schema.t1, regress_rls_schema.t1_2, regress_rls_schema.t1_3, regress_rls_schema.t2, regress_rls_schema.t2_3, regress_rls_schema.t3_3, regress_rls_schema.tbl1, regress_rls_schema.test_qual_pushdown, regress_rls_schema.uaccount, regress_rls_schema.x1, regress_rls_schema.y2, regress_rls_schema.z1_blacklist, regress_rls_schema.z2, trigger_test.accounts, trigger_test.update_log" }, { @@ -308,6 +308,18 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", "MinimumVersionsFixedIn": null }, + { + "IssueType": "migration_caveats", + "ObjectType": "TABLE", + "ObjectName": "extension_example.testhstore", + "Reason": "Unsupported datatype for Live migration with fall-forward/fallback - extension_example.hstore on column - h", + "SqlStatement": "CREATE TABLE extension_example.testhstore (\n h extension_example.hstore\n);", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", + "Suggestion": "", + "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null + }, { "IssueType": "migration_caveats", "ObjectType": "TABLE", diff --git a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json index 45fb806f22..6b5b6a2203 100755 --- a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json @@ -226,7 +226,20 @@ } ], "Notes": null, - "MigrationCaveats": null, + "MigrationCaveats": [ + { + "FeatureName": "Unsupported Data Types for Live Migration with Fall-forward/Fallback", + "Objects": [ + { + "ObjectName": "public.osm_changeset.tags (public.hstore)", + "SqlStatement": "" + } + ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "FeatureDescription": "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows.", + "MinimumVersionsFixedIn": null + } + ], "UnsupportedQueryConstructs": null, "UnsupportedPlPgSqlObjects": null } diff --git a/migtests/tests/pg/osm/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/osm/expected_files/expected_schema_analysis_report.json index 8e78f4cbb8..dfbd45e73a 100755 --- a/migtests/tests/pg/osm/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/osm/expected_files/expected_schema_analysis_report.json @@ -66,6 +66,18 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", "MinimumVersionsFixedIn": null }, + { + "IssueType": "migration_caveats", + "ObjectType": "TABLE", + "ObjectName": "public.osm_changeset", + "Reason": "Unsupported datatype for Live migration with fall-forward/fallback - public.hstore on column - tags", + "SqlStatement": "CREATE TABLE public.osm_changeset (id bigint NOT NULL, user_id bigint, created_at timestamp, min_lat numeric(10, 7), max_lat numeric(10, 7), min_lon numeric(10, 7), max_lon numeric(10, 7), closed_at timestamp, open boolean, num_changes int, user_name varchar(255), tags public.hstore, geom public.geometry(polygon, 4326)) WITH (colocation=false);", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/osm/export-dir/schema/tables/table.sql", + "Suggestion": "", + "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null + }, { "IssueType": "unsupported_features", "ObjectType": "INDEX", diff --git a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json index 47797ad8d4..d290ee72f7 100755 --- a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json @@ -30,7 +30,7 @@ { "ObjectType": "TABLE", "TotalCount": 9, - "InvalidCount": 2, + "InvalidCount": 3, "ObjectNames": "public.agent_statuses, public.agents, public.countries, public.expenses, public.expensive_items, public.gear_names, public.points, public.reports, public.secret_missions" }, { @@ -264,6 +264,10 @@ { "FeatureName": "Unsupported Data Types for Live Migration with Fall-forward/Fallback", "Objects": [ + { + "ObjectName": "public.reports.report_tsv (tsvector)", + "SqlStatement": "" + }, { "ObjectName": "public.reports.report_tsv (tsvector)", "SqlStatement": "" diff --git a/migtests/tests/pg/sample-is/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/sample-is/expected_files/expected_schema_analysis_report.json index b1f90fab0d..a179067f9b 100755 --- a/migtests/tests/pg/sample-is/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/sample-is/expected_files/expected_schema_analysis_report.json @@ -48,6 +48,18 @@ ] }, "Issues": [ + { + "IssueType": "migration_caveats", + "ObjectType": "TABLE", + "ObjectName": "public.reports", + "Reason": "Unsupported datatype for Live migration with fall-forward/fallback - public.hstore on column - attrs", + "SqlStatement": "CREATE TABLE public.reports (\n agent_uuid uuid,\n \"time\" timestamp with time zone,\n attrs public.hstore DEFAULT ''::public.hstore,\n report text,\n report_tsv tsvector\n);", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/sample-is/export-dir/schema/tables/table.sql", + "Suggestion": "", + "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null + }, { "IssueType": "unsupported_features", "ObjectType": "TABLE", From 4f779a3c0c457678f6b30440d1686af8c2b965ac Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Thu, 16 Jan 2025 17:41:18 +0530 Subject: [PATCH 27/31] fix build --- yb-voyager/src/tgtdb/suites/oracleSuite.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/yb-voyager/src/tgtdb/suites/oracleSuite.go b/yb-voyager/src/tgtdb/suites/oracleSuite.go index d8856e2369..544b25a104 100644 --- a/yb-voyager/src/tgtdb/suites/oracleSuite.go +++ b/yb-voyager/src/tgtdb/suites/oracleSuite.go @@ -17,7 +17,6 @@ package tgtdbsuite import ( "encoding/base64" - "encoding/json" "fmt" "math" "strconv" @@ -30,7 +29,7 @@ import ( var OraValueConverterSuite = map[string]ConverterFn{ "DATE": func(columnValue string, formatIfRequired bool, dbzmSchema *schemareg.ColumnSchema) (string, error) { // from oracle for DATE type debezium gives epoch milliseconds with type `io.debezium.time.Timestamp` - epochMilliSecs, err := strconv.ParseInt(columnValue, 10, 64) + epochMilliSecs, err := strconv.ParseInt(columnValue, 10, 64) if err != nil { return columnValue, fmt.Errorf("parsing epoch milliseconds: %v", err) } @@ -205,4 +204,4 @@ var OraValueConverterSuite = map[string]ConverterFn{ } return columnValue, nil }, -} \ No newline at end of file +} From 1830dad12440048589479b7a128c6f703179ca85 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Thu, 16 Jan 2025 18:16:00 +0530 Subject: [PATCH 28/31] add --yes to import-data --- migtests/scripts/functions.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index f01d384a06..f731ff28e5 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -150,8 +150,6 @@ EOF run_sqlplus_as_sys ${pdb_name} "create-pdb-tablespace.sql" cp ${SCRIPTS}/oracle/live-grants.sql oracle-inputs.sql run_sqlplus_as_sys ${cdb_name} "oracle-inputs.sql" - rm create-pdb-tablespace.sql - rm oracle-inputs.sql } grant_permissions_for_live_migration_pg() { @@ -394,6 +392,7 @@ import_data() { --send-diagnostics=false --truncate-splits true --max-retries 1 + --yes " if [ "${SOURCE_DB_TYPE}" != "postgresql" ] From 88d0cf5d841c812a53431133b87851d8ab0d5cc4 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Thu, 16 Jan 2025 19:31:12 +0530 Subject: [PATCH 29/31] live-migration validateAfterchanges fix --- migtests/tests/pg/datatypes/validateAfterChanges | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/migtests/tests/pg/datatypes/validateAfterChanges b/migtests/tests/pg/datatypes/validateAfterChanges index 6d329580ba..364c513879 100755 --- a/migtests/tests/pg/datatypes/validateAfterChanges +++ b/migtests/tests/pg/datatypes/validateAfterChanges @@ -133,9 +133,10 @@ def migration_completed_checks(tgt): print("hstore_example:") expected_hstore_values=['"f1"=>"1", "f2"=>"{\\"key1=value1, key2=value2\\"}"', None, '"json_field"=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', - '"weight"=>"11.2 ounces", "ISBN-13"=>"978-1449370000", "language"=>"English", "paperback"=>"243", "publisher"=>"postgresqltutorial.com"', '"key1"=>"value1", "key2"=>"value2"', - '"\\"{key1:value1,key2:value2}\\""=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', '"a\\"b"=>"d\\\\\\"a"', '"{\\"key1=value1, key2=value2\\"}"=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', - '"key7"=>"value7", "key8"=>"123", "key9"=>"true"', '"\\"{\\"\\"key1\\"\\":\\"\\"value1\\"\\",\\"\\"key2\\"\\":\\"\\"value2\\"\\"}\\""=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', ''] + '"weight"=>"11.2 ounces", "ISBN-13"=>"978-1449370000", "language"=>"English", "paperback"=>"243", "publisher"=>"postgresqltutorial.com"', + '"key1"=>"value1", "key2"=>"value2", "key3"=>"value3"', '"\\"{key1:value1,key2:value2}\\""=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', + '"a\\"b"=>"d\\\\\\"a"', '"{\\"key1=value1, key2=value2\\"}"=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', '"key5"=>"value5", "key6"=>"value6"', + '"\\"{\\"\\"key1\\"\\":\\"\\"value1\\"\\",\\"\\"key2\\"\\":\\"\\"value2\\"\\"}\\""=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', ''] tgt.assert_distinct_values_of_col("hstore_example", "data", "public", expected_distinct_values=expected_hstore_values) if __name__ == "__main__": From 03e2b961e083daae32921af95ca4d8611c77a1c9 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Thu, 16 Jan 2025 20:13:57 +0530 Subject: [PATCH 30/31] fix sample-is --- .../pg/sample-is/expected_files/expectedAssessmentReport.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json index d290ee72f7..68784943a0 100755 --- a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json @@ -30,7 +30,7 @@ { "ObjectType": "TABLE", "TotalCount": 9, - "InvalidCount": 3, + "InvalidCount": 2, "ObjectNames": "public.agent_statuses, public.agents, public.countries, public.expenses, public.expensive_items, public.gear_names, public.points, public.reports, public.secret_missions" }, { @@ -269,7 +269,7 @@ "SqlStatement": "" }, { - "ObjectName": "public.reports.report_tsv (tsvector)", + "ObjectName": "public.reports.attrs (public.hstore)", "SqlStatement": "" } ], From cb2794140d8ac801ca0db81496e548a35a0204c7 Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Fri, 17 Jan 2025 11:51:34 +0530 Subject: [PATCH 31/31] clean up --- .github/workflows/pg-17-migtests.yml | 8 -------- migtests/scripts/run-test.sh | 8 -------- 2 files changed, 16 deletions(-) diff --git a/.github/workflows/pg-17-migtests.yml b/.github/workflows/pg-17-migtests.yml index a8fd402f3c..27aacc0c68 100644 --- a/.github/workflows/pg-17-migtests.yml +++ b/.github/workflows/pg-17-migtests.yml @@ -230,14 +230,6 @@ jobs: # - name: "TEST: pg-basic-non-public-fall-back-test" # run: migtests/scripts/live-migration-fallb-run-test.sh pg/basic-non-public-live-test - - name: "TEST: pg-datatypes-live-test" - if: ${{ !cancelled() && matrix.test_group == 'live_basic' }} - run: migtests/scripts/live-migration-run-test.sh pg/datatypes - - - name: "TEST: pg-datatypes-fall-forward-test" - if: ${{ !cancelled() && matrix.test_group == 'live_basic' }} - run: migtests/scripts/live-migration-fallf-run-test.sh pg/datatypes - - name: "TEST: pg-datatypes-fall-back-test" if: ${{ !cancelled() && matrix.test_group == 'live_basic' }} run: migtests/scripts/live-migration-fallb-run-test.sh pg/datatypes diff --git a/migtests/scripts/run-test.sh b/migtests/scripts/run-test.sh index 0cc8b1ab4d..e40cc60d10 100755 --- a/migtests/scripts/run-test.sh +++ b/migtests/scripts/run-test.sh @@ -136,14 +136,6 @@ main() { fi fi - if [ "${TEST_DIR}" = "${TESTS_DIR}/pg/datatypes" ]; then - cat ${EXPORT_DIR}/data/hstore_example_data.sql - if [ "${BETA_FAST_DATA_EXPORT}" = "1" ]; then - cat ${EXPORT_DIR}/data/schemas/source_db_exporter/hstore_example_schema.json - cat ${EXPORT_DIR}/logs/debezium-source_db_exporter.log - fi - fi - step "Fix data." if [ -x "${TEST_DIR}/fix-data" ] then