diff --git a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java index 86ff13efbe..c33c54f660 100644 --- a/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java +++ b/debezium-server-voyager/debezium-server-voyagerexporter/src/main/java/io/debezium/server/ybexporter/DebeziumRecordTransformer.java @@ -16,12 +16,16 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Map; +import java.util.HashMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class ensures of doing any transformation of the record received from debezium * before actually writing that record. */ public class DebeziumRecordTransformer implements RecordTransformer { + private static final Logger LOGGER = LoggerFactory.getLogger(DebeziumRecordTransformer.class); private JsonConverter jsonConverter; public DebeziumRecordTransformer(){ @@ -70,6 +74,42 @@ private String makeFieldValueSerializable(Object fieldValue, Field field){ case BYTES: case STRUCT: return toKafkaConnectJsonConverted(fieldValue, field); + case MAP: + StringBuilder mapString = new StringBuilder(); + for (Map.Entry entry : ((HashMap) fieldValue).entrySet()) { + String key = entry.getKey(); + String val = entry.getValue(); + LOGGER.debug("[MAP] before transforming key - {}", key); + LOGGER.debug("[MAP] before transforming value - {}", val); + /* + Escaping the key and value here for the double quote (")" and backslash char (\) with a backslash character as mentioned here + https://www.postgresql.org/docs/9/hstore.html#:~:text=To%20include%20a%20double%20quote%20or%20a%20backslash%20in%20a%20key%20or%20value%2C%20escape%20it%20with%20a%20backslash. + + Following the order of escaping the backslash first and then the double quote becasue first escape the backslashes in the string and adding the backslash for escaping to handle case like + e.g. key - "a\"b" -> (first escaping) -> "a\\"b" -> (second escaping) -> "a\\\"b" + */ + key = key.replace("\\", "\\\\"); // escaping backslash \ -> \\ ( "a\b" -> "a\\b" ) " + val = val.replace("\\", "\\\\"); + key = key.replace("\"", "\\\""); // escaping double quotes " -> \" ( "a"b" -> "a\"b" ) " + val = val.replace("\"", "\\\""); + + LOGGER.debug("[MAP] after transforming key - {}", key); + LOGGER.debug("[MAP] after transforming value - {}", val); + + mapString.append("\""); + mapString.append(key); + mapString.append("\""); + mapString.append(" => "); + mapString.append("\""); + mapString.append(val); + mapString.append("\""); + mapString.append(","); + } + if(mapString.length() == 0) { + return ""; + } + return mapString.toString().substring(0, mapString.length() - 1); + } return fieldValue.toString(); } diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index f01d384a06..f731ff28e5 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -150,8 +150,6 @@ EOF run_sqlplus_as_sys ${pdb_name} "create-pdb-tablespace.sql" cp ${SCRIPTS}/oracle/live-grants.sql oracle-inputs.sql run_sqlplus_as_sys ${cdb_name} "oracle-inputs.sql" - rm create-pdb-tablespace.sql - rm oracle-inputs.sql } grant_permissions_for_live_migration_pg() { @@ -394,6 +392,7 @@ import_data() { --send-diagnostics=false --truncate-splits true --max-retries 1 + --yes " if [ "${SOURCE_DB_TYPE}" != "postgresql" ] diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql index 48908efa2d..1507ee7938 100755 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql @@ -390,12 +390,13 @@ CREATE TABLE public.locations ( CREATE TABLE image (title text, raster lo); +CREATE TABLE employees (id INT PRIMARY KEY, salary INT, data hstore); + -- IS JSON Predicate CREATE TABLE public.json_data ( id SERIAL PRIMARY KEY, data_column TEXT NOT NULL CHECK (data_column IS JSON) ); -CREATE TABLE employees (id INT PRIMARY KEY, salary INT); -- create table with multirange data types -- Create tables with primary keys directly diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 8bb85fa16a..942be5e30c 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -2143,5 +2143,16 @@ "GH": "https://github.com/yugabyte/yugabyte-db/issues/25575", "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#postgresql-12-and-later-features", "MinimumVersionsFixedIn": null + }, + { + "IssueType": "migration_caveats", + "ObjectType": "TABLE", + "ObjectName": "employees", + "Reason": "Unsupported datatype for Live migration with fall-forward/fallback - hstore on column - data", + "SqlStatement": "CREATE TABLE employees (id INT PRIMARY KEY, salary INT, data hstore);", + "Suggestion": "", + "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null } ] diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index 2a24c33dd6..80dee631a4 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -33,7 +33,7 @@ { "ObjectType": "TABLE", "TotalCount": 64, - "InvalidCount": 53, + "InvalidCount": 54, "ObjectNames": "test_table_in_type_file, sales_data, salaries2, sales, test_1, test_2, test_non_pk_multi_column_list, test_3, test_4, test_5, test_6, test_7, test_8, test_9, order_details, public.employees4, enum_example.bugs, table_xyz, table_abc, table_1, table_test, test_interval, public.range_columns_partition_test, public.range_columns_partition_test_copy, anydata_test, anydataset_test, anytype_test, uritype_test, \"Test\", public.meeting, public.pr, public.foreign_def_test, public.users, foreign_def_test1, foreign_def_test2, unique_def_test, unique_def_test1, test_xml_type, test_xid_type, public.test_jsonb, public.inet_type, public.citext_type, public.documents, public.ts_query_table, combined_tbl, combined_tbl1, test_udt, test_arr_enum, public.locations, public.xml_data_example, image, public.json_data, employees, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table, users_unique_nulls_distinct, users_unique_nulls_not_distinct, sales_unique_nulls_not_distinct, sales_unique_nulls_not_distinct_alter, users_unique_nulls_not_distinct_index" }, { "ObjectType": "INDEX", diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index d26b9969e8..bac1b1b592 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -20,9 +20,9 @@ }, { "ObjectType": "EXTENSION", - "TotalCount": 4, + "TotalCount": 5, "InvalidCount": 0, - "ObjectNames": "citext, pgcrypto, pg_stat_statements, lo" + "ObjectNames": "citext, pgcrypto, pg_stat_statements, lo, hstore" }, { "ObjectType": "TYPE", @@ -1107,7 +1107,7 @@ "SchemaName": "public", "ObjectName": "combined_tbl", "RowCount": 0, - "ColumnCount": 12, + "ColumnCount": 13, "Reads": 0, "Writes": 0, "ReadsPerSecond": 0, @@ -2779,6 +2779,10 @@ { "ObjectName": "schema2.products.item (schema2.item_details)", "SqlStatement": "" + }, + { + "ObjectName": "public.combined_tbl.data (public.hstore)", + "SqlStatement": "" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index c0ffcb0d5d..d194862313 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -206,6 +206,8 @@ CREATE TYPE public.address_type AS ( ); CREATE EXTENSION lo; + +CREATE EXTENSION hstore; --other misc types create table public.combined_tbl ( id int, @@ -220,6 +222,7 @@ create table public.combined_tbl ( address address_type, raster lo, arr_enum enum_kind[], + data hstore, PRIMARY KEY (id, arr_enum) ); diff --git a/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json b/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json index 5eee6a3d6a..f3aaa0379d 100644 --- a/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json +++ b/migtests/tests/pg/datatypes/data-migration-report-live-migration-fallf.json @@ -271,5 +271,44 @@ "exported_updates": 0, "exported_deletes": 0, "final_row_count": 2 + }, + { + "table_name": "public.\"hstore_example\"", + "db_type": "source", + "exported_snapshot_rows": 13, + "imported_snapshot_rows": 0, + "imported_inserts": 0, + "imported_updates": 0, + "imported_deletes": 0, + "exported_inserts": 3, + "exported_updates": 5, + "exported_deletes": 0, + "final_row_count": 16 + }, + { + "table_name": "public.\"hstore_example\"", + "db_type": "target", + "exported_snapshot_rows": 0, + "imported_snapshot_rows": 13, + "imported_inserts": 3, + "imported_updates": 5, + "imported_deletes": 0, + "exported_inserts": 0, + "exported_updates": 0, + "exported_deletes": 0, + "final_row_count": 16 + }, + { + "table_name": "public.\"hstore_example\"", + "db_type": "source-replica", + "exported_snapshot_rows": 0, + "imported_snapshot_rows": 13, + "imported_inserts": 3, + "imported_updates": 5, + "imported_deletes": 0, + "exported_inserts": 0, + "exported_updates": 0, + "exported_deletes": 0, + "final_row_count": 16 } ] \ No newline at end of file diff --git a/migtests/tests/pg/datatypes/export_data_status-report.json b/migtests/tests/pg/datatypes/export_data_status-report.json index beca513dfb..dc91417c35 100644 --- a/migtests/tests/pg/datatypes/export_data_status-report.json +++ b/migtests/tests/pg/datatypes/export_data_status-report.json @@ -33,5 +33,10 @@ "table_name": "null_and_default", "status": "DONE", "exported_count": 2 + }, + { + "exported_count": 13, + "status": "DONE", + "table_name": "hstore_example" } ] diff --git a/migtests/tests/pg/datatypes/import_data_status-report.json b/migtests/tests/pg/datatypes/import_data_status-report.json index c90c3670b7..d7453e7876 100644 --- a/migtests/tests/pg/datatypes/import_data_status-report.json +++ b/migtests/tests/pg/datatypes/import_data_status-report.json @@ -34,6 +34,13 @@ "imported_count": 3, "percentage_complete": 100 }, + { + "table_name": "public.\"hstore_example\"", + "status": "DONE", + "total_count": 13, + "imported_count": 13, + "percentage_complete": 100 + }, { "table_name": "public.\"null_and_default\"", "status": "DONE", diff --git a/migtests/tests/pg/datatypes/pg_datatypes_data.sql b/migtests/tests/pg/datatypes/pg_datatypes_data.sql index d9b41cd6e1..230b9aae68 100644 --- a/migtests/tests/pg/datatypes/pg_datatypes_data.sql +++ b/migtests/tests/pg/datatypes/pg_datatypes_data.sql @@ -34,3 +34,22 @@ select * from datatypes2; insert into null_and_default (id) VALUES (1); insert into null_and_default VALUES(2, NULL, NULL, NULL); +INSERT INTO hstore_example (data) +VALUES + ('"key1"=>"value1", "key2"=>"value2"'), + (hstore('a"b', 'd\"a')), + (NULL), + (''), + ('key1 => value1, key2 => value2'), + (hstore(ARRAY['key1', 'key2'], ARRAY['value1', 'value2'])), + ('key7 => value7, key8 => 123, key9 => true'), + ('"paperback" => "243", + "publisher" => "postgresqltutorial.com", + "language" => "English", + "ISBN-13" => "978-1449370000", + "weight" => "11.2 ounces"'), + (hstore(ROW(1,'{"key1=value1, key2=value2"}'))), + (hstore('json_field', '{"key1=value1, key2={"key1=value1, key2=value2"}"}')), --hstore() key and values need no extra processing + ('"{\"key1=value1, key2=value2\"}"=>"{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}"'), --single quotes string need to escaped properly + (hstore('"{""key1"":""value1"",""key2"":""value2""}"', '{"key1=value1, key2={"key1=value1, key2=value2"}"}')), + (hstore('"{key1:value1,key2:value2}"', '{"key1=value1, key2={"key1=value1, key2=value2"}"}')); diff --git a/migtests/tests/pg/datatypes/pg_datatypes_schema.sql b/migtests/tests/pg/datatypes/pg_datatypes_schema.sql index ed26faa3ba..35ea49c61f 100644 --- a/migtests/tests/pg/datatypes/pg_datatypes_schema.sql +++ b/migtests/tests/pg/datatypes/pg_datatypes_schema.sql @@ -41,5 +41,9 @@ create table datatypes2(id serial primary key, v1 json, v2 BIT(10), v3 int ARRAY drop table if exists null_and_default; create table null_and_default(id int PRIMARY KEY, b boolean default false, i int default 10, val varchar default 'testdefault'); +create EXTENSION hstore; - +CREATE TABLE hstore_example ( + id SERIAL PRIMARY KEY, + data hstore +); diff --git a/migtests/tests/pg/datatypes/source_delta.sql b/migtests/tests/pg/datatypes/source_delta.sql index 8a54829cde..eefaa46681 100644 --- a/migtests/tests/pg/datatypes/source_delta.sql +++ b/migtests/tests/pg/datatypes/source_delta.sql @@ -51,5 +51,34 @@ UPDATE datatypes2 SET v1 = '{"updated": true}', v2 = B'0101010101', v5 = B'101010101010101010101010101010', v3 = ARRAY[5, 6, 7, 8], v4 = '{{"e", "f"}, {"g", "h"}}' WHERE v1 IS NULL; +UPDATE hstore_example +SET data = data || 'key3 => value3' +WHERE id = 1; +UPDATE hstore_example +SET data = hstore('{"key1=value1, key2=value2"}', '{"key1=value1, key2={"key1=value1, key2=value2"}"}') +WHERE id = 3; +UPDATE hstore_example +SET data = '"{\"key1=value1, key2=value2\"}"=>"{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}"' +WHERE id = 7; + +INSERT INTO hstore_example (data) +VALUES + ('key5 => value5, key6 => value6'); + +INSERT INTO hstore_example (data) +VALUES + (hstore('{"key1=value1, key2=value2"}', '{"key1=value1, key2={"key1=value1, key2=value2"}"}')); + +INSERT INTO hstore_example (data) +VALUES + (''); + +UPDATE hstore_example +SET data = NULL +WHERE id = 5; + +UPDATE hstore_example +SET data = '' +WHERE id = 6; \ No newline at end of file diff --git a/migtests/tests/pg/datatypes/target_delta.sql b/migtests/tests/pg/datatypes/target_delta.sql index 5b277ff516..d5502ae47d 100644 --- a/migtests/tests/pg/datatypes/target_delta.sql +++ b/migtests/tests/pg/datatypes/target_delta.sql @@ -61,3 +61,26 @@ SET v1 = '{"new": "data"}', v2 = B'1111000011', v5=B'001010100101010101010101010 DELETE FROM datatypes2 WHERE 5 = ANY(v3); + +-- NOT WORKING WIT H YB CDC GRPC connector as of now +-- INSERT INTO hstore_example (data) +-- VALUES +-- ('key7 => value7, key8 => value8'); + +-- UPDATE hstore_example +-- SET data = delete(data, 'key2') +-- WHERE id = 8; + +-- DELETE FROM hstore_example WHERE data ? 'key5'; + +-- INSERT INTO hstore_example (data) +-- VALUES +-- (hstore('"{""key1"":""value1"",""key2"":""value2""}"', '{"key1=value1, key2={"key1=value1, key2=value2"}"}')); + +-- UPDATE hstore_example +-- SET data = hstore('{"key1=value1, key2=value2"}', '{"key1=value1, key2={"key1=value1, key2=value2"}"}') +-- WHERE id = 15; + +-- UPDATE hstore_example +-- SET data = '"{\"key1=value1, key2=value2\"}"=>"{\"key1=value1, key2={\"key1=value1, key2=value2\"}\"}"' +-- WHERE id = 14; diff --git a/migtests/tests/pg/datatypes/validate b/migtests/tests/pg/datatypes/validate index 5eba851246..923037216a 100755 --- a/migtests/tests/pg/datatypes/validate +++ b/migtests/tests/pg/datatypes/validate @@ -26,6 +26,7 @@ EXPECTED_ROW_COUNT = { 'datetime_type2': 2, 'null_and_default' :2, 'decimal_types': 3, + 'hstore_example': 13, } EXPECTED_SUM_OF_COLUMN = { @@ -73,7 +74,7 @@ def migration_completed_checks_fb(): def migration_completed_checks(tgt): table_list = tgt.get_table_names("public") print("table_list:", table_list) - assert len(table_list) == 7 + assert len(table_list) == 8 got_row_count = tgt.row_count_of_all_tables("public") for table_name, row_count in EXPECTED_ROW_COUNT.items(): @@ -146,6 +147,14 @@ def migration_completed_checks(tgt): tgt.assert_all_values_of_col("null_and_default", "i", "public", expected_values=[10, None]) tgt.assert_all_values_of_col("null_and_default", "b", "public", expected_values=[False, None]) + print("hstore_example:") + expected_hstore_values=['"f1"=>"1", "f2"=>"{\\"key1=value1, key2=value2\\"}"', None, '"json_field"=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', + '"weight"=>"11.2 ounces", "ISBN-13"=>"978-1449370000", "language"=>"English", "paperback"=>"243", "publisher"=>"postgresqltutorial.com"', '"key1"=>"value1", "key2"=>"value2"', + '"\\"{key1:value1,key2:value2}\\""=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', '"a\\"b"=>"d\\\\\\"a"', '"{\\"key1=value1, key2=value2\\"}"=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', + '"key7"=>"value7", "key8"=>"123", "key9"=>"true"', '"\\"{\\"\\"key1\\"\\":\\"\\"value1\\"\\",\\"\\"key2\\"\\":\\"\\"value2\\"\\"}\\""=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', ''] + tgt.assert_distinct_values_of_col("hstore_example", "data", "public", expected_distinct_values=expected_hstore_values) + + def YB_specific_checks(tgt): yb.verify_colocation(tgt, "postgresql") diff --git a/migtests/tests/pg/datatypes/validateAfterChanges b/migtests/tests/pg/datatypes/validateAfterChanges index f5aa16c726..364c513879 100755 --- a/migtests/tests/pg/datatypes/validateAfterChanges +++ b/migtests/tests/pg/datatypes/validateAfterChanges @@ -29,6 +29,7 @@ EXPECTED_ROW_COUNT = { 'datetime_type2': 2, 'null_and_default' :2, 'decimal_types': 4, + 'hstore_example': 16, } EXPECTED_SUM_OF_COLUMN = { @@ -65,6 +66,7 @@ EXPECTED_ROW_COUNT_FF = { 'datetime_type2': 3, 'null_and_default' :2, 'decimal_types': 4, + 'hstore_example': 16, } EXPECTED_SUM_OF_COLUMN_FF = { @@ -128,6 +130,14 @@ def migration_completed_checks(tgt): expected_distinct_values = EXPECTED_DISNTICT_VALUES['v5'] tgt.assert_distinct_values_of_col("datatypes2", "v5", "public", None, expected_distinct_values = expected_distinct_values) + + print("hstore_example:") + expected_hstore_values=['"f1"=>"1", "f2"=>"{\\"key1=value1, key2=value2\\"}"', None, '"json_field"=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', + '"weight"=>"11.2 ounces", "ISBN-13"=>"978-1449370000", "language"=>"English", "paperback"=>"243", "publisher"=>"postgresqltutorial.com"', + '"key1"=>"value1", "key2"=>"value2", "key3"=>"value3"', '"\\"{key1:value1,key2:value2}\\""=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', + '"a\\"b"=>"d\\\\\\"a"', '"{\\"key1=value1, key2=value2\\"}"=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', '"key5"=>"value5", "key6"=>"value6"', + '"\\"{\\"\\"key1\\"\\":\\"\\"value1\\"\\",\\"\\"key2\\"\\":\\"\\"value2\\"\\"}\\""=>"{\\"key1=value1, key2={\\"key1=value1, key2=value2\\"}\\"}"', ''] + tgt.assert_distinct_values_of_col("hstore_example", "data", "public", expected_distinct_values=expected_hstore_values) if __name__ == "__main__": main() \ No newline at end of file diff --git a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json index 6bc3e3653b..2a7d6f823f 100755 --- a/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/omnibus/expected_files/expectedAssessmentReport.json @@ -65,7 +65,7 @@ { "ObjectType": "TABLE", "TotalCount": 339, - "InvalidCount": 8, + "InvalidCount": 9, "ObjectNames": "fn_examples.ordinary_table, agg_ex.my_table, am_examples.am_partitioned, am_examples.fast_emp4000, am_examples.heaptable, am_examples.tableam_parted_heapx, am_examples.tableam_parted_1_heapx, am_examples.tableam_parted_2_heapx, am_examples.tableam_parted_heap2, am_examples.tableam_parted_c_heap2, am_examples.tableam_parted_d_heap2, am_examples.tableam_tbl_heap2, am_examples.tableam_tbl_heapx, am_examples.tableam_tblas_heap2, am_examples.tableam_tblas_heapx, base_type_examples.default_test, composite_type_examples.ordinary_table, composite_type_examples.equivalent_rowtype, composite_type_examples.i_0, composite_type_examples.i_1, composite_type_examples.i_2, composite_type_examples.i_3, composite_type_examples.i_4, composite_type_examples.i_5, composite_type_examples.i_6, composite_type_examples.i_7, composite_type_examples.i_8, composite_type_examples.i_9, composite_type_examples.i_10, composite_type_examples.i_11, composite_type_examples.i_12, composite_type_examples.i_13, composite_type_examples.i_14, composite_type_examples.i_15, composite_type_examples.i_16, composite_type_examples.i_17, composite_type_examples.i_18, composite_type_examples.i_19, composite_type_examples.i_20, composite_type_examples.i_21, composite_type_examples.i_22, composite_type_examples.i_23, composite_type_examples.i_24, composite_type_examples.i_25, composite_type_examples.i_26, composite_type_examples.i_27, composite_type_examples.i_28, composite_type_examples.i_29, composite_type_examples.i_30, composite_type_examples.i_31, composite_type_examples.i_32, composite_type_examples.i_33, composite_type_examples.i_34, composite_type_examples.i_35, composite_type_examples.i_36, composite_type_examples.i_37, composite_type_examples.i_38, composite_type_examples.i_39, composite_type_examples.i_40, composite_type_examples.i_41, composite_type_examples.i_42, composite_type_examples.i_43, composite_type_examples.i_44, composite_type_examples.i_45, composite_type_examples.i_46, composite_type_examples.i_47, composite_type_examples.i_48, composite_type_examples.i_49, composite_type_examples.i_50, composite_type_examples.i_51, composite_type_examples.i_52, composite_type_examples.i_53, composite_type_examples.i_54, composite_type_examples.i_55, composite_type_examples.i_56, composite_type_examples.i_57, composite_type_examples.i_58, composite_type_examples.i_59, composite_type_examples.i_60, composite_type_examples.i_61, composite_type_examples.i_62, composite_type_examples.i_63, composite_type_examples.i_64, composite_type_examples.i_65, composite_type_examples.i_66, composite_type_examples.i_67, composite_type_examples.i_68, composite_type_examples.i_69, composite_type_examples.i_70, composite_type_examples.i_71, composite_type_examples.i_72, composite_type_examples.i_73, composite_type_examples.i_74, composite_type_examples.i_75, composite_type_examples.i_76, composite_type_examples.i_77, composite_type_examples.i_78, composite_type_examples.i_79, composite_type_examples.i_80, composite_type_examples.i_81, composite_type_examples.i_82, composite_type_examples.i_83, composite_type_examples.i_84, composite_type_examples.i_85, composite_type_examples.i_86, composite_type_examples.i_87, composite_type_examples.i_88, composite_type_examples.i_89, composite_type_examples.i_90, composite_type_examples.i_91, composite_type_examples.i_92, composite_type_examples.i_93, composite_type_examples.i_94, composite_type_examples.i_95, composite_type_examples.i_96, composite_type_examples.i_97, composite_type_examples.i_98, composite_type_examples.i_99, composite_type_examples.i_100, composite_type_examples.i_101, composite_type_examples.i_102, composite_type_examples.i_103, composite_type_examples.i_104, composite_type_examples.i_105, composite_type_examples.i_106, composite_type_examples.i_107, composite_type_examples.i_108, composite_type_examples.i_109, composite_type_examples.i_110, composite_type_examples.i_111, composite_type_examples.i_112, composite_type_examples.i_113, composite_type_examples.i_114, composite_type_examples.i_115, composite_type_examples.i_116, composite_type_examples.i_117, composite_type_examples.i_118, composite_type_examples.i_119, composite_type_examples.i_120, composite_type_examples.i_121, composite_type_examples.i_122, composite_type_examples.i_123, composite_type_examples.i_124, composite_type_examples.i_125, composite_type_examples.i_126, composite_type_examples.i_127, composite_type_examples.i_128, composite_type_examples.i_129, composite_type_examples.i_130, composite_type_examples.i_131, composite_type_examples.i_132, composite_type_examples.i_133, composite_type_examples.i_134, composite_type_examples.i_135, composite_type_examples.i_136, composite_type_examples.i_137, composite_type_examples.i_138, composite_type_examples.i_139, composite_type_examples.i_140, composite_type_examples.i_141, composite_type_examples.i_142, composite_type_examples.i_143, composite_type_examples.i_144, composite_type_examples.i_145, composite_type_examples.i_146, composite_type_examples.i_147, composite_type_examples.i_148, composite_type_examples.i_149, composite_type_examples.i_150, composite_type_examples.i_151, composite_type_examples.i_152, composite_type_examples.i_153, composite_type_examples.i_154, composite_type_examples.i_155, composite_type_examples.i_156, composite_type_examples.i_157, composite_type_examples.i_158, composite_type_examples.i_159, composite_type_examples.i_160, composite_type_examples.i_161, composite_type_examples.i_162, composite_type_examples.i_163, composite_type_examples.i_164, composite_type_examples.i_165, composite_type_examples.i_166, composite_type_examples.i_167, composite_type_examples.i_168, composite_type_examples.i_169, composite_type_examples.i_170, composite_type_examples.i_171, composite_type_examples.i_172, composite_type_examples.i_173, composite_type_examples.i_174, composite_type_examples.i_175, composite_type_examples.i_176, composite_type_examples.i_177, composite_type_examples.i_178, composite_type_examples.i_179, composite_type_examples.i_180, composite_type_examples.i_181, composite_type_examples.i_182, composite_type_examples.i_183, composite_type_examples.i_184, composite_type_examples.i_185, composite_type_examples.i_186, composite_type_examples.i_187, composite_type_examples.i_188, composite_type_examples.i_189, composite_type_examples.i_190, composite_type_examples.i_191, composite_type_examples.i_192, composite_type_examples.i_193, composite_type_examples.i_194, composite_type_examples.i_195, composite_type_examples.i_196, composite_type_examples.i_197, composite_type_examples.i_198, composite_type_examples.i_199, composite_type_examples.i_200, composite_type_examples.i_201, composite_type_examples.i_202, composite_type_examples.i_203, composite_type_examples.i_204, composite_type_examples.i_205, composite_type_examples.i_206, composite_type_examples.i_207, composite_type_examples.i_208, composite_type_examples.i_209, composite_type_examples.i_210, composite_type_examples.i_211, composite_type_examples.i_212, composite_type_examples.i_213, composite_type_examples.i_214, composite_type_examples.i_215, composite_type_examples.i_216, composite_type_examples.i_217, composite_type_examples.i_218, composite_type_examples.i_219, composite_type_examples.i_220, composite_type_examples.i_221, composite_type_examples.i_222, composite_type_examples.i_223, composite_type_examples.i_224, composite_type_examples.i_225, composite_type_examples.i_226, composite_type_examples.i_227, composite_type_examples.i_228, composite_type_examples.i_229, composite_type_examples.i_230, composite_type_examples.i_231, composite_type_examples.i_232, composite_type_examples.i_233, composite_type_examples.i_234, composite_type_examples.i_235, composite_type_examples.i_236, composite_type_examples.i_237, composite_type_examples.i_238, composite_type_examples.i_239, composite_type_examples.i_240, composite_type_examples.i_241, composite_type_examples.i_242, composite_type_examples.i_243, composite_type_examples.i_244, composite_type_examples.i_245, composite_type_examples.i_246, composite_type_examples.i_247, composite_type_examples.i_248, composite_type_examples.i_249, composite_type_examples.i_250, composite_type_examples.i_251, composite_type_examples.i_252, composite_type_examples.i_253, composite_type_examples.i_254, composite_type_examples.i_255, composite_type_examples.i_256, composite_type_examples.inherited_table, domain_examples.even_numbers, domain_examples.us_snail_addy, enum_example._bug_severity, enum_example.bugs, enum_example.bugs_clone, extension_example.testhstore, idx_ex.films, ordinary_tables.binary_examples, ordinary_tables.bit_string_examples, ordinary_tables.boolean_examples, ordinary_tables.character_examples, ordinary_tables.geometric_examples, ordinary_tables.money_example, ordinary_tables.network_addr_examples, ordinary_tables.numeric_type_examples, ordinary_tables.\"time\", range_type_example.example_tbl, regress_rls_schema.b1, regress_rls_schema.category, regress_rls_schema.dependee, regress_rls_schema.dependent, regress_rls_schema.dob_t1, regress_rls_schema.dob_t2, regress_rls_schema.document, regress_rls_schema.part_document, regress_rls_schema.part_document_fiction, regress_rls_schema.part_document_nonfiction, regress_rls_schema.part_document_satire, regress_rls_schema.r1, regress_rls_schema.r1_2, regress_rls_schema.r1_3, regress_rls_schema.r1_4, regress_rls_schema.r1_5, regress_rls_schema.r2, regress_rls_schema.r2_3, regress_rls_schema.r2_4, regress_rls_schema.r2_5, regress_rls_schema.rec1, regress_rls_schema.rec2, regress_rls_schema.ref_tbl, regress_rls_schema.y1, regress_rls_schema.rls_tbl, regress_rls_schema.rls_tbl_2, regress_rls_schema.rls_tbl_3, regress_rls_schema.z1, regress_rls_schema.s1, regress_rls_schema.s2, regress_rls_schema.t, regress_rls_schema.t1, regress_rls_schema.t1_2, regress_rls_schema.t1_3, regress_rls_schema.t2, regress_rls_schema.t2_3, regress_rls_schema.t3_3, regress_rls_schema.tbl1, regress_rls_schema.test_qual_pushdown, regress_rls_schema.uaccount, regress_rls_schema.x1, regress_rls_schema.y2, regress_rls_schema.z1_blacklist, regress_rls_schema.z2, trigger_test.accounts, trigger_test.update_log" }, { @@ -5644,6 +5644,10 @@ { "ObjectName": "composite_type_examples.basic_view._nested (composite_type_examples.nested)", "SqlStatement": "" + }, + { + "ObjectName": "extension_example.testhstore.h (extension_example.hstore)", + "SqlStatement": "" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", diff --git a/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json index bde78722cb..0d75055022 100755 --- a/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/omnibus/expected_files/expected_schema_analysis_report.json @@ -67,7 +67,7 @@ { "ObjectType": "TABLE", "TotalCount": 339, - "InvalidCount": 8, + "InvalidCount": 9, "ObjectNames": "fn_examples.ordinary_table, agg_ex.my_table, am_examples.am_partitioned, am_examples.fast_emp4000, am_examples.heaptable, am_examples.tableam_parted_heapx, am_examples.tableam_parted_1_heapx, am_examples.tableam_parted_2_heapx, am_examples.tableam_parted_heap2, am_examples.tableam_parted_c_heap2, am_examples.tableam_parted_d_heap2, am_examples.tableam_tbl_heap2, am_examples.tableam_tbl_heapx, am_examples.tableam_tblas_heap2, am_examples.tableam_tblas_heapx, base_type_examples.default_test, composite_type_examples.ordinary_table, composite_type_examples.equivalent_rowtype, composite_type_examples.i_0, composite_type_examples.i_1, composite_type_examples.i_2, composite_type_examples.i_3, composite_type_examples.i_4, composite_type_examples.i_5, composite_type_examples.i_6, composite_type_examples.i_7, composite_type_examples.i_8, composite_type_examples.i_9, composite_type_examples.i_10, composite_type_examples.i_11, composite_type_examples.i_12, composite_type_examples.i_13, composite_type_examples.i_14, composite_type_examples.i_15, composite_type_examples.i_16, composite_type_examples.i_17, composite_type_examples.i_18, composite_type_examples.i_19, composite_type_examples.i_20, composite_type_examples.i_21, composite_type_examples.i_22, composite_type_examples.i_23, composite_type_examples.i_24, composite_type_examples.i_25, composite_type_examples.i_26, composite_type_examples.i_27, composite_type_examples.i_28, composite_type_examples.i_29, composite_type_examples.i_30, composite_type_examples.i_31, composite_type_examples.i_32, composite_type_examples.i_33, composite_type_examples.i_34, composite_type_examples.i_35, composite_type_examples.i_36, composite_type_examples.i_37, composite_type_examples.i_38, composite_type_examples.i_39, composite_type_examples.i_40, composite_type_examples.i_41, composite_type_examples.i_42, composite_type_examples.i_43, composite_type_examples.i_44, composite_type_examples.i_45, composite_type_examples.i_46, composite_type_examples.i_47, composite_type_examples.i_48, composite_type_examples.i_49, composite_type_examples.i_50, composite_type_examples.i_51, composite_type_examples.i_52, composite_type_examples.i_53, composite_type_examples.i_54, composite_type_examples.i_55, composite_type_examples.i_56, composite_type_examples.i_57, composite_type_examples.i_58, composite_type_examples.i_59, composite_type_examples.i_60, composite_type_examples.i_61, composite_type_examples.i_62, composite_type_examples.i_63, composite_type_examples.i_64, composite_type_examples.i_65, composite_type_examples.i_66, composite_type_examples.i_67, composite_type_examples.i_68, composite_type_examples.i_69, composite_type_examples.i_70, composite_type_examples.i_71, composite_type_examples.i_72, composite_type_examples.i_73, composite_type_examples.i_74, composite_type_examples.i_75, composite_type_examples.i_76, composite_type_examples.i_77, composite_type_examples.i_78, composite_type_examples.i_79, composite_type_examples.i_80, composite_type_examples.i_81, composite_type_examples.i_82, composite_type_examples.i_83, composite_type_examples.i_84, composite_type_examples.i_85, composite_type_examples.i_86, composite_type_examples.i_87, composite_type_examples.i_88, composite_type_examples.i_89, composite_type_examples.i_90, composite_type_examples.i_91, composite_type_examples.i_92, composite_type_examples.i_93, composite_type_examples.i_94, composite_type_examples.i_95, composite_type_examples.i_96, composite_type_examples.i_97, composite_type_examples.i_98, composite_type_examples.i_99, composite_type_examples.i_100, composite_type_examples.i_101, composite_type_examples.i_102, composite_type_examples.i_103, composite_type_examples.i_104, composite_type_examples.i_105, composite_type_examples.i_106, composite_type_examples.i_107, composite_type_examples.i_108, composite_type_examples.i_109, composite_type_examples.i_110, composite_type_examples.i_111, composite_type_examples.i_112, composite_type_examples.i_113, composite_type_examples.i_114, composite_type_examples.i_115, composite_type_examples.i_116, composite_type_examples.i_117, composite_type_examples.i_118, composite_type_examples.i_119, composite_type_examples.i_120, composite_type_examples.i_121, composite_type_examples.i_122, composite_type_examples.i_123, composite_type_examples.i_124, composite_type_examples.i_125, composite_type_examples.i_126, composite_type_examples.i_127, composite_type_examples.i_128, composite_type_examples.i_129, composite_type_examples.i_130, composite_type_examples.i_131, composite_type_examples.i_132, composite_type_examples.i_133, composite_type_examples.i_134, composite_type_examples.i_135, composite_type_examples.i_136, composite_type_examples.i_137, composite_type_examples.i_138, composite_type_examples.i_139, composite_type_examples.i_140, composite_type_examples.i_141, composite_type_examples.i_142, composite_type_examples.i_143, composite_type_examples.i_144, composite_type_examples.i_145, composite_type_examples.i_146, composite_type_examples.i_147, composite_type_examples.i_148, composite_type_examples.i_149, composite_type_examples.i_150, composite_type_examples.i_151, composite_type_examples.i_152, composite_type_examples.i_153, composite_type_examples.i_154, composite_type_examples.i_155, composite_type_examples.i_156, composite_type_examples.i_157, composite_type_examples.i_158, composite_type_examples.i_159, composite_type_examples.i_160, composite_type_examples.i_161, composite_type_examples.i_162, composite_type_examples.i_163, composite_type_examples.i_164, composite_type_examples.i_165, composite_type_examples.i_166, composite_type_examples.i_167, composite_type_examples.i_168, composite_type_examples.i_169, composite_type_examples.i_170, composite_type_examples.i_171, composite_type_examples.i_172, composite_type_examples.i_173, composite_type_examples.i_174, composite_type_examples.i_175, composite_type_examples.i_176, composite_type_examples.i_177, composite_type_examples.i_178, composite_type_examples.i_179, composite_type_examples.i_180, composite_type_examples.i_181, composite_type_examples.i_182, composite_type_examples.i_183, composite_type_examples.i_184, composite_type_examples.i_185, composite_type_examples.i_186, composite_type_examples.i_187, composite_type_examples.i_188, composite_type_examples.i_189, composite_type_examples.i_190, composite_type_examples.i_191, composite_type_examples.i_192, composite_type_examples.i_193, composite_type_examples.i_194, composite_type_examples.i_195, composite_type_examples.i_196, composite_type_examples.i_197, composite_type_examples.i_198, composite_type_examples.i_199, composite_type_examples.i_200, composite_type_examples.i_201, composite_type_examples.i_202, composite_type_examples.i_203, composite_type_examples.i_204, composite_type_examples.i_205, composite_type_examples.i_206, composite_type_examples.i_207, composite_type_examples.i_208, composite_type_examples.i_209, composite_type_examples.i_210, composite_type_examples.i_211, composite_type_examples.i_212, composite_type_examples.i_213, composite_type_examples.i_214, composite_type_examples.i_215, composite_type_examples.i_216, composite_type_examples.i_217, composite_type_examples.i_218, composite_type_examples.i_219, composite_type_examples.i_220, composite_type_examples.i_221, composite_type_examples.i_222, composite_type_examples.i_223, composite_type_examples.i_224, composite_type_examples.i_225, composite_type_examples.i_226, composite_type_examples.i_227, composite_type_examples.i_228, composite_type_examples.i_229, composite_type_examples.i_230, composite_type_examples.i_231, composite_type_examples.i_232, composite_type_examples.i_233, composite_type_examples.i_234, composite_type_examples.i_235, composite_type_examples.i_236, composite_type_examples.i_237, composite_type_examples.i_238, composite_type_examples.i_239, composite_type_examples.i_240, composite_type_examples.i_241, composite_type_examples.i_242, composite_type_examples.i_243, composite_type_examples.i_244, composite_type_examples.i_245, composite_type_examples.i_246, composite_type_examples.i_247, composite_type_examples.i_248, composite_type_examples.i_249, composite_type_examples.i_250, composite_type_examples.i_251, composite_type_examples.i_252, composite_type_examples.i_253, composite_type_examples.i_254, composite_type_examples.i_255, composite_type_examples.i_256, composite_type_examples.inherited_table, domain_examples.even_numbers, domain_examples.us_snail_addy, enum_example._bug_severity, enum_example.bugs, enum_example.bugs_clone, extension_example.testhstore, idx_ex.films, ordinary_tables.binary_examples, ordinary_tables.bit_string_examples, ordinary_tables.boolean_examples, ordinary_tables.character_examples, ordinary_tables.geometric_examples, ordinary_tables.money_example, ordinary_tables.network_addr_examples, ordinary_tables.numeric_type_examples, ordinary_tables.\"time\", range_type_example.example_tbl, regress_rls_schema.b1, regress_rls_schema.category, regress_rls_schema.dependee, regress_rls_schema.dependent, regress_rls_schema.dob_t1, regress_rls_schema.dob_t2, regress_rls_schema.document, regress_rls_schema.part_document, regress_rls_schema.part_document_fiction, regress_rls_schema.part_document_nonfiction, regress_rls_schema.part_document_satire, regress_rls_schema.r1, regress_rls_schema.r1_2, regress_rls_schema.r1_3, regress_rls_schema.r1_4, regress_rls_schema.r1_5, regress_rls_schema.r2, regress_rls_schema.r2_3, regress_rls_schema.r2_4, regress_rls_schema.r2_5, regress_rls_schema.rec1, regress_rls_schema.rec2, regress_rls_schema.ref_tbl, regress_rls_schema.y1, regress_rls_schema.rls_tbl, regress_rls_schema.rls_tbl_2, regress_rls_schema.rls_tbl_3, regress_rls_schema.z1, regress_rls_schema.s1, regress_rls_schema.s2, regress_rls_schema.t, regress_rls_schema.t1, regress_rls_schema.t1_2, regress_rls_schema.t1_3, regress_rls_schema.t2, regress_rls_schema.t2_3, regress_rls_schema.t3_3, regress_rls_schema.tbl1, regress_rls_schema.test_qual_pushdown, regress_rls_schema.uaccount, regress_rls_schema.x1, regress_rls_schema.y2, regress_rls_schema.z1_blacklist, regress_rls_schema.z2, trigger_test.accounts, trigger_test.update_log" }, { @@ -308,6 +308,18 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", "MinimumVersionsFixedIn": null }, + { + "IssueType": "migration_caveats", + "ObjectType": "TABLE", + "ObjectName": "extension_example.testhstore", + "Reason": "Unsupported datatype for Live migration with fall-forward/fallback - extension_example.hstore on column - h", + "SqlStatement": "CREATE TABLE extension_example.testhstore (\n h extension_example.hstore\n);", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/omnibus/export-dir/schema/tables/table.sql", + "Suggestion": "", + "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null + }, { "IssueType": "migration_caveats", "ObjectType": "TABLE", diff --git a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json index 45fb806f22..6b5b6a2203 100755 --- a/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/osm/expected_files/expectedAssessmentReport.json @@ -226,7 +226,20 @@ } ], "Notes": null, - "MigrationCaveats": null, + "MigrationCaveats": [ + { + "FeatureName": "Unsupported Data Types for Live Migration with Fall-forward/Fallback", + "Objects": [ + { + "ObjectName": "public.osm_changeset.tags (public.hstore)", + "SqlStatement": "" + } + ], + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "FeatureDescription": "There are some data types in the schema that are not supported by live migration with fall-forward/fall-back. These columns will be excluded when exporting and importing data in live migration workflows.", + "MinimumVersionsFixedIn": null + } + ], "UnsupportedQueryConstructs": null, "UnsupportedPlPgSqlObjects": null } diff --git a/migtests/tests/pg/osm/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/osm/expected_files/expected_schema_analysis_report.json index 8e78f4cbb8..dfbd45e73a 100755 --- a/migtests/tests/pg/osm/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/osm/expected_files/expected_schema_analysis_report.json @@ -66,6 +66,18 @@ "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-yugabytedb", "MinimumVersionsFixedIn": null }, + { + "IssueType": "migration_caveats", + "ObjectType": "TABLE", + "ObjectName": "public.osm_changeset", + "Reason": "Unsupported datatype for Live migration with fall-forward/fallback - public.hstore on column - tags", + "SqlStatement": "CREATE TABLE public.osm_changeset (id bigint NOT NULL, user_id bigint, created_at timestamp, min_lat numeric(10, 7), max_lat numeric(10, 7), min_lon numeric(10, 7), max_lon numeric(10, 7), closed_at timestamp, open boolean, num_changes int, user_name varchar(255), tags public.hstore, geom public.geometry(polygon, 4326)) WITH (colocation=false);", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/osm/export-dir/schema/tables/table.sql", + "Suggestion": "", + "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null + }, { "IssueType": "unsupported_features", "ObjectType": "INDEX", diff --git a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json index 47797ad8d4..68784943a0 100755 --- a/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json +++ b/migtests/tests/pg/sample-is/expected_files/expectedAssessmentReport.json @@ -267,6 +267,10 @@ { "ObjectName": "public.reports.report_tsv (tsvector)", "SqlStatement": "" + }, + { + "ObjectName": "public.reports.attrs (public.hstore)", + "SqlStatement": "" } ], "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", diff --git a/migtests/tests/pg/sample-is/expected_files/expected_schema_analysis_report.json b/migtests/tests/pg/sample-is/expected_files/expected_schema_analysis_report.json index b1f90fab0d..a179067f9b 100755 --- a/migtests/tests/pg/sample-is/expected_files/expected_schema_analysis_report.json +++ b/migtests/tests/pg/sample-is/expected_files/expected_schema_analysis_report.json @@ -48,6 +48,18 @@ ] }, "Issues": [ + { + "IssueType": "migration_caveats", + "ObjectType": "TABLE", + "ObjectName": "public.reports", + "Reason": "Unsupported datatype for Live migration with fall-forward/fallback - public.hstore on column - attrs", + "SqlStatement": "CREATE TABLE public.reports (\n agent_uuid uuid,\n \"time\" timestamp with time zone,\n attrs public.hstore DEFAULT ''::public.hstore,\n report text,\n report_tsv tsvector\n);", + "FilePath": "/Users/priyanshigupta/Documents/voyager/yb-voyager/migtests/tests/pg/sample-is/export-dir/schema/tables/table.sql", + "Suggestion": "", + "GH": "https://github.com/yugabyte/yb-voyager/issues/1731", + "DocsLink": "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#unsupported-datatypes-by-voyager-during-live-migration", + "MinimumVersionsFixedIn": null + }, { "IssueType": "unsupported_features", "ObjectType": "TABLE", diff --git a/yb-voyager/src/dbzm/config.go b/yb-voyager/src/dbzm/config.go index b8b2909e6f..48739fee5d 100644 --- a/yb-voyager/src/dbzm/config.go +++ b/yb-voyager/src/dbzm/config.go @@ -27,6 +27,7 @@ import ( "github.com/google/uuid" log "github.com/sirupsen/logrus" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils" ) diff --git a/yb-voyager/src/srcdb/yugabytedb.go b/yb-voyager/src/srcdb/yugabytedb.go index 96daa1307b..3af954ae91 100644 --- a/yb-voyager/src/srcdb/yugabytedb.go +++ b/yb-voyager/src/srcdb/yugabytedb.go @@ -38,7 +38,7 @@ import ( ) // Apart from these we also skip UDT columns and error out for array of enums as unsupported tables. -var YugabyteUnsupportedDataTypesForDbzm = []string{"BOX", "CIRCLE", "LINE", "LSEG", "PATH", "PG_LSN", "POINT", "POLYGON", "TSQUERY", "TSVECTOR", "TXID_SNAPSHOT", "GEOMETRY", "GEOGRAPHY", "RASTER"} +var YugabyteUnsupportedDataTypesForDbzm = []string{"BOX", "CIRCLE", "LINE", "LSEG", "PATH", "PG_LSN", "POINT", "POLYGON", "TSQUERY", "TSVECTOR", "TXID_SNAPSHOT", "GEOMETRY", "GEOGRAPHY", "RASTER", "HSTORE"} type YugabyteDB struct { source *Source diff --git a/yb-voyager/src/tgtdb/suites/oracleSuite.go b/yb-voyager/src/tgtdb/suites/oracleSuite.go index 12ee3dec96..544b25a104 100644 --- a/yb-voyager/src/tgtdb/suites/oracleSuite.go +++ b/yb-voyager/src/tgtdb/suites/oracleSuite.go @@ -17,7 +17,6 @@ package tgtdbsuite import ( "encoding/base64" - "encoding/json" "fmt" "math" "strconv" @@ -30,7 +29,7 @@ import ( var OraValueConverterSuite = map[string]ConverterFn{ "DATE": func(columnValue string, formatIfRequired bool, dbzmSchema *schemareg.ColumnSchema) (string, error) { // from oracle for DATE type debezium gives epoch milliseconds with type `io.debezium.time.Timestamp` - epochMilliSecs, err := strconv.ParseInt(columnValue, 10, 64) + epochMilliSecs, err := strconv.ParseInt(columnValue, 10, 64) if err != nil { return columnValue, fmt.Errorf("parsing epoch milliseconds: %v", err) } @@ -137,18 +136,6 @@ var OraValueConverterSuite = map[string]ConverterFn{ } return string(hexValue), nil }, - "MAP": func(columnValue string, _ bool, dbzmSchema *schemareg.ColumnSchema) (string, error) { - mapValue := make(map[string]interface{}) - err := json.Unmarshal([]byte(columnValue), &mapValue) - if err != nil { - return columnValue, fmt.Errorf("parsing map: %v", err) - } - var transformedMapValue string - for key, value := range mapValue { - transformedMapValue = transformedMapValue + fmt.Sprintf("\"%s\"=>\"%s\",", key, value) - } - return fmt.Sprintf("'%s'", transformedMapValue[:len(transformedMapValue)-1]), nil //remove last comma and add quotes - }, "STRING": func(columnValue string, formatIfRequired bool, dbzmSchema *schemareg.ColumnSchema) (string, error) { if formatIfRequired { formattedColumnValue := strings.Replace(columnValue, "'", "''", -1) @@ -217,4 +204,4 @@ var OraValueConverterSuite = map[string]ConverterFn{ } return columnValue, nil }, -} \ No newline at end of file +} diff --git a/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go b/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go index 5ca6291479..e71ee948e8 100644 --- a/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go +++ b/yb-voyager/src/tgtdb/suites/yugabytedbSuite.go @@ -19,13 +19,13 @@ package tgtdbsuite import ( "encoding/base64" "encoding/binary" - "encoding/json" "fmt" "strconv" "strings" "time" "github.com/samber/lo" + "github.com/yugabyte/yb-voyager/yb-voyager/src/utils/schemareg" ) @@ -173,17 +173,8 @@ var YBValueConverterSuite = map[string]ConverterFn{ } return string(hexValue), nil }, - "MAP": func(columnValue string, _ bool, _ *schemareg.ColumnSchema) (string, error) { - mapValue := make(map[string]interface{}) - err := json.Unmarshal([]byte(columnValue), &mapValue) - if err != nil { - return columnValue, fmt.Errorf("parsing map: %v", err) - } - var transformedMapValue string - for key, value := range mapValue { - transformedMapValue = transformedMapValue + fmt.Sprintf("\"%s\"=>\"%s\",", key, value) - } - return fmt.Sprintf("'%s'", transformedMapValue[:len(transformedMapValue)-1]), nil //remove last comma and add quotes + "MAP": func(columnValue string, formatIfRequired bool, dbzmSchema *schemareg.ColumnSchema) (string, error) { + return quoteValueIfRequiredWithEscaping(columnValue, formatIfRequired, dbzmSchema) //handled in exporter plugin }, "STRING": quoteValueIfRequiredWithEscaping, }