From db871b42c4947a8657b7860452bff0b47ada20ae Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 22 May 2018 12:36:52 +0300 Subject: [PATCH 1/4] Support for GENERATED (aka virtual) columns --- go/base/context.go | 2 ++ go/logic/applier.go | 2 +- go/logic/inspect.go | 30 ++++++++++++------- go/mysql/utils.go | 17 +++++++---- localtests/generated-columns57/create.sql | 30 +++++++++++++++++++ .../generated-columns57/ignore_versions | 1 + localtests/json57dml/create.sql | 2 +- 7 files changed, 67 insertions(+), 17 deletions(-) create mode 100644 localtests/generated-columns57/create.sql create mode 100644 localtests/generated-columns57/ignore_versions diff --git a/go/base/context.go b/go/base/context.go index 94039e38b..2357c54a9 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -186,8 +186,10 @@ type MigrationContext struct { OriginalTableColumnsOnApplier *sql.ColumnList OriginalTableColumns *sql.ColumnList + OriginalTableVirtualColumns *sql.ColumnList OriginalTableUniqueKeys [](*sql.UniqueKey) GhostTableColumns *sql.ColumnList + GhostTableVirtualColumns *sql.ColumnList GhostTableUniqueKeys [](*sql.UniqueKey) UniqueKey *sql.UniqueKey SharedColumns *sql.ColumnList diff --git a/go/logic/applier.go b/go/logic/applier.go index afdc45fd4..ae59f62e5 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -117,7 +117,7 @@ func (this *Applier) validateAndReadTimeZone() error { // readTableColumns reads table columns on applier func (this *Applier) readTableColumns() (err error) { log.Infof("Examining table structure on applier") - this.migrationContext.OriginalTableColumnsOnApplier, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName) + this.migrationContext.OriginalTableColumnsOnApplier, _, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName) if err != nil { return err } diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 2568afed9..dc69b8123 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -89,24 +89,24 @@ func (this *Inspector) ValidateOriginalTable() (err error) { return nil } -func (this *Inspector) InspectTableColumnsAndUniqueKeys(tableName string) (columns *sql.ColumnList, uniqueKeys [](*sql.UniqueKey), err error) { +func (this *Inspector) InspectTableColumnsAndUniqueKeys(tableName string) (columns *sql.ColumnList, virtualColumns *sql.ColumnList, uniqueKeys [](*sql.UniqueKey), err error) { uniqueKeys, err = this.getCandidateUniqueKeys(tableName) if err != nil { - return columns, uniqueKeys, err + return columns, virtualColumns, uniqueKeys, err } if len(uniqueKeys) == 0 { - return columns, uniqueKeys, fmt.Errorf("No PRIMARY nor UNIQUE key found in table! Bailing out") + return columns, virtualColumns, uniqueKeys, fmt.Errorf("No PRIMARY nor UNIQUE key found in table! Bailing out") } - columns, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, tableName) + columns, virtualColumns, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, tableName) if err != nil { - return columns, uniqueKeys, err + return columns, virtualColumns, uniqueKeys, err } - return columns, uniqueKeys, nil + return columns, virtualColumns, uniqueKeys, nil } func (this *Inspector) InspectOriginalTable() (err error) { - this.migrationContext.OriginalTableColumns, this.migrationContext.OriginalTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.OriginalTableName) + this.migrationContext.OriginalTableColumns, this.migrationContext.OriginalTableVirtualColumns, this.migrationContext.OriginalTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.OriginalTableName) if err != nil { return err } @@ -122,7 +122,7 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { return fmt.Errorf("It seems like table structure is not identical between master and replica. This scenario is not supported.") } - this.migrationContext.GhostTableColumns, this.migrationContext.GhostTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.GetGhostTableName()) + this.migrationContext.GhostTableColumns, this.migrationContext.GhostTableVirtualColumns, this.migrationContext.GhostTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.GetGhostTableName()) if err != nil { return err } @@ -166,7 +166,7 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { } } - this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns = this.getSharedColumns(this.migrationContext.OriginalTableColumns, this.migrationContext.GhostTableColumns, this.migrationContext.ColumnRenameMap) + this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns = this.getSharedColumns(this.migrationContext.OriginalTableColumns, this.migrationContext.GhostTableColumns, this.migrationContext.OriginalTableVirtualColumns, this.migrationContext.GhostTableVirtualColumns, this.migrationContext.ColumnRenameMap) log.Infof("Shared columns are %s", this.migrationContext.SharedColumns) // By fact that a non-empty unique key exists we also know the shared columns are non-empty @@ -692,7 +692,7 @@ func (this *Inspector) getSharedUniqueKeys(originalUniqueKeys, ghostUniqueKeys [ } // getSharedColumns returns the intersection of two lists of columns in same order as the first list -func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.ColumnList, columnRenameMap map[string]string) (*sql.ColumnList, *sql.ColumnList) { +func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.ColumnList, originalVirtualColumns, ghostVirtualColumns *sql.ColumnList, columnRenameMap map[string]string) (*sql.ColumnList, *sql.ColumnList) { sharedColumnNames := []string{} for _, originalColumn := range originalColumns.Names() { isSharedColumn := false @@ -709,6 +709,16 @@ func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.Colum isSharedColumn = false } } + for _, virtualColumn := range originalVirtualColumns.Names() { + if strings.EqualFold(originalColumn, virtualColumn) { + isSharedColumn = false + } + } + for _, virtualColumn := range ghostVirtualColumns.Names() { + if strings.EqualFold(originalColumn, virtualColumn) { + isSharedColumn = false + } + } if isSharedColumn { sharedColumnNames = append(sharedColumnNames, originalColumn) } diff --git a/go/mysql/utils.go b/go/mysql/utils.go index 532cbb4d5..67ed7f81f 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -8,6 +8,7 @@ package mysql import ( gosql "database/sql" "fmt" + "strings" "sync" "time" @@ -178,7 +179,7 @@ func GetInstanceKey(db *gosql.DB) (instanceKey *InstanceKey, err error) { } // GetTableColumns reads column list from given table -func GetTableColumns(db *gosql.DB, databaseName, tableName string) (*sql.ColumnList, error) { +func GetTableColumns(db *gosql.DB, databaseName, tableName string) (*sql.ColumnList, *sql.ColumnList, error) { query := fmt.Sprintf(` show columns from %s.%s `, @@ -186,18 +187,24 @@ func GetTableColumns(db *gosql.DB, databaseName, tableName string) (*sql.ColumnL sql.EscapeName(tableName), ) columnNames := []string{} + virtualColumnNames := []string{} err := sqlutils.QueryRowsMap(db, query, func(rowMap sqlutils.RowMap) error { - columnNames = append(columnNames, rowMap.GetString("Field")) + columnName := rowMap.GetString("Field") + columnNames = append(columnNames, columnName) + if strings.Contains(rowMap.GetString("Extra"), " GENERATED") { + log.Debugf("%s is a generated column", columnName) + virtualColumnNames = append(virtualColumnNames, columnName) + } return nil }) if err != nil { - return nil, err + return nil, nil, err } if len(columnNames) == 0 { - return nil, log.Errorf("Found 0 columns on %s.%s. Bailing out", + return nil, nil, log.Errorf("Found 0 columns on %s.%s. Bailing out", sql.EscapeName(databaseName), sql.EscapeName(tableName), ) } - return sql.NewColumnList(columnNames), nil + return sql.NewColumnList(columnNames), sql.NewColumnList(virtualColumnNames), nil } diff --git a/localtests/generated-columns57/create.sql b/localtests/generated-columns57/create.sql new file mode 100644 index 000000000..e244ca3c0 --- /dev/null +++ b/localtests/generated-columns57/create.sql @@ -0,0 +1,30 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + a int not null, + b int not null, + sum_ab int as (a + b) virtual not null, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test (id, a, b) values (null, 2,3); + insert into gh_ost_test (id, a, b) values (null, 2,4); + insert into gh_ost_test (id, a, b) values (null, 2,5); + insert into gh_ost_test (id, a, b) values (null, 2,6); + insert into gh_ost_test (id, a, b) values (null, 2,7); + insert into gh_ost_test (id, a, b) values (null, 2,8); + insert into gh_ost_test (id, a, b) values (null, 2,9); + insert into gh_ost_test (id, a, b) values (null, 2,0); + insert into gh_ost_test (id, a, b) values (null, 2,1); + insert into gh_ost_test (id, a, b) values (null, 2,2); +end ;; diff --git a/localtests/generated-columns57/ignore_versions b/localtests/generated-columns57/ignore_versions new file mode 100644 index 000000000..b6de5f8d9 --- /dev/null +++ b/localtests/generated-columns57/ignore_versions @@ -0,0 +1 @@ +(5.5|5.6) diff --git a/localtests/json57dml/create.sql b/localtests/json57dml/create.sql index ac209414d..0e76b2eca 100644 --- a/localtests/json57dml/create.sql +++ b/localtests/json57dml/create.sql @@ -19,7 +19,7 @@ create event gh_ost_test begin insert into gh_ost_test (id, i, j) values (null, 11, '"sometext"'); insert into gh_ost_test (id, i, j) values (null, 13, '{"key":"val"}'); - insert into gh_ost_test (id, i, j) values (null, 17, '{"is-it": true, "count": 3, "elements": []}'); + insert into gh_ost_test (id, i, j) values (null, 17, '{"is-it": true, "count": 3, "elements": []}'); insert into gh_ost_test (id, i, j) values (null, 19, '{"text":"Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum semper nisi. Aenean vulputate eleifend tellus. Aenean leo ligula, porttitor eu, consequat vitae, eleifend ac, enim. Aliquam lorem ante, dapibus in, viverra quis, feugiat a, tellus. Phasellus viverra nulla ut metus varius laoreet. Quisque rutrum. Aenean imperdiet. Etiam ultricies nisi vel augue. Curabitur ullamcorper ultricies nisi. Nam eget dui. Etiam rhoncus. Maecenas tempus, tellus eget condimentum rhoncus, sem quam semper libero, sit amet adipiscing sem neque sed ipsum. Nam quam nunc, blandit vel, luctus pulvinar, hendrerit id, lorem. Maecenas nec odio et ante tincidunt tempus. Donec vitae sapien ut libero venenatis faucibus. Nullam quis ante. Etiam sit amet orci eget eros faucibus tincidunt. Duis leo. Sed fringilla mauris sit amet nibh. Donec sodales sagittis magna. Sed consequat, leo eget bibendum sodales, augue velit cursus nunc, quis gravida magna mi a libero. Fusce vulputate eleifend sapien. Vestibulum purus quam, scelerisque ut, mollis sed, nonummy id, metus. Nullam accumsan lorem in dui. Cras ultricies mi eu turpis hendrerit fringilla. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; In ac dui quis mi consectetuer lacinia. Nam pretium turpis et arcu. Duis arcu tortor, suscipit eget, imperdiet nec, imperdiet iaculis, ipsum. Sed aliquam ultrices mauris. Integer ante arcu, accumsan a, consectetuer eget, posuere ut, mauris. Praesent adipiscing. Phasellus ullamcorper ipsum rutrum nunc. Nunc nonummy metus. Vestibulum volutpat pretium libero. Cras id dui. Aenean ut eros et nisl sagittis vestibulum. Nullam nulla eros, ultricies sit amet, nonummy id, imperdiet feugiat, pede. Sed lectus. Donec mollis hendrerit risus. Phasellus nec sem in justo pellentesque facilisis. Etiam imperdiet imperdiet orci. Nunc nec neque. Phasellus leo dolor, tempus non, auctor et, hendrerit quis, nisi. Curabitur ligula sapien, tincidunt non, euismod vitae, posuere imperdiet, leo. Maecenas malesuada. Praesent congue erat at massa. Sed cursus turpis vitae tortor. Donec posuere vulputate arcu. Phasellus accumsan cursus velit. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Sed aliquam, nisi quis porttitor congue, elit erat euismod orci, ac"}'); update gh_ost_test set j = '{"updated": 11}', updated = 1 where i = 11 and updated = 0; From 03b8dadda691d9080075c6a71ad41cb734102fa6 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 22 May 2018 12:55:57 +0300 Subject: [PATCH 2/4] added generated column rename test --- .../generated-columns-rename57/create.sql | 30 +++++++++++++++++++ .../generated-columns-rename57/extra_args | 1 + .../ignore_versions | 1 + 3 files changed, 32 insertions(+) create mode 100644 localtests/generated-columns-rename57/create.sql create mode 100644 localtests/generated-columns-rename57/extra_args create mode 100644 localtests/generated-columns-rename57/ignore_versions diff --git a/localtests/generated-columns-rename57/create.sql b/localtests/generated-columns-rename57/create.sql new file mode 100644 index 000000000..e244ca3c0 --- /dev/null +++ b/localtests/generated-columns-rename57/create.sql @@ -0,0 +1,30 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + a int not null, + b int not null, + sum_ab int as (a + b) virtual not null, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test (id, a, b) values (null, 2,3); + insert into gh_ost_test (id, a, b) values (null, 2,4); + insert into gh_ost_test (id, a, b) values (null, 2,5); + insert into gh_ost_test (id, a, b) values (null, 2,6); + insert into gh_ost_test (id, a, b) values (null, 2,7); + insert into gh_ost_test (id, a, b) values (null, 2,8); + insert into gh_ost_test (id, a, b) values (null, 2,9); + insert into gh_ost_test (id, a, b) values (null, 2,0); + insert into gh_ost_test (id, a, b) values (null, 2,1); + insert into gh_ost_test (id, a, b) values (null, 2,2); +end ;; diff --git a/localtests/generated-columns-rename57/extra_args b/localtests/generated-columns-rename57/extra_args new file mode 100644 index 000000000..6a190988a --- /dev/null +++ b/localtests/generated-columns-rename57/extra_args @@ -0,0 +1 @@ +--alter="change sum_ab total_ab int as (a + b) virtual not null" --approve-renamed-columns diff --git a/localtests/generated-columns-rename57/ignore_versions b/localtests/generated-columns-rename57/ignore_versions new file mode 100644 index 000000000..b6de5f8d9 --- /dev/null +++ b/localtests/generated-columns-rename57/ignore_versions @@ -0,0 +1 @@ +(5.5|5.6) From 5275ad93421d89d2bbb0c24a675e6ddca3923b8c Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 22 May 2018 13:05:12 +0300 Subject: [PATCH 3/4] updated docs --- doc/requirements-and-limitations.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/doc/requirements-and-limitations.md b/doc/requirements-and-limitations.md index 3d174a79a..99e3ee3b9 100644 --- a/doc/requirements-and-limitations.md +++ b/doc/requirements-and-limitations.md @@ -26,10 +26,6 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th - Triggers are not supported. They may be supported in the future. -- MySQL 5.7 generated columns are not supported. They may be supported in the future. - -- MySQL 5.7 `POINT` column type is not supported. - - MySQL 5.7 `JSON` columns are supported but not as part of `PRIMARY KEY` - The two _before_ & _after_ tables must share a `PRIMARY KEY` or other `UNIQUE KEY`. This key will be used by `gh-ost` to iterate through the table rows when copying. [Read more](shared-key.md) @@ -53,4 +49,4 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th - Migrating a `FEDERATED` table is unsupported and is irrelevant to the problem `gh-ost` tackles. -- `ALTER TABLE ... RENAME TO some_other_name` is not supported (and you shouldn't use `gh-ost` for such a trivial operation). \ No newline at end of file +- `ALTER TABLE ... RENAME TO some_other_name` is not supported (and you shouldn't use `gh-ost` for such a trivial operation). From 2c6b79c4e78916cc7cc0068a4e95dc5cf4709c42 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 22 May 2018 14:09:48 +0300 Subject: [PATCH 4/4] test: add generated column --- localtests/generated-columns-add57/create.sql | 29 +++++++++++++++++++ localtests/generated-columns-add57/extra_args | 1 + .../generated-columns-add57/ghost_columns | 1 + .../generated-columns-add57/ignore_versions | 1 + localtests/generated-columns-add57/order_by | 1 + .../generated-columns-add57/orig_columns | 1 + 6 files changed, 34 insertions(+) create mode 100644 localtests/generated-columns-add57/create.sql create mode 100644 localtests/generated-columns-add57/extra_args create mode 100644 localtests/generated-columns-add57/ghost_columns create mode 100644 localtests/generated-columns-add57/ignore_versions create mode 100644 localtests/generated-columns-add57/order_by create mode 100644 localtests/generated-columns-add57/orig_columns diff --git a/localtests/generated-columns-add57/create.sql b/localtests/generated-columns-add57/create.sql new file mode 100644 index 000000000..ef795baa9 --- /dev/null +++ b/localtests/generated-columns-add57/create.sql @@ -0,0 +1,29 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + a int not null, + b int not null, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test (id, a, b) values (null, 2,3); + insert into gh_ost_test (id, a, b) values (null, 2,4); + insert into gh_ost_test (id, a, b) values (null, 2,5); + insert into gh_ost_test (id, a, b) values (null, 2,6); + insert into gh_ost_test (id, a, b) values (null, 2,7); + insert into gh_ost_test (id, a, b) values (null, 2,8); + insert into gh_ost_test (id, a, b) values (null, 2,9); + insert into gh_ost_test (id, a, b) values (null, 2,0); + insert into gh_ost_test (id, a, b) values (null, 2,1); + insert into gh_ost_test (id, a, b) values (null, 2,2); +end ;; diff --git a/localtests/generated-columns-add57/extra_args b/localtests/generated-columns-add57/extra_args new file mode 100644 index 000000000..b2bf5bc00 --- /dev/null +++ b/localtests/generated-columns-add57/extra_args @@ -0,0 +1 @@ +--alter="add column sum_ab int as (a + b) virtual not null" diff --git a/localtests/generated-columns-add57/ghost_columns b/localtests/generated-columns-add57/ghost_columns new file mode 100644 index 000000000..bd17155d3 --- /dev/null +++ b/localtests/generated-columns-add57/ghost_columns @@ -0,0 +1 @@ +id, a, b diff --git a/localtests/generated-columns-add57/ignore_versions b/localtests/generated-columns-add57/ignore_versions new file mode 100644 index 000000000..b6de5f8d9 --- /dev/null +++ b/localtests/generated-columns-add57/ignore_versions @@ -0,0 +1 @@ +(5.5|5.6) diff --git a/localtests/generated-columns-add57/order_by b/localtests/generated-columns-add57/order_by new file mode 100644 index 000000000..074d1eeb4 --- /dev/null +++ b/localtests/generated-columns-add57/order_by @@ -0,0 +1 @@ +id diff --git a/localtests/generated-columns-add57/orig_columns b/localtests/generated-columns-add57/orig_columns new file mode 100644 index 000000000..bd17155d3 --- /dev/null +++ b/localtests/generated-columns-add57/orig_columns @@ -0,0 +1 @@ +id, a, b