Merge pull request #595 from github/support-generated-columns

Support for GENERATED (aka virtual) columns

doc/requirements-and-limitations.md

@@ -26,10 +26,6 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th
 
 - Triggers are not supported. They may be supported in the future.
 
-- MySQL 5.7 generated columns are not supported. They may be supported in the future.
-
-- MySQL 5.7 `POINT` column type is not supported.
-
 - MySQL 5.7 `JSON` columns are supported but not as part of `PRIMARY KEY`
 
 - The two _before_ & _after_ tables must share a `PRIMARY KEY` or other `UNIQUE KEY`. This key will be used by `gh-ost` to iterate through the table rows when copying. [Read more](shared-key.md)
@@ -53,4 +49,4 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th
 
 - Migrating a `FEDERATED` table is unsupported and is irrelevant to the problem `gh-ost` tackles.
 
-- `ALTER TABLE ... RENAME TO some_other_name` is not supported (and you shouldn't use `gh-ost` for such a trivial operation).
+- `ALTER TABLE ... RENAME TO some_other_name` is not supported (and you shouldn't use `gh-ost` for such a trivial operation).

go/base/context.go

@@ -186,8 +186,10 @@ type MigrationContext struct {
 
 	OriginalTableColumnsOnApplier    *sql.ColumnList
 	OriginalTableColumns             *sql.ColumnList
+	OriginalTableVirtualColumns      *sql.ColumnList
 	OriginalTableUniqueKeys          [](*sql.UniqueKey)
 	GhostTableColumns                *sql.ColumnList
+	GhostTableVirtualColumns         *sql.ColumnList
 	GhostTableUniqueKeys             [](*sql.UniqueKey)
 	UniqueKey                        *sql.UniqueKey
 	SharedColumns                    *sql.ColumnList

go/logic/applier.go

@@ -117,7 +117,7 @@ func (this *Applier) validateAndReadTimeZone() error {
 // readTableColumns reads table columns on applier
 func (this *Applier) readTableColumns() (err error) {
 	log.Infof("Examining table structure on applier")
-	this.migrationContext.OriginalTableColumnsOnApplier, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName)
+	this.migrationContext.OriginalTableColumnsOnApplier, _, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName)
 	if err != nil {
 		return err
 	}

go/logic/inspect.go

@@ -89,24 +89,24 @@ func (this *Inspector) ValidateOriginalTable() (err error) {
 	return nil
 }
 
-func (this *Inspector) InspectTableColumnsAndUniqueKeys(tableName string) (columns *sql.ColumnList, uniqueKeys [](*sql.UniqueKey), err error) {
+func (this *Inspector) InspectTableColumnsAndUniqueKeys(tableName string) (columns *sql.ColumnList, virtualColumns *sql.ColumnList, uniqueKeys [](*sql.UniqueKey), err error) {
 	uniqueKeys, err = this.getCandidateUniqueKeys(tableName)
 	if err != nil {
-		return columns, uniqueKeys, err
+		return columns, virtualColumns, uniqueKeys, err
 	}
 	if len(uniqueKeys) == 0 {
-		return columns, uniqueKeys, fmt.Errorf("No PRIMARY nor UNIQUE key found in table! Bailing out")
+		return columns, virtualColumns, uniqueKeys, fmt.Errorf("No PRIMARY nor UNIQUE key found in table! Bailing out")
 	}
-	columns, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, tableName)
+	columns, virtualColumns, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, tableName)
 	if err != nil {
-		return columns, uniqueKeys, err
+		return columns, virtualColumns, uniqueKeys, err
 	}
 
-	return columns, uniqueKeys, nil
+	return columns, virtualColumns, uniqueKeys, nil
 }
 
 func (this *Inspector) InspectOriginalTable() (err error) {
-	this.migrationContext.OriginalTableColumns, this.migrationContext.OriginalTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.OriginalTableName)
+	this.migrationContext.OriginalTableColumns, this.migrationContext.OriginalTableVirtualColumns, this.migrationContext.OriginalTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.OriginalTableName)
 	if err != nil {
 		return err
 	}
@@ -122,7 +122,7 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) {
 		return fmt.Errorf("It seems like table structure is not identical between master and replica. This scenario is not supported.")
 	}
 
-	this.migrationContext.GhostTableColumns, this.migrationContext.GhostTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.GetGhostTableName())
+	this.migrationContext.GhostTableColumns, this.migrationContext.GhostTableVirtualColumns, this.migrationContext.GhostTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.GetGhostTableName())
 	if err != nil {
 		return err
 	}
@@ -166,7 +166,7 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) {
 		}
 	}
 
-	this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns = this.getSharedColumns(this.migrationContext.OriginalTableColumns, this.migrationContext.GhostTableColumns, this.migrationContext.ColumnRenameMap)
+	this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns = this.getSharedColumns(this.migrationContext.OriginalTableColumns, this.migrationContext.GhostTableColumns, this.migrationContext.OriginalTableVirtualColumns, this.migrationContext.GhostTableVirtualColumns, this.migrationContext.ColumnRenameMap)
 	log.Infof("Shared columns are %s", this.migrationContext.SharedColumns)
 	// By fact that a non-empty unique key exists we also know the shared columns are non-empty
 
@@ -692,7 +692,7 @@ func (this *Inspector) getSharedUniqueKeys(originalUniqueKeys, ghostUniqueKeys [
 }
 
 // getSharedColumns returns the intersection of two lists of columns in same order as the first list
-func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.ColumnList, columnRenameMap map[string]string) (*sql.ColumnList, *sql.ColumnList) {
+func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.ColumnList, originalVirtualColumns, ghostVirtualColumns *sql.ColumnList, columnRenameMap map[string]string) (*sql.ColumnList, *sql.ColumnList) {
 	sharedColumnNames := []string{}
 	for _, originalColumn := range originalColumns.Names() {
 		isSharedColumn := false
@@ -709,6 +709,16 @@ func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.Colum
 				isSharedColumn = false
 			}
 		}
+		for _, virtualColumn := range originalVirtualColumns.Names() {
+			if strings.EqualFold(originalColumn, virtualColumn) {
+				isSharedColumn = false
+			}
+		}
+		for _, virtualColumn := range ghostVirtualColumns.Names() {
+			if strings.EqualFold(originalColumn, virtualColumn) {
+				isSharedColumn = false
+			}
+		}
 		if isSharedColumn {
 			sharedColumnNames = append(sharedColumnNames, originalColumn)
 		}

go/mysql/utils.go

@@ -8,6 +8,7 @@ package mysql
 import (
 	gosql "database/sql"
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 
@@ -178,26 +179,32 @@ func GetInstanceKey(db *gosql.DB) (instanceKey *InstanceKey, err error) {
 }
 
 // GetTableColumns reads column list from given table
-func GetTableColumns(db *gosql.DB, databaseName, tableName string) (*sql.ColumnList, error) {
+func GetTableColumns(db *gosql.DB, databaseName, tableName string) (*sql.ColumnList, *sql.ColumnList, error) {
 	query := fmt.Sprintf(`
 		show columns from %s.%s
 		`,
 		sql.EscapeName(databaseName),
 		sql.EscapeName(tableName),
 	)
 	columnNames := []string{}
+	virtualColumnNames := []string{}
 	err := sqlutils.QueryRowsMap(db, query, func(rowMap sqlutils.RowMap) error {
-		columnNames = append(columnNames, rowMap.GetString("Field"))
+		columnName := rowMap.GetString("Field")
+		columnNames = append(columnNames, columnName)
+		if strings.Contains(rowMap.GetString("Extra"), " GENERATED") {
+			log.Debugf("%s is a generated column", columnName)
+			virtualColumnNames = append(virtualColumnNames, columnName)
+		}
 		return nil
 	})
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	if len(columnNames) == 0 {
-		return nil, log.Errorf("Found 0 columns on %s.%s. Bailing out",
+		return nil, nil, log.Errorf("Found 0 columns on %s.%s. Bailing out",
 			sql.EscapeName(databaseName),
 			sql.EscapeName(tableName),
 		)
 	}
-	return sql.NewColumnList(columnNames), nil
+	return sql.NewColumnList(columnNames), sql.NewColumnList(virtualColumnNames), nil
 }

localtests/generated-columns-add57/create.sql

@@ -0,0 +1,29 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  a int not null,
+  b int not null,
+  primary key(id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test (id, a, b) values (null, 2,3);
+  insert into gh_ost_test (id, a, b) values (null, 2,4);
+  insert into gh_ost_test (id, a, b) values (null, 2,5);
+  insert into gh_ost_test (id, a, b) values (null, 2,6);
+  insert into gh_ost_test (id, a, b) values (null, 2,7);
+  insert into gh_ost_test (id, a, b) values (null, 2,8);
+  insert into gh_ost_test (id, a, b) values (null, 2,9);
+  insert into gh_ost_test (id, a, b) values (null, 2,0);
+  insert into gh_ost_test (id, a, b) values (null, 2,1);
+  insert into gh_ost_test (id, a, b) values (null, 2,2);
+end ;;

localtests/generated-columns-add57/extra_args

@@ -0,0 +1 @@
+--alter="add column sum_ab int as (a + b) virtual not null"

localtests/generated-columns-add57/ghost_columns

@@ -0,0 +1 @@
+id, a, b

localtests/generated-columns-add57/ignore_versions

@@ -0,0 +1 @@
+(5.5|5.6)

localtests/generated-columns-add57/order_by

@@ -0,0 +1 @@
+id

localtests/generated-columns-add57/orig_columns

@@ -0,0 +1 @@
+id, a, b

localtests/generated-columns-rename57/create.sql

@@ -0,0 +1,30 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  a int not null,
+  b int not null,
+  sum_ab int as (a + b) virtual not null,
+  primary key(id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test (id, a, b) values (null, 2,3);
+  insert into gh_ost_test (id, a, b) values (null, 2,4);
+  insert into gh_ost_test (id, a, b) values (null, 2,5);
+  insert into gh_ost_test (id, a, b) values (null, 2,6);
+  insert into gh_ost_test (id, a, b) values (null, 2,7);
+  insert into gh_ost_test (id, a, b) values (null, 2,8);
+  insert into gh_ost_test (id, a, b) values (null, 2,9);
+  insert into gh_ost_test (id, a, b) values (null, 2,0);
+  insert into gh_ost_test (id, a, b) values (null, 2,1);
+  insert into gh_ost_test (id, a, b) values (null, 2,2);
+end ;;

localtests/generated-columns-rename57/extra_args

@@ -0,0 +1 @@
+--alter="change sum_ab total_ab int as (a + b) virtual not null" --approve-renamed-columns

localtests/generated-columns-rename57/ignore_versions

@@ -0,0 +1 @@
+(5.5|5.6)

localtests/generated-columns57/create.sql

@@ -0,0 +1,30 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  a int not null,
+  b int not null,
+  sum_ab int as (a + b) virtual not null,
+  primary key(id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test (id, a, b) values (null, 2,3);
+  insert into gh_ost_test (id, a, b) values (null, 2,4);
+  insert into gh_ost_test (id, a, b) values (null, 2,5);
+  insert into gh_ost_test (id, a, b) values (null, 2,6);
+  insert into gh_ost_test (id, a, b) values (null, 2,7);
+  insert into gh_ost_test (id, a, b) values (null, 2,8);
+  insert into gh_ost_test (id, a, b) values (null, 2,9);
+  insert into gh_ost_test (id, a, b) values (null, 2,0);
+  insert into gh_ost_test (id, a, b) values (null, 2,1);
+  insert into gh_ost_test (id, a, b) values (null, 2,2);
+end ;;

localtests/generated-columns57/ignore_versions

@@ -0,0 +1 @@
+(5.5|5.6)

localtests/json57dml/create.sql

@@ -19,7 +19,7 @@ create event gh_ost_test
 begin
   insert into gh_ost_test (id, i, j) values (null, 11, '"sometext"');
   insert into gh_ost_test (id, i, j) values (null, 13, '{"key":"val"}');
-    insert into gh_ost_test (id, i, j) values (null, 17, '{"is-it": true, "count": 3, "elements": []}');
+  insert into gh_ost_test (id, i, j) values (null, 17, '{"is-it": true, "count": 3, "elements": []}');
   insert into gh_ost_test (id, i, j) values (null, 19, '{"text":"Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum semper nisi. Aenean vulputate eleifend tellus. Aenean leo ligula, porttitor eu, consequat vitae, eleifend ac, enim. Aliquam lorem ante, dapibus in, viverra quis, feugiat a, tellus. Phasellus viverra nulla ut metus varius laoreet. Quisque rutrum. Aenean imperdiet. Etiam ultricies nisi vel augue. Curabitur ullamcorper ultricies nisi. Nam eget dui. Etiam rhoncus. Maecenas tempus, tellus eget condimentum rhoncus, sem quam semper libero, sit amet adipiscing sem neque sed ipsum. Nam quam nunc, blandit vel, luctus pulvinar, hendrerit id, lorem. Maecenas nec odio et ante tincidunt tempus. Donec vitae sapien ut libero venenatis faucibus. Nullam quis ante. Etiam sit amet orci eget eros faucibus tincidunt. Duis leo. Sed fringilla mauris sit amet nibh. Donec sodales sagittis magna. Sed consequat, leo eget bibendum sodales, augue velit cursus nunc, quis gravida magna mi a libero. Fusce vulputate eleifend sapien. Vestibulum purus quam, scelerisque ut, mollis sed, nonummy id, metus. Nullam accumsan lorem in dui. Cras ultricies mi eu turpis hendrerit fringilla. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; In ac dui quis mi consectetuer lacinia. Nam pretium turpis et arcu. Duis arcu tortor, suscipit eget, imperdiet nec, imperdiet iaculis, ipsum. Sed aliquam ultrices mauris. Integer ante arcu, accumsan a, consectetuer eget, posuere ut, mauris. Praesent adipiscing. Phasellus ullamcorper ipsum rutrum nunc. Nunc nonummy metus. Vestibulum volutpat pretium libero. Cras id dui. Aenean ut eros et nisl sagittis vestibulum. Nullam nulla eros, ultricies sit amet, nonummy id, imperdiet feugiat, pede. Sed lectus. Donec mollis hendrerit risus. Phasellus nec sem in justo pellentesque facilisis. Etiam imperdiet imperdiet orci. Nunc nec neque. Phasellus leo dolor, tempus non, auctor et, hendrerit quis, nisi. Curabitur ligula sapien, tincidunt non, euismod vitae, posuere imperdiet, leo. Maecenas malesuada. Praesent congue erat at massa. Sed cursus turpis vitae tortor. Donec posuere vulputate arcu. Phasellus accumsan cursus velit. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Sed aliquam, nisi quis porttitor congue, elit erat euismod orci, ac"}');
 
   update gh_ost_test set j = '{"updated": 11}', updated = 1 where i = 11 and updated = 0;