diff --git a/go/test/endtoend/vtgate/queries/reference/main_test.go b/go/test/endtoend/vtgate/queries/reference/main_test.go new file mode 100644 index 00000000000..4c9440ca4ff --- /dev/null +++ b/go/test/endtoend/vtgate/queries/reference/main_test.go @@ -0,0 +1,283 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reference + +import ( + "context" + "flag" + "fmt" + "os" + "testing" + "time" + + "vitess.io/vitess/go/mysql" + + querypb "vitess.io/vitess/go/vt/proto/query" + "vitess.io/vitess/go/vt/vtgate/vtgateconn" + + "vitess.io/vitess/go/test/endtoend/cluster" +) + +var ( + clusterInstance *cluster.LocalProcessCluster + cell = "zone1" + hostname = "localhost" + vtParams mysql.ConnParams + + unshardedKeyspaceName = "uks" + unshardedSQLSchema = ` + CREATE TABLE IF NOT EXISTS zip( + id BIGINT NOT NULL AUTO_INCREMENT, + code5 INT(5) NOT NULL, + PRIMARY KEY(id) + ) ENGINE=InnoDB; + + INSERT INTO zip(id, code5) + VALUES (1, 47107), + (2, 82845), + (3, 11237); + + CREATE TABLE IF NOT EXISTS zip_detail( + id BIGINT NOT NULL AUTO_INCREMENT, + zip_id BIGINT NOT NULL, + discontinued_at DATE, + PRIMARY KEY(id) + ) ENGINE=InnoDB; + + ` + unshardedVSchema = ` + { + "sharded":false, + "tables": { + "zip": {}, + "zip_detail": {} + } + } + ` + shardedKeyspaceName = "sks" + shardedSQLSchema = ` + CREATE TABLE IF NOT EXISTS delivery_failure ( + id BIGINT NOT NULL, + zip_detail_id BIGINT NOT NULL, + reason VARCHAR(255), + PRIMARY KEY(id) + ) ENGINE=InnoDB; + ` + shardedVSchema = ` + { + "sharded": true, + "vindexes": { + "hash": { + "type": "hash" + } + }, + "tables": { + "delivery_failure": { + "columnVindexes": [ + { + "column": "id", + "name": "hash" + } + ] + }, + "zip_detail": { + "type": "reference", + "source": "` + unshardedKeyspaceName + `.zip_detail" + } + } + } + ` +) + +func TestMain(m *testing.M) { + defer cluster.PanicHandler(nil) + flag.Parse() + + exitCode := func() int { + clusterInstance = cluster.NewCluster(cell, hostname) + defer clusterInstance.Teardown() + + // Start topo server + if err := clusterInstance.StartTopo(); err != nil { + return 1 + } + + // Start keyspace + uKeyspace := &cluster.Keyspace{ + Name: unshardedKeyspaceName, + SchemaSQL: unshardedSQLSchema, + VSchema: unshardedVSchema, + } + if err := clusterInstance.StartUnshardedKeyspace(*uKeyspace, 0, false); err != nil { + return 1 + } + + sKeyspace := &cluster.Keyspace{ + Name: shardedKeyspaceName, + SchemaSQL: shardedSQLSchema, + VSchema: shardedVSchema, + } + if err := clusterInstance.StartKeyspace(*sKeyspace, []string{"-80", "80-"}, 0, false); err != nil { + return 1 + } + + // Start vtgate + if err := clusterInstance.StartVtgate(); err != nil { + return 1 + } + + if err := clusterInstance.WaitForTabletsToHealthyInVtgate(); err != nil { + return 1 + } + + vtParams = mysql.ConnParams{ + Host: "localhost", + Port: clusterInstance.VtgateMySQLPort, + } + + // TODO(maxeng) remove when we have a proper way to check + // materialization lag and cutover. + done := make(chan bool, 1) + expectRows := 2 + go func() { + ctx := context.Background() + vtgateAddr := fmt.Sprintf("%s:%d", clusterInstance.Hostname, clusterInstance.VtgateProcess.GrpcPort) + vtgateConn, err := vtgateconn.Dial(ctx, vtgateAddr) + if err != nil { + done <- false + return + } + defer vtgateConn.Close() + + maxWait := time.After(300 * time.Second) + for _, ks := range clusterInstance.Keyspaces { + if ks.Name != shardedKeyspaceName { + continue + } + for _, s := range ks.Shards { + var ok bool + for !ok { + select { + case <-maxWait: + fmt.Println("Waited too long for materialization, cancelling.") + done <- false + return + default: + } + shard := fmt.Sprintf("%s/%s@primary", ks.Name, s.Name) + session := vtgateConn.Session(shard, nil) + _, err := session.Execute(ctx, "SHOW CREATE TABLE zip_detail", map[string]*querypb.BindVariable{}) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to SHOW CREATE TABLE zip_detail; might not exist yet: %v\n", err) + time.Sleep(1 * time.Second) + continue + } + qr, err := session.Execute(ctx, "SELECT * FROM zip_detail", map[string]*querypb.BindVariable{}) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to query sharded keyspace for zip_detail rows: %v\n", err) + done <- false + return + } + if len(qr.Rows) != expectRows { + fmt.Fprintf(os.Stderr, "Shard %s doesn't yet have expected number of zip_detail rows\n", shard) + time.Sleep(10 * time.Second) + continue + } + fmt.Fprintf(os.Stdout, "Shard %s has expected number of zip_detail rows.\n", shard) + ok = true + } + } + fmt.Println("All shards have expected number of zip_detail rows.") + done <- true + } + }() + + // Materialize zip_detail to sharded keyspace. + output, err := clusterInstance.VtctlProcess.ExecuteCommandWithOutput( + "Materialize", + "--", + "--tablet_types", + "PRIMARY", + `{ + "workflow": "copy_zip_detail", + "source_keyspace": "`+unshardedKeyspaceName+`", + "target_keyspace": "`+shardedKeyspaceName+`", + "tablet_types": "PRIMARY", + "table_settings": [ + { + "target_table": "zip_detail", + "source_expression": "select * from zip_detail", + "create_ddl": "copy" + } + ] + }`, + ) + fmt.Fprintf(os.Stderr, "Output from materialize: %s\n", output) + if err != nil { + fmt.Fprintf(os.Stderr, "Got error trying to start materialize zip_detail: %v\n", err) + return 1 + } + + ctx := context.Background() + vtgateAddr := fmt.Sprintf("%s:%d", clusterInstance.Hostname, clusterInstance.VtgateProcess.GrpcPort) + vtgateConn, err := vtgateconn.Dial(ctx, vtgateAddr) + if err != nil { + return 1 + } + defer vtgateConn.Close() + + session := vtgateConn.Session("@primary", nil) + // INSERT some zip_detail rows. + if _, err := session.Execute(ctx, ` + INSERT INTO zip_detail(id, zip_id, discontinued_at) + VALUES (1, 1, '2022-05-13'), + (2, 2, '2022-08-15') + `, map[string]*querypb.BindVariable{}); err != nil { + return 1 + } + + // INSERT some delivery_failure rows. + if _, err := session.Execute(ctx, ` + INSERT INTO delivery_failure(id, zip_detail_id, reason) + VALUES (1, 1, 'Failed delivery due to discontinued zipcode.'), + (2, 2, 'Failed delivery due to discontinued zipcode.'), + (3, 3, 'Failed delivery due to unknown reason.'); + `, map[string]*querypb.BindVariable{}); err != nil { + return 1 + } + + if ok := <-done; !ok { + fmt.Fprintf(os.Stderr, "Materialize did not succeed.\n") + return 1 + } + + // Stop materialize zip_detail to sharded keyspace. + err = clusterInstance.VtctlProcess.ExecuteCommand( + "Workflow", + "--", + shardedKeyspaceName+".copy_zip_detail", + "delete", + ) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to stop materialization workflow: %v", err) + return 1 + } + + return m.Run() + }() + os.Exit(exitCode) +} diff --git a/go/test/endtoend/vtgate/queries/reference/reference_test.go b/go/test/endtoend/vtgate/queries/reference/reference_test.go new file mode 100644 index 00000000000..75efc840880 --- /dev/null +++ b/go/test/endtoend/vtgate/queries/reference/reference_test.go @@ -0,0 +1,139 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reference + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/test/endtoend/utils" + + "vitess.io/vitess/go/test/endtoend/cluster" +) + +func start(t *testing.T) (*mysql.Conn, func()) { + ctx := context.Background() + vtConn, err := mysql.Connect(ctx, &vtParams) + require.NoError(t, err) + + return vtConn, func() { + vtConn.Close() + cluster.PanicHandler(t) + } +} + +// TestGlobalReferenceRouting tests that unqualified queries for reference +// tables go to the right place. +// +// Given: +// - Unsharded keyspace `uks` and sharded keyspace `sks`. +// - Source table `uks.zip_detail` and a reference table `sks.zip_detail`, +// initially with the same rows. +// - Unsharded table `uks.zip` and sharded table `sks.delivery_failure`. +// +// When: we execute `INSERT INTO zip_detail ...`, +// Then: `zip_detail` should be routed to `uks`. +// +// When: we execute `UPDATE zip_detail ...`, +// Then: `zip_detail` should be routed to `uks`. +// +// When: we execute `SELECT ... FROM zip JOIN zip_detail ...`, +// Then: `zip_detail` should be routed to `uks`. +// +// When: we execute `SELECT ... FROM delivery_failure JOIN zip_detail ...`, +// Then: `zip_detail` should be routed to `sks`. +// +// When: we execute `DELETE FROM zip_detail ...`, +// Then: `zip_detail` should be routed to `uks`. +func TestReferenceRouting(t *testing.T) { + conn, closer := start(t) + defer closer() + + // INSERT should route an unqualified zip_detail to unsharded keyspace. + utils.Exec(t, conn, "INSERT INTO zip_detail(id, zip_id, discontinued_at) VALUES(3, 1, DATE('2022-12-03'))") + // Verify with qualified zip_detail queries to each keyspace. The unsharded + // keyspace should have an extra row. + utils.AssertMatches( + t, + conn, + "SELECT COUNT(zd.id) FROM "+unshardedKeyspaceName+".zip_detail zd WHERE id = 3", + `[[INT64(1)]]`, + ) + utils.AssertMatches( + t, + conn, + "SELECT COUNT(zd.id) FROM "+shardedKeyspaceName+".zip_detail zd WHERE id = 3", + `[[INT64(0)]]`, + ) + + // UPDATE should route an unqualified zip_detail to unsharded keyspace. + utils.Exec(t, conn, + "UPDATE zip_detail SET discontinued_at = NULL WHERE id = 2") + // Verify with qualified zip_detail queries to each keyspace. The unsharded + // keyspace should have a matching row, but not the sharded keyspace. + utils.AssertMatches( + t, + conn, + "SELECT COUNT(id) FROM "+unshardedKeyspaceName+".zip_detail WHERE discontinued_at IS NULL", + `[[INT64(1)]]`, + ) + utils.AssertMatches( + t, + conn, + "SELECT COUNT(id) FROM "+shardedKeyspaceName+".zip_detail WHERE discontinued_at IS NULL", + `[[INT64(0)]]`, + ) + + // SELECT a table in unsharded keyspace and JOIN unqualified zip_detail. + utils.AssertMatches( + t, + conn, + "SELECT COUNT(zd.id) FROM zip z JOIN zip_detail zd ON z.id = zd.zip_id WHERE zd.id = 3", + `[[INT64(1)]]`, + ) + + // SELECT a table in sharded keyspace and JOIN unqualified zip_detail. + // Use gen4 planner to avoid errors from gen3 planner. + utils.AssertMatches( + t, + conn, + `SELECT /*vt+ PLANNER=gen4 */ COUNT(zd.id) + FROM delivery_failure df + JOIN zip_detail zd ON zd.id = df.zip_detail_id WHERE zd.id = 3`, + `[[INT64(0)]]`, + ) + + // DELETE should route an unqualified zip_detail to unsharded keyspace. + utils.Exec(t, conn, "DELETE FROM zip_detail") + // Verify with qualified zip_detail queries to each keyspace. The unsharded + // keyspace should not have any rows; the sharded keyspace should. + utils.AssertMatches( + t, + conn, + "SELECT COUNT(id) FROM "+unshardedKeyspaceName+".zip_detail", + `[[INT64(0)]]`, + ) + utils.AssertMatches( + t, + conn, + "SELECT COUNT(id) FROM "+shardedKeyspaceName+".zip_detail", + `[[INT64(2)]]`, + ) +} diff --git a/go/vt/proto/vschema/vschema.pb.go b/go/vt/proto/vschema/vschema.pb.go index 33177f204ff..e86c1613682 100644 --- a/go/vt/proto/vschema/vschema.pb.go +++ b/go/vt/proto/vschema/vschema.pb.go @@ -324,6 +324,8 @@ type Table struct { // an authoritative list for the table. This allows // us to expand 'select *' expressions. ColumnListAuthoritative bool `protobuf:"varint,6,opt,name=column_list_authoritative,json=columnListAuthoritative,proto3" json:"column_list_authoritative,omitempty"` + // reference tables may optionally indicate their source table. + Source string `protobuf:"bytes,7,opt,name=source,proto3" json:"source,omitempty"` } func (x *Table) Reset() { @@ -400,6 +402,13 @@ func (x *Table) GetColumnListAuthoritative() bool { return false } +func (x *Table) GetSource() string { + if x != nil { + return x.Source + } + return "" +} + // ColumnVindex is used to associate a column to a vindex. type ColumnVindex struct { state protoimpl.MessageState @@ -804,7 +813,7 @@ var file_vschema_proto_rawDesc = []byte{ 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, - 0x99, 0x02, 0x0a, 0x05, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, + 0xb1, 0x02, 0x0a, 0x05, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x3e, 0x0a, 0x0f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x5f, 0x76, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x76, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, @@ -821,54 +830,55 @@ var file_vschema_proto_rawDesc = []byte{ 0x3a, 0x0a, 0x19, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x61, 0x74, 0x69, 0x76, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x17, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x75, - 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x61, 0x74, 0x69, 0x76, 0x65, 0x22, 0x54, 0x0a, 0x0c, 0x43, - 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x56, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x16, 0x0a, 0x06, 0x63, - 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6c, - 0x75, 0x6d, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, - 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, - 0x73, 0x22, 0x43, 0x0a, 0x0d, 0x41, 0x75, 0x74, 0x6f, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x65, - 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x65, - 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x22, 0x3d, 0x0a, 0x06, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, - 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x0b, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, - 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0xa7, 0x02, 0x0a, 0x0a, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x12, 0x40, 0x0a, 0x09, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x76, 0x73, 0x63, 0x68, 0x65, 0x6d, - 0x61, 0x2e, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x4b, 0x65, 0x79, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x09, 0x6b, 0x65, 0x79, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x3a, 0x0a, 0x0d, 0x72, 0x6f, 0x75, 0x74, 0x69, 0x6e, - 0x67, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, - 0x76, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, - 0x75, 0x6c, 0x65, 0x73, 0x52, 0x0c, 0x72, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, - 0x65, 0x73, 0x12, 0x4a, 0x0a, 0x13, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x72, 0x6f, 0x75, 0x74, - 0x69, 0x6e, 0x67, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x76, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, - 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x52, 0x11, 0x73, 0x68, 0x61, - 0x72, 0x64, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x1a, 0x4f, - 0x0a, 0x0e, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x27, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x11, 0x2e, 0x76, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x4b, 0x65, 0x79, 0x73, - 0x70, 0x61, 0x63, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, - 0x44, 0x0a, 0x11, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, - 0x75, 0x6c, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x05, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x76, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x52, 0x05, - 0x72, 0x75, 0x6c, 0x65, 0x73, 0x22, 0x6e, 0x0a, 0x10, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x6f, - 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x72, 0x6f, - 0x6d, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0c, 0x66, 0x72, 0x6f, 0x6d, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1f, - 0x0a, 0x0b, 0x74, 0x6f, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x6f, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, - 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x73, 0x68, 0x61, 0x72, 0x64, 0x42, 0x26, 0x5a, 0x24, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2e, - 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x74, 0x2f, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x61, 0x74, 0x69, 0x76, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x22, 0x54, 0x0a, 0x0c, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x56, 0x69, 0x6e, + 0x64, 0x65, 0x78, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, + 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x22, 0x43, 0x0a, 0x0d, 0x41, 0x75, 0x74, + 0x6f, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, + 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6c, 0x75, + 0x6d, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x22, 0x3d, + 0x0a, 0x06, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x04, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0b, 0x2e, 0x71, 0x75, 0x65, + 0x72, 0x79, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0xa7, 0x02, + 0x0a, 0x0a, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x40, 0x0a, 0x09, + 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x22, 0x2e, 0x76, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x53, 0x72, 0x76, 0x56, 0x53, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x09, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x3a, + 0x0a, 0x0d, 0x72, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x76, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, + 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x52, 0x0c, 0x72, 0x6f, + 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x4a, 0x0a, 0x13, 0x73, 0x68, + 0x61, 0x72, 0x64, 0x5f, 0x72, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x72, 0x75, 0x6c, 0x65, + 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x76, 0x73, 0x63, 0x68, 0x65, 0x6d, + 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x75, + 0x6c, 0x65, 0x73, 0x52, 0x11, 0x73, 0x68, 0x61, 0x72, 0x64, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, + 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x1a, 0x4f, 0x0a, 0x0e, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x76, 0x73, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x2e, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x44, 0x0a, 0x11, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x05, + 0x72, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x76, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x6f, 0x75, 0x74, 0x69, + 0x6e, 0x67, 0x52, 0x75, 0x6c, 0x65, 0x52, 0x05, 0x72, 0x75, 0x6c, 0x65, 0x73, 0x22, 0x6e, 0x0a, + 0x10, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x6f, 0x75, 0x74, 0x69, 0x6e, 0x67, 0x52, 0x75, 0x6c, + 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x72, 0x6f, 0x6d, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x66, 0x72, 0x6f, 0x6d, 0x4b, 0x65, + 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x6f, 0x5f, 0x6b, 0x65, 0x79, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x6f, 0x4b, + 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x42, 0x26, 0x5a, + 0x24, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, 0x65, 0x73, + 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/go/vt/proto/vschema/vschema_vtproto.pb.go b/go/vt/proto/vschema/vschema_vtproto.pb.go index 9aa144453ec..fe96380be2f 100644 --- a/go/vt/proto/vschema/vschema_vtproto.pb.go +++ b/go/vt/proto/vschema/vschema_vtproto.pb.go @@ -306,6 +306,13 @@ func (m *Table) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.Source) > 0 { + i -= len(m.Source) + copy(dAtA[i:], m.Source) + i = encodeVarint(dAtA, i, uint64(len(m.Source))) + i-- + dAtA[i] = 0x3a + } if m.ColumnListAuthoritative { i-- if m.ColumnListAuthoritative { @@ -845,6 +852,10 @@ func (m *Table) SizeVT() (n int) { if m.ColumnListAuthoritative { n += 2 } + l = len(m.Source) + if l > 0 { + n += 1 + l + sov(uint64(l)) + } if m.unknownFields != nil { n += len(m.unknownFields) } @@ -2005,6 +2016,38 @@ func (m *Table) UnmarshalVT(dAtA []byte) error { } } m.ColumnListAuthoritative = bool(v != 0) + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Source", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Source = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skip(dAtA[iNdEx:]) diff --git a/go/vt/vtgate/engine/plan_description_test.go b/go/vt/vtgate/engine/plan_description_test.go index 6170970419a..0d985b9b606 100644 --- a/go/vt/vtgate/engine/plan_description_test.go +++ b/go/vt/vtgate/engine/plan_description_test.go @@ -39,7 +39,7 @@ func TestCreateRoutePlanDescription(t *testing.T) { TargetDestination: key.DestinationAllShards{}, Other: map[string]any{ "Query": route.Query, - "Table": route.TableName, + "Table": route.GetTableName(), "FieldQuery": route.FieldQuery, "Vindex": route.Vindex.String(), }, @@ -97,7 +97,7 @@ func getDescriptionFor(route *Route) PrimitiveDescription { TargetDestination: key.DestinationAllShards{}, Other: map[string]any{ "Query": route.Query, - "Table": route.TableName, + "Table": route.GetTableName(), "FieldQuery": route.FieldQuery, "Vindex": route.Vindex.String(), }, diff --git a/go/vt/vtgate/engine/route.go b/go/vt/vtgate/engine/route.go index 3d4ef704d65..a001b1d55f6 100644 --- a/go/vt/vtgate/engine/route.go +++ b/go/vt/vtgate/engine/route.go @@ -51,7 +51,7 @@ type Route struct { // Query specifies the query to be executed. Query string - // TableName specifies the table to send the query to. + // TableName specifies the tables to send the query to. TableName string // FieldQuery specifies the query to be executed for a GetFieldInfo request. @@ -441,7 +441,7 @@ func (route *Route) sort(in *sqltypes.Result) (*sqltypes.Result, error) { func (route *Route) description() PrimitiveDescription { other := map[string]any{ "Query": route.Query, - "Table": route.TableName, + "Table": route.GetTableName(), "FieldQuery": route.FieldQuery, } if route.Vindex != nil { diff --git a/go/vt/vtgate/executor_dml_test.go b/go/vt/vtgate/executor_dml_test.go index 6e51b3be77b..7f396338fb9 100644 --- a/go/vt/vtgate/executor_dml_test.go +++ b/go/vt/vtgate/executor_dml_test.go @@ -2344,6 +2344,45 @@ func TestUpdateLastInsertID(t *testing.T) { assertQueries(t, sbc1, wantQueries) } +func TestUpdateReference(t *testing.T) { + executor, sbc1, sbc2, sbclookup := createExecutorEnv() + + logChan := QueryLogger.Subscribe("Test") + defer QueryLogger.Unsubscribe(logChan) + + _, err := executorExec(executor, "update zip_detail set status = 'CLOSED' where id = 1", nil) + require.NoError(t, err) + wantQueries := []*querypb.BoundQuery{{ + Sql: "update zip_detail set `status` = 'CLOSED' where id = 1", + BindVariables: map[string]*querypb.BindVariable{}, + }} + assertQueries(t, sbc1, nil) + assertQueries(t, sbc2, nil) + assertQueries(t, sbclookup, wantQueries) + + testQueryLog(t, logChan, "TestExecute", "UPDATE", "update zip_detail set status = 'CLOSED' where id = 1", 1) + + sbclookup.Queries = nil + + _, err = executorExec(executor, "update TestUnsharded.zip_detail set status = 'CLOSED' where id = 1", nil) + require.NoError(t, err) + wantQueries = []*querypb.BoundQuery{{ + Sql: "update zip_detail set `status` = 'CLOSED' where id = 1", + BindVariables: map[string]*querypb.BindVariable{}, + }} + assertQueries(t, sbc1, nil) + assertQueries(t, sbc2, nil) + assertQueries(t, sbclookup, wantQueries) + + testQueryLog(t, logChan, "TestExecute", "UPDATE", + "update TestUnsharded.zip_detail set status = 'CLOSED' where id = 1", 1) + + sbclookup.Queries = nil + + _, err = executorExec(executor, "update TestExecutor.zip_detail set status = 'CLOSED' where id = 1", nil) + require.Error(t, err) +} + func TestDeleteLookupOwnedEqual(t *testing.T) { executor, sbc1, sbc2, _ := createExecutorEnv() @@ -2371,6 +2410,44 @@ func TestDeleteLookupOwnedEqual(t *testing.T) { assertQueries(t, sbc2, sbc2wantQueries) } +func TestDeleteReference(t *testing.T) { + executor, sbc1, sbc2, sbclookup := createExecutorEnv() + + logChan := QueryLogger.Subscribe("Test") + defer QueryLogger.Unsubscribe(logChan) + + _, err := executorExec(executor, "delete from zip_detail where id = 1", nil) + require.NoError(t, err) + wantQueries := []*querypb.BoundQuery{{ + Sql: "delete from zip_detail where id = 1", + BindVariables: map[string]*querypb.BindVariable{}, + }} + assertQueries(t, sbc1, nil) + assertQueries(t, sbc2, nil) + assertQueries(t, sbclookup, wantQueries) + + testQueryLog(t, logChan, "TestExecute", "DELETE", "delete from zip_detail where id = 1", 1) + + sbclookup.Queries = nil + + _, err = executorExec(executor, "delete from zip_detail where id = 1", nil) + require.NoError(t, err) + wantQueries = []*querypb.BoundQuery{{ + Sql: "delete from zip_detail where id = 1", + BindVariables: map[string]*querypb.BindVariable{}, + }} + assertQueries(t, sbc1, nil) + assertQueries(t, sbc2, nil) + assertQueries(t, sbclookup, wantQueries) + + testQueryLog(t, logChan, "TestExecute", "DELETE", "delete from zip_detail where id = 1", 1) + + sbclookup.Queries = nil + + _, err = executorExec(executor, "delete from TestExecutor.zip_detail where id = 1", nil) + require.Error(t, err) +} + func TestReservedConnDML(t *testing.T) { executor, _, _, sbc := createExecutorEnv() @@ -2761,3 +2838,42 @@ func TestInsertSelectFromTable(t *testing.T) { testQueryLog(t, logChan, "TestInsertSelect", "INSERT", "insert into user(id, name) select c1, c2 from music", 9) // 8 from select and 1 from insert. } } + +func TestInsertReference(t *testing.T) { + executor, sbc1, sbc2, sbclookup := createExecutorEnv() + + logChan := QueryLogger.Subscribe("Test") + defer QueryLogger.Unsubscribe(logChan) + + _, err := executorExec(executor, "insert into zip_detail(id, status) values (1, 'CLOSED')", nil) + require.NoError(t, err) + wantQueries := []*querypb.BoundQuery{{ + Sql: "insert into zip_detail(id, `status`) values (1, 'CLOSED')", + BindVariables: map[string]*querypb.BindVariable{}, + }} + assertQueries(t, sbc1, nil) + assertQueries(t, sbc2, nil) + assertQueries(t, sbclookup, wantQueries) + + testQueryLog(t, logChan, "TestExecute", "INSERT", "insert into zip_detail(id, status) values (1, 'CLOSED')", 1) + + sbclookup.Queries = nil + + _, err = executorExec(executor, "insert into TestUnsharded.zip_detail(id, status) values (1, 'CLOSED')", nil) + require.NoError(t, err) + wantQueries = []*querypb.BoundQuery{{ + Sql: "insert into zip_detail(id, `status`) values (1, 'CLOSED')", + BindVariables: map[string]*querypb.BindVariable{}, + }} + assertQueries(t, sbc1, nil) + assertQueries(t, sbc2, nil) + assertQueries(t, sbclookup, wantQueries) + + testQueryLog(t, logChan, "TestExecute", "INSERT", + "insert into TestUnsharded.zip_detail(id, status) values (1, 'CLOSED')", 1) + + sbclookup.Queries = nil + + _, err = executorExec(executor, "insert into TestExecutor.zip_detail(id, status) values (1, 'CLOSED')", nil) + require.Error(t, err) +} diff --git a/go/vt/vtgate/executor_framework_test.go b/go/vt/vtgate/executor_framework_test.go index ad10fbadbe2..6c5acbc72cf 100644 --- a/go/vt/vtgate/executor_framework_test.go +++ b/go/vt/vtgate/executor_framework_test.go @@ -390,7 +390,11 @@ var executorVSchema = ` "type": "VARCHAR" } ] - } + }, + "zip_detail": { + "type": "reference", + "source": "TestUnsharded.zip_detail" + } } } ` @@ -428,7 +432,8 @@ var unshardedVSchema = ` "nrl_lu_idx": {}, "nv_lu_idx": {}, "lu_idx": {}, - "simple": {} + "simple": {}, + "zip_detail": {} } } ` diff --git a/go/vt/vtgate/executor_scatter_stats.go b/go/vt/vtgate/executor_scatter_stats.go index 56daa37819f..946558e22fd 100644 --- a/go/vt/vtgate/executor_scatter_stats.go +++ b/go/vt/vtgate/executor_scatter_stats.go @@ -109,7 +109,7 @@ func (e *Executor) gatherScatterStats() (statsResults, error) { PercentTimeOfScatters: 100 * float64(execTime) / float64(scatterExecTime), PercentCountOfReads: 100 * float64(execCount) / float64(readOnlyCount), PercentCountOfScatters: 100 * float64(execCount) / float64(scatterCount), - From: route.Keyspace.Name + "." + route.TableName, + From: route.Keyspace.Name + "." + route.GetTableName(), Count: execCount, } } diff --git a/go/vt/vtgate/executor_select_test.go b/go/vt/vtgate/executor_select_test.go index bcdc7b3a29e..4dd36d9a416 100644 --- a/go/vt/vtgate/executor_select_test.go +++ b/go/vt/vtgate/executor_select_test.go @@ -3331,6 +3331,112 @@ func TestGen4MultiColMultiEqual(t *testing.T) { utils.MustMatch(t, wantQueries, sbc2.Queries) } +func TestGen4SelectUnqualifiedReferenceTable(t *testing.T) { + executor, sbc1, sbc2, sbclookup := createExecutorEnv() + executor.pv = querypb.ExecuteOptions_Gen4 + + query := "select * from zip_detail" + _, err := executorExec(executor, query, nil) + require.NoError(t, err) + wantQueries := []*querypb.BoundQuery{ + { + Sql: query, + BindVariables: map[string]*querypb.BindVariable{}, + }, + } + utils.MustMatch(t, wantQueries, sbclookup.Queries) + require.Nil(t, sbc1.Queries) + require.Nil(t, sbc2.Queries) +} + +func TestGen4SelectQualifiedReferenceTable(t *testing.T) { + executor, sbc1, sbc2, sbclookup := createExecutorEnv() + executor.pv = querypb.ExecuteOptions_Gen4 + + query := fmt.Sprintf("select * from %s.zip_detail", KsTestSharded) + _, err := executorExec(executor, query, nil) + require.NoError(t, err) + wantQueries := []*querypb.BoundQuery{ + { + Sql: "select * from zip_detail", + BindVariables: map[string]*querypb.BindVariable{}, + }, + } + require.Nil(t, sbclookup.Queries) + utils.MustMatch(t, wantQueries, sbc1.Queries) + require.Nil(t, sbc2.Queries) +} + +func TestGen4JoinUnqualifiedReferenceTable(t *testing.T) { + executor, sbc1, sbc2, sbclookup := createExecutorEnv() + executor.pv = querypb.ExecuteOptions_Gen4 + + query := "select * from user join zip_detail on user.zip_detail_id = zip_detail.id" + _, err := executorExec(executor, query, nil) + require.NoError(t, err) + wantQueries := []*querypb.BoundQuery{ + { + Sql: "select * from `user`, zip_detail where `user`.zip_detail_id = zip_detail.id", + BindVariables: map[string]*querypb.BindVariable{}, + }, + } + require.Nil(t, sbclookup.Queries) + utils.MustMatch(t, wantQueries, sbc1.Queries) + utils.MustMatch(t, wantQueries, sbc2.Queries) + + sbc1.Queries = nil + sbc2.Queries = nil + + query = "select * from simple join zip_detail on simple.zip_detail_id = zip_detail.id" + _, err = executorExec(executor, query, nil) + require.NoError(t, err) + wantQueries = []*querypb.BoundQuery{ + { + Sql: "select * from `simple` join zip_detail on `simple`.zip_detail_id = zip_detail.id", + BindVariables: map[string]*querypb.BindVariable{}, + }, + } + utils.MustMatch(t, wantQueries, sbclookup.Queries) + require.Nil(t, sbc1.Queries) + require.Nil(t, sbc2.Queries) +} + +func TestGen4CrossShardJoinQualifiedReferenceTable(t *testing.T) { + executor, sbc1, sbc2, sbclookup := createExecutorEnv() + executor.pv = querypb.ExecuteOptions_Gen4 + + query := "select user.id from user join TestUnsharded.zip_detail on user.zip_detail_id = TestUnsharded.zip_detail.id" + _, err := executorExec(executor, query, nil) + require.NoError(t, err) + + shardedWantQueries := []*querypb.BoundQuery{ + { + Sql: "select `user`.id from `user`, zip_detail where `user`.zip_detail_id = zip_detail.id", + BindVariables: map[string]*querypb.BindVariable{}, + }, + } + require.Nil(t, sbclookup.Queries) + utils.MustMatch(t, shardedWantQueries, sbc1.Queries) + utils.MustMatch(t, shardedWantQueries, sbc2.Queries) + + sbclookup.Queries = nil + sbc1.Queries = nil + sbc2.Queries = nil + + query = "select simple.id from simple join TestExecutor.zip_detail on simple.zip_detail_id = TestExecutor.zip_detail.id" + _, err = executorExec(executor, query, nil) + require.NoError(t, err) + unshardedWantQueries := []*querypb.BoundQuery{ + { + Sql: "select `simple`.id from `simple`, zip_detail where `simple`.zip_detail_id = zip_detail.id", + BindVariables: map[string]*querypb.BindVariable{}, + }, + } + utils.MustMatch(t, unshardedWantQueries, sbclookup.Queries) + require.Nil(t, sbc1.Queries) + require.Nil(t, sbc2.Queries) +} + func TestRegionRange(t *testing.T) { // Special setup: Don't use createExecutorEnv. diff --git a/go/vt/vtgate/executor_test.go b/go/vt/vtgate/executor_test.go index 4deedb6ba8d..17377b252bc 100644 --- a/go/vt/vtgate/executor_test.go +++ b/go/vt/vtgate/executor_test.go @@ -1030,6 +1030,7 @@ func TestExecutorShow(t *testing.T) { buildVarCharRow("user_msgs"), buildVarCharRow("user_seq"), buildVarCharRow("wo_lu_idx"), + buildVarCharRow("zip_detail"), }, } utils.MustMatch(t, wantqr, qr, query) diff --git a/go/vt/vtgate/planbuilder/builder.go b/go/vt/vtgate/planbuilder/builder.go index e0645a972de..098bc20706f 100644 --- a/go/vt/vtgate/planbuilder/builder.go +++ b/go/vt/vtgate/planbuilder/builder.go @@ -83,7 +83,7 @@ func tablesFromSemantics(semTable *semantics.SemTable) []string { if vindexTable == nil { continue } - tables[vindexTable.ToString()] = nil + tables[vindexTable.String()] = nil } names := make([]string, 0, len(tables)) diff --git a/go/vt/vtgate/planbuilder/from.go b/go/vt/vtgate/planbuilder/from.go index 18d653778a7..b24026a2bd6 100644 --- a/go/vt/vtgate/planbuilder/from.go +++ b/go/vt/vtgate/planbuilder/from.go @@ -25,6 +25,7 @@ import ( "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/vtgate/evalengine" + querypb "vitess.io/vitess/go/vt/proto/query" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/vterrors" @@ -230,6 +231,14 @@ func (pb *primitiveBuilder) buildTablePrimitive(tableExpr *sqlparser.AliasedTabl return nil } + sourceTable, err := pb.tryRedirectGen4InsertToSource(vschemaTable) + if err != nil { + return err + } + if sourceTable != nil { + vschemaTable = sourceTable + } + rb, st := newRoute(sel) pb.plan, pb.st = rb, st if err := st.AddVSchemaTable(alias, vschemaTable, rb); err != nil { @@ -310,6 +319,26 @@ func (pb *primitiveBuilder) processJoin(ajoin *sqlparser.JoinTableExpr, reserved return pb.join(rpb, ajoin, reservedVars, where) } +// If the primitiveBuilder context is a Gen4 planner, the statement is an +// INSERT, and the vschema table is a reference with a valid source reference, +// then redirect the INSERT back to the source. +func (pb *primitiveBuilder) tryRedirectGen4InsertToSource(vschemaTable *vindexes.Table) (*vindexes.Table, error) { + if pb.stmt == nil { + return nil, nil + } + if _, ok := pb.stmt.(*sqlparser.Insert); !ok { + return nil, nil + } + if pb.vschema.Planner() == querypb.ExecuteOptions_V3 { + return nil, nil + } + if vschemaTable.Type != vindexes.TypeReference || vschemaTable.Source == nil { + return nil, nil + } + vschemaTable, _, _, _, _, err := pb.vschema.FindTableOrVindex(vschemaTable.Source.TableName) + return vschemaTable, err +} + // convertToLeftJoin converts a right join into a left join. func convertToLeftJoin(ajoin *sqlparser.JoinTableExpr) { newRHS := ajoin.LeftExpr diff --git a/go/vt/vtgate/planbuilder/gen4_planner.go b/go/vt/vtgate/planbuilder/gen4_planner.go index be9d29f60fe..92a3c78956d 100644 --- a/go/vt/vtgate/planbuilder/gen4_planner.go +++ b/go/vt/vtgate/planbuilder/gen4_planner.go @@ -88,26 +88,26 @@ func gen4SelectStmtPlanner( sel.SQLCalcFoundRows = false } - getPlan := func(selStatement sqlparser.SelectStatement) (logicalPlan, *semantics.SemTable, error) { + getPlan := func(selStatement sqlparser.SelectStatement) (logicalPlan, *semantics.SemTable, []string, error) { return newBuildSelectPlan(selStatement, reservedVars, vschema, plannerVersion) } - plan, st, err := getPlan(stmt) + plan, _, tablesUsed, err := getPlan(stmt) if err != nil { return nil, err } if shouldRetryAfterPredicateRewriting(plan) { // by transforming the predicates to CNF, the planner will sometimes find better plans - primitive, st := gen4PredicateRewrite(stmt, getPlan) - if primitive != nil { - return newPlanResult(primitive, tablesFromSemantics(st)...), nil + plan2, _, tablesUsed := gen4PredicateRewrite(stmt, getPlan) + if plan2 != nil { + return newPlanResult(plan2.Primitive(), tablesUsed...), nil } } primitive := plan.Primitive() if !isSel { - return newPlanResult(primitive, tablesFromSemantics(st)...), nil + return newPlanResult(primitive, tablesUsed...), nil } // this is done because engine.Route doesn't handle the empty result well @@ -122,7 +122,7 @@ func gen4SelectStmtPlanner( prim.SendTo.NoRoutesSpecialHandling = true } } - return newPlanResult(primitive, tablesFromSemantics(st)...), nil + return newPlanResult(primitive, tablesUsed...), nil } func gen4planSQLCalcFoundRows(vschema plancontext.VSchema, sel *sqlparser.Select, query string, reservedVars *sqlparser.ReservedVars) (*planResult, error) { @@ -137,33 +137,33 @@ func gen4planSQLCalcFoundRows(vschema plancontext.VSchema, sel *sqlparser.Select // record any warning as planner warning. vschema.PlannerWarning(semTable.Warning) - plan, err := buildSQLCalcFoundRowsPlan(query, sel, reservedVars, vschema, planSelectGen4) + plan, tablesUsed, err := buildSQLCalcFoundRowsPlan(query, sel, reservedVars, vschema, planSelectGen4) if err != nil { return nil, err } - return newPlanResult(plan.Primitive(), tablesFromSemantics(semTable)...), nil + return newPlanResult(plan.Primitive(), tablesUsed...), nil } -func planSelectGen4(reservedVars *sqlparser.ReservedVars, vschema plancontext.VSchema, sel *sqlparser.Select) (*jointab, logicalPlan, error) { - plan, _, err := newBuildSelectPlan(sel, reservedVars, vschema, 0) +func planSelectGen4(reservedVars *sqlparser.ReservedVars, vschema plancontext.VSchema, sel *sqlparser.Select) (*jointab, logicalPlan, []string, error) { + plan, _, tablesUsed, err := newBuildSelectPlan(sel, reservedVars, vschema, 0) if err != nil { - return nil, nil, err + return nil, nil, nil, err } - return nil, plan, nil + return nil, plan, tablesUsed, nil } -func gen4PredicateRewrite(stmt sqlparser.Statement, getPlan func(selStatement sqlparser.SelectStatement) (logicalPlan, *semantics.SemTable, error)) (engine.Primitive, *semantics.SemTable) { +func gen4PredicateRewrite(stmt sqlparser.Statement, getPlan func(selStatement sqlparser.SelectStatement) (logicalPlan, *semantics.SemTable, []string, error)) (logicalPlan, *semantics.SemTable, []string) { rewritten, isSel := sqlparser.RewritePredicate(stmt).(sqlparser.SelectStatement) if !isSel { // Fail-safe code, should never happen - return nil, nil + return nil, nil, nil } - plan2, st, err := getPlan(rewritten) + plan2, st, op, err := getPlan(rewritten) if err == nil && !shouldRetryAfterPredicateRewriting(plan2) { // we only use this new plan if it's better than the old one we got - return plan2.Primitive(), st + return plan2, st, op } - return nil, nil + return nil, nil, nil } func newBuildSelectPlan( @@ -171,14 +171,14 @@ func newBuildSelectPlan( reservedVars *sqlparser.ReservedVars, vschema plancontext.VSchema, version querypb.ExecuteOptions_PlannerVersion, -) (plan logicalPlan, semTable *semantics.SemTable, err error) { +) (plan logicalPlan, semTable *semantics.SemTable, tablesUsed []string, err error) { ksName := "" if ks, _ := vschema.DefaultKeyspace(); ks != nil { ksName = ks.Name } semTable, err = semantics.Analyze(selStmt, ksName, vschema) if err != nil { - return nil, nil, err + return nil, nil, nil, err } // record any warning as planner warning. vschema.PlannerWarning(semTable.Warning) @@ -186,35 +186,35 @@ func newBuildSelectPlan( ctx := plancontext.NewPlanningContext(reservedVars, semTable, vschema, version) if ks, _ := semTable.SingleUnshardedKeyspace(); ks != nil { - plan, err = unshardedShortcut(ctx, selStmt, ks) + plan, tablesUsed, err = unshardedShortcut(ctx, selStmt, ks) if err != nil { - return nil, nil, err + return nil, nil, nil, err } plan, err = pushCommentDirectivesOnPlan(plan, selStmt) if err != nil { - return nil, nil, err + return nil, nil, nil, err } - return plan, semTable, err + return plan, semTable, tablesUsed, err } // From this point on, we know it is not an unsharded query and return the NotUnshardedErr if there is any if semTable.NotUnshardedErr != nil { - return nil, nil, semTable.NotUnshardedErr + return nil, nil, nil, semTable.NotUnshardedErr } err = queryRewrite(semTable, reservedVars, selStmt) if err != nil { - return nil, nil, err + return nil, nil, nil, err } op, err := operators.PlanQuery(ctx, selStmt) if err != nil { - return nil, nil, err + return nil, nil, nil, err } plan, err = transformToLogicalPlan(ctx, op, true) if err != nil { - return nil, nil, err + return nil, nil, nil, err } plan = optimizePlan(plan) @@ -222,20 +222,20 @@ func newBuildSelectPlan( sel, isSel := selStmt.(*sqlparser.Select) if isSel { if err = setMiscFunc(plan, sel); err != nil { - return nil, nil, err + return nil, nil, nil, err } } if err = plan.WireupGen4(ctx); err != nil { - return nil, nil, err + return nil, nil, nil, err } plan, err = pushCommentDirectivesOnPlan(plan, selStmt) if err != nil { - return nil, nil, err + return nil, nil, nil, err } - return plan, semTable, nil + return plan, semTable, operators.TablesUsed(op), nil } // optimizePlan removes unnecessary simpleProjections that have been created while planning @@ -293,7 +293,7 @@ func gen4UpdateStmtPlanner( edml.Opcode = engine.Unsharded edml.Query = generateQuery(updStmt) upd := &engine.Update{DML: edml} - return newPlanResult(upd, tablesFromSemantics(semTable)...), nil + return newPlanResult(upd, operators.QualifiedTables(ks, tables)...), nil } if semTable.NotUnshardedErr != nil { @@ -328,7 +328,7 @@ func gen4UpdateStmtPlanner( return nil, err } - return newPlanResult(plan.Primitive(), tablesFromSemantics(semTable)...), nil + return newPlanResult(plan.Primitive(), operators.TablesUsed(op)...), nil } func gen4DeleteStmtPlanner( @@ -372,7 +372,7 @@ func gen4DeleteStmtPlanner( edml.Opcode = engine.Unsharded edml.Query = generateQuery(deleteStmt) del := &engine.Delete{DML: edml} - return newPlanResult(del, tablesFromSemantics(semTable)...), nil + return newPlanResult(del, operators.QualifiedTables(ks, tables)...), nil } if err := checkIfDeleteSupported(deleteStmt, semTable); err != nil { @@ -406,7 +406,7 @@ func gen4DeleteStmtPlanner( return nil, err } - return newPlanResult(plan.Primitive(), tablesFromSemantics(semTable)...), nil + return newPlanResult(plan.Primitive(), operators.TablesUsed(op)...), nil } func rewriteRoutedTables(stmt sqlparser.Statement, vschema plancontext.VSchema) (err error) { diff --git a/go/vt/vtgate/planbuilder/insert.go b/go/vt/vtgate/planbuilder/insert.go index 7a0ab6dcf8c..23153760551 100644 --- a/go/vt/vtgate/planbuilder/insert.go +++ b/go/vt/vtgate/planbuilder/insert.go @@ -36,8 +36,8 @@ import ( // buildInsertPlan builds the route for an INSERT statement. func buildInsertPlan(stmt sqlparser.Statement, reservedVars *sqlparser.ReservedVars, vschema plancontext.VSchema) (*planResult, error) { + pb := newStmtAwarePrimitiveBuilder(vschema, newJointab(reservedVars), stmt) ins := stmt.(*sqlparser.Insert) - pb := newPrimitiveBuilder(vschema, newJointab(reservedVars)) exprs := sqlparser.TableExprs{&sqlparser.AliasedTableExpr{Expr: ins.Table}} rb, err := pb.processDMLTable(exprs, reservedVars, nil) if err != nil { diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index c0b0b43e850..0d973ba674b 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -153,13 +153,7 @@ func transformApplyJoinPlan(ctx *plancontext.PlanningContext, n *operators.Apply }, nil } -func transformRoutePlan(ctx *plancontext.PlanningContext, op *operators.Route) (logicalPlan, error) { - switch src := op.Source.(type) { - case *operators.Update: - return transformUpdatePlan(ctx, op, src) - case *operators.Delete: - return transformDeletePlan(ctx, op, src) - } +func routeToEngineRoute(ctx *plancontext.PlanningContext, op *operators.Route) (*engine.Route, error) { tableNames, err := getAllTableNames(op) if err != nil { return nil, err @@ -170,24 +164,38 @@ func transformRoutePlan(ctx *plancontext.PlanningContext, op *operators.Route) ( vindex = op.Selected.FoundVindex values = op.Selected.Values } + return &engine.Route{ + TableName: strings.Join(tableNames, ", "), + RoutingParameters: &engine.RoutingParameters{ + Opcode: op.RouteOpCode, + Keyspace: op.Keyspace, + Vindex: vindex, + Values: values, + SysTableTableName: op.SysTableTableName, + SysTableTableSchema: op.SysTableTableSchema, + }, + }, nil +} + +func transformRoutePlan(ctx *plancontext.PlanningContext, op *operators.Route) (logicalPlan, error) { + switch src := op.Source.(type) { + case *operators.Update: + return transformUpdatePlan(ctx, op, src) + case *operators.Delete: + return transformDeletePlan(ctx, op, src) + } condition := getVindexPredicate(ctx, op) sel, err := operators.ToSQL(ctx, op.Source) if err != nil { return nil, err } replaceSubQuery(ctx, sel) + eroute, err := routeToEngineRoute(ctx, op) + if err != nil { + return nil, err + } return &routeGen4{ - eroute: &engine.Route{ - TableName: strings.Join(tableNames, ", "), - RoutingParameters: &engine.RoutingParameters{ - Opcode: op.RouteOpCode, - Keyspace: op.Keyspace, - Vindex: vindex, - Values: values, - SysTableTableName: op.SysTableTableName, - SysTableTableSchema: op.SysTableTableSchema, - }, - }, + eroute: eroute, Select: sel, tables: operators.TableID(op), condition: condition, diff --git a/go/vt/vtgate/planbuilder/operators/delete.go b/go/vt/vtgate/planbuilder/operators/delete.go index ab5448b07fb..d33acd8b013 100644 --- a/go/vt/vtgate/planbuilder/operators/delete.go +++ b/go/vt/vtgate/planbuilder/operators/delete.go @@ -53,3 +53,10 @@ func (d *Delete) Clone(inputs []ops.Operator) ops.Operator { AST: d.AST, } } + +func (d *Delete) TablesUsed() []string { + if d.VTable != nil { + return SingleQualifiedIdentifier(d.VTable.Keyspace, d.VTable.Name) + } + return nil +} diff --git a/go/vt/vtgate/planbuilder/operators/helpers.go b/go/vt/vtgate/planbuilder/operators/helpers.go index ea1c9923ad7..8b63c664d06 100644 --- a/go/vt/vtgate/planbuilder/operators/helpers.go +++ b/go/vt/vtgate/planbuilder/operators/helpers.go @@ -17,11 +17,15 @@ limitations under the License. package operators import ( + "fmt" + "sort" + "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtgate/planbuilder/operators/ops" "vitess.io/vitess/go/vt/vtgate/planbuilder/operators/rewrite" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" "vitess.io/vitess/go/vt/vtgate/semantics" + "vitess.io/vitess/go/vt/vtgate/vindexes" ) // Compact will optimise the operator tree into a smaller but equivalent version @@ -78,6 +82,24 @@ func TableID(op ops.Operator) (result semantics.TableSet) { return } +// TableUser is used to signal that this operator directly interacts with one or more tables +type TableUser interface { + TablesUsed() []string +} + +func TablesUsed(op ops.Operator) []string { + addString, collect := collectSortedUniqueStrings() + _ = rewrite.Visit(op, func(this ops.Operator) error { + if tbl, ok := this.(TableUser); ok { + for _, u := range tbl.TablesUsed() { + addString(u) + } + } + return nil + }) + return collect() +} + func UnresolvedPredicates(op ops.Operator, st *semantics.SemTable) (result []sqlparser.Expr) { type unresolved interface { // UnsolvedPredicates returns any predicates that have dependencies on the given Operator and @@ -117,3 +139,68 @@ func CostOf(op ops.Operator) (cost int) { }) return } + +func QualifiedIdentifier(ks *vindexes.Keyspace, i sqlparser.IdentifierCS) string { + return QualifiedString(ks, i.String()) +} + +func QualifiedString(ks *vindexes.Keyspace, s string) string { + return fmt.Sprintf("%s.%s", ks.Name, s) +} + +func QualifiedStrings(ks *vindexes.Keyspace, ss []string) []string { + add, collect := collectSortedUniqueStrings() + for _, s := range ss { + add(QualifiedString(ks, s)) + } + return collect() +} + +func QualifiedTableName(ks *vindexes.Keyspace, t sqlparser.TableName) string { + return QualifiedIdentifier(ks, t.Name) +} + +func QualifiedTableNames(ks *vindexes.Keyspace, ts []sqlparser.TableName) []string { + add, collect := collectSortedUniqueStrings() + for _, t := range ts { + add(QualifiedTableName(ks, t)) + } + return collect() +} + +func QualifiedTables(ks *vindexes.Keyspace, vts []*vindexes.Table) []string { + add, collect := collectSortedUniqueStrings() + for _, vt := range vts { + add(QualifiedIdentifier(ks, vt.Name)) + } + return collect() +} + +func SingleQualifiedIdentifier(ks *vindexes.Keyspace, i sqlparser.IdentifierCS) []string { + return SingleQualifiedString(ks, i.String()) +} + +func SingleQualifiedString(ks *vindexes.Keyspace, s string) []string { + return []string{QualifiedString(ks, s)} +} + +func SingleQualifiedTableName(ks *vindexes.Keyspace, t sqlparser.TableName) []string { + return SingleQualifiedIdentifier(ks, t.Name) +} + +func collectSortedUniqueStrings() (add func(string), collect func() []string) { + uniq := make(map[string]any) + add = func(v string) { + uniq[v] = nil + } + collect = func() []string { + sorted := make([]string, 0, len(uniq)) + for v := range uniq { + sorted = append(sorted, v) + } + sort.Strings(sorted) + return sorted + } + + return add, collect +} diff --git a/go/vt/vtgate/planbuilder/operators/querygraph.go b/go/vt/vtgate/planbuilder/operators/querygraph.go index 5bd24ba23c2..b22fdf6907e 100644 --- a/go/vt/vtgate/planbuilder/operators/querygraph.go +++ b/go/vt/vtgate/planbuilder/operators/querygraph.go @@ -204,3 +204,14 @@ func (qg *QueryGraph) AddPredicate(ctx *plancontext.PlanningContext, expr sqlpar } return qg, nil } + +// Clone implements the Operator interface +func (qt *QueryTable) Clone() *QueryTable { + return &QueryTable{ + ID: qt.ID, + Alias: sqlparser.CloneRefOfAliasedTableExpr(qt.Alias), + Table: sqlparser.CloneTableName(qt.Table), + Predicates: qt.Predicates, + IsInfSchema: qt.IsInfSchema, + } +} diff --git a/go/vt/vtgate/planbuilder/operators/route.go b/go/vt/vtgate/planbuilder/operators/route.go index 1b2b4192c2e..2d68241027c 100644 --- a/go/vt/vtgate/planbuilder/operators/route.go +++ b/go/vt/vtgate/planbuilder/operators/route.go @@ -55,6 +55,13 @@ type ( // TargetDestination specifies an explicit target destination tablet type TargetDestination key.Destination + + // Alternates contains alternate routes to equivalent sources in + // other keyspaces. + Alternates map[*vindexes.Keyspace]*Route + + // Routes that have been merged into this one. + MergedWith []*Route } // VindexPlusPredicates is a struct used to store all the predicates that the vindex can be used to query @@ -717,35 +724,72 @@ func (r *Route) planIsExpr(ctx *plancontext.PlanningContext, node *sqlparser.IsE return r.haveMatchingVindex(ctx, node, vdValue, column, val, opcodeF, justTheVindex) } -func createRoute(ctx *plancontext.PlanningContext, table *QueryTable, solves semantics.TableSet) (*Route, error) { - if table.IsInfSchema { - return createInfSchemaRoute(ctx, table) +// createRoute returns either an information_schema route, or else consults the +// VSchema to find a suitable table, and then creates a route from that. +func createRoute( + ctx *plancontext.PlanningContext, + queryTable *QueryTable, + solves semantics.TableSet, +) (*Route, error) { + if queryTable.IsInfSchema { + return createInfSchemaRoute(ctx, queryTable) } - vschemaTable, _, _, _, target, err := ctx.VSchema.FindTableOrVindex(table.Table) + return findVSchemaTableAndCreateRoute(ctx, queryTable, queryTable.Table, solves, true /*planAlternates*/) +} + +// findVSchemaTableAndCreateRoute consults the VSchema to find a suitable +// table, and then creates a route from that. +func findVSchemaTableAndCreateRoute( + ctx *plancontext.PlanningContext, + queryTable *QueryTable, + tableName sqlparser.TableName, + solves semantics.TableSet, + planAlternates bool, +) (*Route, error) { + vschemaTable, _, _, _, target, err := ctx.VSchema.FindTableOrVindex(tableName) if target != nil { return nil, vterrors.Errorf(vtrpcpb.Code_UNIMPLEMENTED, "unsupported: SELECT with a target destination") } if err != nil { return nil, err } - if vschemaTable.Name.String() != table.Table.Name.String() { + + return createRouteFromVSchemaTable( + ctx, + queryTable, + vschemaTable, + solves, + planAlternates, + ) +} + +// createRouteFromTable creates a route from the given VSchema table. +func createRouteFromVSchemaTable( + ctx *plancontext.PlanningContext, + queryTable *QueryTable, + vschemaTable *vindexes.Table, + solves semantics.TableSet, + planAlternates bool, +) (*Route, error) { + if vschemaTable.Name.String() != queryTable.Table.Name.String() { // we are dealing with a routed table - name := table.Table.Name - table.Table.Name = vschemaTable.Name - astTable, ok := table.Alias.Expr.(sqlparser.TableName) + queryTable = queryTable.Clone() + name := queryTable.Table.Name + queryTable.Table.Name = vschemaTable.Name + astTable, ok := queryTable.Alias.Expr.(sqlparser.TableName) if !ok { return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] a derived table should never be a routed table") } realTableName := sqlparser.NewIdentifierCS(vschemaTable.Name.String()) astTable.Name = realTableName - if table.Alias.As.IsEmpty() { + if queryTable.Alias.As.IsEmpty() { // if the user hasn't specified an alias, we'll insert one here so the old table name still works - table.Alias.As = sqlparser.NewIdentifierCS(name.String()) + queryTable.Alias.As = sqlparser.NewIdentifierCS(name.String()) } } plan := &Route{ Source: &Table{ - QTable: table, + QTable: queryTable, VTable: vschemaTable, }, Keyspace: vschemaTable.Keyspace, @@ -782,24 +826,24 @@ func createRoute(ctx *plancontext.PlanningContext, table *QueryTable, solves sem default: plan.RouteOpCode = engine.Scatter } - for _, predicate := range table.Predicates { - err = plan.UpdateRoutingLogic(ctx, predicate) + for _, predicate := range queryTable.Predicates { + err := plan.UpdateRoutingLogic(ctx, predicate) if err != nil { return nil, err } } - if plan.RouteOpCode == engine.Scatter && len(table.Predicates) > 0 { + if plan.RouteOpCode == engine.Scatter && len(queryTable.Predicates) > 0 { // If we have a scatter query, it's worth spending a little extra time seeing if we can't improve it - oldPredicates := table.Predicates - table.Predicates = nil + oldPredicates := queryTable.Predicates + queryTable.Predicates = nil plan.SeenPredicates = nil for _, pred := range oldPredicates { rewritten := sqlparser.RewritePredicate(pred) predicates := sqlparser.SplitAndExpression(nil, rewritten.(sqlparser.Expr)) for _, predicate := range predicates { - table.Predicates = append(table.Predicates, predicate) - err = plan.UpdateRoutingLogic(ctx, predicate) + queryTable.Predicates = append(queryTable.Predicates, predicate) + err := plan.UpdateRoutingLogic(ctx, predicate) if err != nil { return nil, err } @@ -808,13 +852,13 @@ func createRoute(ctx *plancontext.PlanningContext, table *QueryTable, solves sem if plan.RouteOpCode == engine.Scatter { // if we _still_ haven't found a better route, we can run this additional rewrite on any ORs we have - for _, expr := range table.Predicates { + for _, expr := range queryTable.Predicates { or, ok := expr.(*sqlparser.OrExpr) if !ok { continue } for _, predicate := range sqlparser.ExtractINFromOR(or) { - err = plan.UpdateRoutingLogic(ctx, predicate) + err := plan.UpdateRoutingLogic(ctx, predicate) if err != nil { return nil, err } @@ -823,9 +867,67 @@ func createRoute(ctx *plancontext.PlanningContext, table *QueryTable, solves sem } } + if planAlternates { + alternates, err := createAlternateRoutesFromVSchemaTable( + ctx, + queryTable, + vschemaTable, + solves, + ) + if err != nil { + return nil, err + } + plan.Alternates = alternates + } + return plan, nil } +func createAlternateRoutesFromVSchemaTable( + ctx *plancontext.PlanningContext, + queryTable *QueryTable, + vschemaTable *vindexes.Table, + solves semantics.TableSet, +) (map[*vindexes.Keyspace]*Route, error) { + routes := make(map[*vindexes.Keyspace]*Route) + + switch vschemaTable.Type { + case "", vindexes.TypeReference: + for ksName, referenceTable := range vschemaTable.ReferencedBy { + route, err := findVSchemaTableAndCreateRoute( + ctx, + queryTable, + sqlparser.TableName{ + Name: referenceTable.Name, + Qualifier: sqlparser.NewIdentifierCS(ksName), + }, + solves, + false, /*planAlternates*/ + ) + if err != nil { + return nil, err + } + routes[route.Keyspace] = route + } + + if vschemaTable.Source != nil { + route, err := findVSchemaTableAndCreateRoute( + ctx, + queryTable, + vschemaTable.Source.TableName, + solves, + false, /*planAlternates*/ + ) + if err != nil { + return nil, err + } + routes[route.Keyspace] = route + } + } + + return routes, nil +} + func (r *Route) AddPredicate(ctx *plancontext.PlanningContext, expr sqlparser.Expr) (ops.Operator, error) { err := r.UpdateRoutingLogic(ctx, expr) if err != nil { @@ -842,3 +944,27 @@ func (r *Route) AddPredicate(ctx *plancontext.PlanningContext, expr sqlparser.Ex func (r *Route) AddColumn(ctx *plancontext.PlanningContext, e sqlparser.Expr) (int, error) { return r.Source.AddColumn(ctx, e) } + +func (r *Route) AlternateInKeyspace(keyspace *vindexes.Keyspace) *Route { + if keyspace.Name == r.Keyspace.Name { + return nil + } + + if route, ok := r.Alternates[keyspace]; ok { + return route + } + + return nil +} + +// TablesUsed returns tables used by MergedWith routes, which are not included +// in Inputs() and thus not a part of the operator tree +func (r *Route) TablesUsed() []string { + addString, collect := collectSortedUniqueStrings() + for _, mw := range r.MergedWith { + for _, u := range TablesUsed(mw) { + addString(u) + } + } + return collect() +} diff --git a/go/vt/vtgate/planbuilder/operators/route_planning.go b/go/vt/vtgate/planbuilder/operators/route_planning.go index 05cbcb3f6f0..2f2de04c555 100644 --- a/go/vt/vtgate/planbuilder/operators/route_planning.go +++ b/go/vt/vtgate/planbuilder/operators/route_planning.go @@ -148,6 +148,14 @@ func buildVindexTableForDML(ctx *plancontext.PlanningContext, tableInfo semantic opCode = engine.Scatter } + if vindexTable.Source != nil { + sourceTable, _, _, _, _, err := ctx.VSchema.FindTableOrVindex(vindexTable.Source.TableName) + if err != nil { + return nil, 0, nil, err + } + vindexTable = sourceTable + } + var dest key.Destination var typ topodatapb.TabletType var err error @@ -463,6 +471,7 @@ func createRouteOperatorForJoin(ctx *plancontext.PlanningContext, aRoute, bRoute SeenPredicates: append(aRoute.SeenPredicates, bRoute.SeenPredicates...), SysTableTableName: sysTableName, Source: join, + MergedWith: []*Route{bRoute}, } if aRoute.SelectedVindex() == bRoute.SelectedVindex() { @@ -499,6 +508,16 @@ func tryMerge( sameKeyspace := aRoute.Keyspace == bRoute.Keyspace + if !sameKeyspace { + if altARoute := aRoute.AlternateInKeyspace(bRoute.Keyspace); altARoute != nil { + aRoute = altARoute + sameKeyspace = true + } else if altBRoute := bRoute.AlternateInKeyspace(aRoute.Keyspace); altBRoute != nil { + bRoute = altBRoute + sameKeyspace = true + } + } + if sameKeyspace || (isDualTable(aRoute) || isDualTable(bRoute)) { tree, err := tryMergeReferenceTable(aRoute, bRoute, merger) if tree != nil || err != nil { diff --git a/go/vt/vtgate/planbuilder/operators/subquery_planning.go b/go/vt/vtgate/planbuilder/operators/subquery_planning.go index 32cf2cac613..9b17699106b 100644 --- a/go/vt/vtgate/planbuilder/operators/subquery_planning.go +++ b/go/vt/vtgate/planbuilder/operators/subquery_planning.go @@ -142,6 +142,8 @@ func mergeSubQueryOp(ctx *plancontext.PlanningContext, outer *Route, inner *Rout } } + outer.MergedWith = append(outer.MergedWith, inner) + return outer, nil } @@ -243,6 +245,9 @@ func tryMergeSubqueryWithRoute( // Special case: Inner query won't return any results / is not routable. if subqueryRoute.RouteOpCode == engine.None { merged, err := merger(outerOp, subqueryRoute) + if err != nil { + return nil, err + } return merged, err } diff --git a/go/vt/vtgate/planbuilder/operators/table.go b/go/vt/vtgate/planbuilder/operators/table.go index 04a23d4e3e1..dcd5150c2d3 100644 --- a/go/vt/vtgate/planbuilder/operators/table.go +++ b/go/vt/vtgate/planbuilder/operators/table.go @@ -79,6 +79,13 @@ func (to *Table) AddCol(col *sqlparser.ColName) { to.Columns = append(to.Columns, col) } +func (to *Table) TablesUsed() []string { + if sqlparser.SystemSchema(to.QTable.Table.Qualifier.String()) { + return nil + } + return SingleQualifiedIdentifier(to.VTable.Keyspace, to.VTable.Name) +} + func addColumn(op ColNameColumns, e sqlparser.Expr) (int, error) { col, ok := e.(*sqlparser.ColName) if !ok { diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index 6d3c481bc94..11c46a326a4 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -58,3 +58,10 @@ func (u *Update) Clone(inputs []ops.Operator) ops.Operator { AST: u.AST, } } + +func (u *Update) TablesUsed() []string { + if u.VTable != nil { + return SingleQualifiedIdentifier(u.VTable.Keyspace, u.VTable.Name) + } + return nil +} diff --git a/go/vt/vtgate/planbuilder/operators/vindex.go b/go/vt/vtgate/planbuilder/operators/vindex.go index 633ce3123fc..b724727c38c 100644 --- a/go/vt/vtgate/planbuilder/operators/vindex.go +++ b/go/vt/vtgate/planbuilder/operators/vindex.go @@ -129,3 +129,9 @@ func (v *Vindex) AddPredicate(ctx *plancontext.PlanningContext, expr sqlparser.E } return v, nil } + +// TablesUsed implements the Operator interface. +// It is not keyspace-qualified. +func (v *Vindex) TablesUsed() []string { + return []string{v.Table.Table.Name.String()} +} diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index 73575695b54..3a857ce1c73 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -251,6 +251,7 @@ func TestPlan(t *testing.T) { testFile(t, "show_cases_no_default_keyspace.json", testOutputTempDir, vschemaWrapper, false) testFile(t, "stream_cases.json", testOutputTempDir, vschemaWrapper, false) testFile(t, "systemtables_cases.json", testOutputTempDir, vschemaWrapper, false) + testFile(t, "reference_cases.json", testOutputTempDir, vschemaWrapper, false) } func TestSysVarSetDisabled(t *testing.T) { diff --git a/go/vt/vtgate/planbuilder/primitive_builder.go b/go/vt/vtgate/planbuilder/primitive_builder.go index 29655e81e41..b7c557518e5 100644 --- a/go/vt/vtgate/planbuilder/primitive_builder.go +++ b/go/vt/vtgate/planbuilder/primitive_builder.go @@ -17,6 +17,7 @@ limitations under the License. package planbuilder import ( + "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" ) @@ -29,6 +30,15 @@ type primitiveBuilder struct { jt *jointab plan logicalPlan st *symtab + stmt sqlparser.Statement +} + +func newStmtAwarePrimitiveBuilder(vschema plancontext.VSchema, jt *jointab, stmt sqlparser.Statement) *primitiveBuilder { + return &primitiveBuilder{ + vschema: vschema, + jt: jt, + stmt: stmt, + } } func newPrimitiveBuilder(vschema plancontext.VSchema, jt *jointab) *primitiveBuilder { diff --git a/go/vt/vtgate/planbuilder/select.go b/go/vt/vtgate/planbuilder/select.go index 4017662b816..2c3b31f0654 100644 --- a/go/vt/vtgate/planbuilder/select.go +++ b/go/vt/vtgate/planbuilder/select.go @@ -177,7 +177,7 @@ func (pb *primitiveBuilder) processSelect(sel *sqlparser.Select, reservedVars *s } sel.SQLCalcFoundRows = false if sel.Limit != nil { - plan, err := buildSQLCalcFoundRowsPlan(query, sel, reservedVars, pb.vschema, planSelectV3) + plan, _, err := buildSQLCalcFoundRowsPlan(query, sel, reservedVars, pb.vschema, planSelectV3) if err != nil { return err } @@ -283,16 +283,16 @@ func buildSQLCalcFoundRowsPlan( sel *sqlparser.Select, reservedVars *sqlparser.ReservedVars, vschema plancontext.VSchema, - planSelect func(reservedVars *sqlparser.ReservedVars, vschema plancontext.VSchema, sel *sqlparser.Select) (*jointab, logicalPlan, error), -) (logicalPlan, error) { - ljt, limitPlan, err := planSelect(reservedVars, vschema, sel) + planSelect func(reservedVars *sqlparser.ReservedVars, vschema plancontext.VSchema, sel *sqlparser.Select) (*jointab, logicalPlan, []string, error), +) (logicalPlan, []string, error) { + ljt, limitPlan, _, err := planSelect(reservedVars, vschema, sel) if err != nil { - return nil, err + return nil, nil, err } statement2, reserved2, err := sqlparser.Parse2(originalQuery) if err != nil { - return nil, err + return nil, nil, err } sel2 := statement2.(*sqlparser.Select) @@ -325,18 +325,18 @@ func buildSQLCalcFoundRowsPlan( reservedVars2 := sqlparser.NewReservedVars("vtg", reserved2) - cjt, countPlan, err := planSelect(reservedVars2, vschema, sel2) + cjt, countPlan, tablesUsed, err := planSelect(reservedVars2, vschema, sel2) if err != nil { - return nil, err + return nil, nil, err } - return &sqlCalcFoundRows{LimitQuery: limitPlan, CountQuery: countPlan, ljt: ljt, cjt: cjt}, nil + return &sqlCalcFoundRows{LimitQuery: limitPlan, CountQuery: countPlan, ljt: ljt, cjt: cjt}, tablesUsed, nil } -func planSelectV3(reservedVars *sqlparser.ReservedVars, vschema plancontext.VSchema, sel *sqlparser.Select) (*jointab, logicalPlan, error) { +func planSelectV3(reservedVars *sqlparser.ReservedVars, vschema plancontext.VSchema, sel *sqlparser.Select) (*jointab, logicalPlan, []string, error) { ljt := newJointab(reservedVars) frpb := newPrimitiveBuilder(vschema, ljt) err := frpb.processSelect(sel, reservedVars, nil, "") - return ljt, frpb.plan, err + return ljt, frpb.plan, nil, err } func handleDualSelects(sel *sqlparser.Select, vschema plancontext.VSchema) (engine.Primitive, error) { diff --git a/go/vt/vtgate/planbuilder/single_sharded_shortcut.go b/go/vt/vtgate/planbuilder/single_sharded_shortcut.go index 868f96cee89..a8eafc21526 100644 --- a/go/vt/vtgate/planbuilder/single_sharded_shortcut.go +++ b/go/vt/vtgate/planbuilder/single_sharded_shortcut.go @@ -20,6 +20,7 @@ import ( "sort" "strings" + "vitess.io/vitess/go/vt/vtgate/planbuilder/operators" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" "vitess.io/vitess/go/vt/sqlparser" @@ -28,7 +29,7 @@ import ( "vitess.io/vitess/go/vt/vtgate/vindexes" ) -func unshardedShortcut(ctx *plancontext.PlanningContext, stmt sqlparser.SelectStatement, ks *vindexes.Keyspace) (logicalPlan, error) { +func unshardedShortcut(ctx *plancontext.PlanningContext, stmt sqlparser.SelectStatement, ks *vindexes.Keyspace) (logicalPlan, []string, error) { // this method is used when the query we are handling has all tables in the same unsharded keyspace sqlparser.Rewrite(stmt, func(cursor *sqlparser.Cursor) bool { switch node := cursor.Node().(type) { @@ -44,7 +45,7 @@ func unshardedShortcut(ctx *plancontext.PlanningContext, stmt sqlparser.SelectSt tableNames, err := getTableNames(ctx.SemTable) if err != nil { - return nil, err + return nil, nil, err } plan := &routeGen4{ eroute: &engine.Route{ @@ -52,19 +53,27 @@ func unshardedShortcut(ctx *plancontext.PlanningContext, stmt sqlparser.SelectSt Opcode: engine.Unsharded, Keyspace: ks, }, - TableName: strings.Join(tableNames, ", "), + TableName: strings.Join(escapedTableNames(tableNames), ", "), }, Select: stmt, } if err := plan.WireupGen4(ctx); err != nil { - return nil, err + return nil, nil, err } - return plan, nil + return plan, operators.QualifiedTableNames(ks, tableNames), nil } -func getTableNames(semTable *semantics.SemTable) ([]string, error) { - tableNameMap := map[string]any{} +func escapedTableNames(tableNames []sqlparser.TableName) []string { + escaped := make([]string, len(tableNames)) + for i, tableName := range tableNames { + escaped[i] = sqlparser.String(tableName) + } + return escaped +} + +func getTableNames(semTable *semantics.SemTable) ([]sqlparser.TableName, error) { + tableNameMap := make(map[string]sqlparser.TableName) for _, tableInfo := range semTable.Tables { tblObj := tableInfo.GetVindexTable() @@ -72,19 +81,24 @@ func getTableNames(semTable *semantics.SemTable) ([]string, error) { // probably a derived table continue } - var name string if tableInfo.IsInfSchema() { - name = "tableName" + tableNameMap["tableName"] = sqlparser.TableName{ + Name: sqlparser.NewIdentifierCS("tableName"), + } } else { - name = sqlparser.String(tblObj.Name) + tableNameMap[sqlparser.String(tblObj.Name)] = sqlparser.TableName{ + Name: tblObj.Name, + } } - tableNameMap[name] = nil } - - var tableNames []string - for name := range tableNameMap { - tableNames = append(tableNames, name) + var keys []string + for k := range tableNameMap { + keys = append(keys, k) + } + sort.Strings(keys) + var tableNames []sqlparser.TableName + for _, k := range keys { + tableNames = append(tableNames, tableNameMap[k]) } - sort.Strings(tableNames) return tableNames, nil } diff --git a/go/vt/vtgate/planbuilder/testdata/dml_cases.json b/go/vt/vtgate/planbuilder/testdata/dml_cases.json index 8d4886e8c39..e16d3aa8eb8 100644 --- a/go/vt/vtgate/planbuilder/testdata/dml_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/dml_cases.json @@ -6218,5 +6218,28 @@ "user.user" ] } + }, + { + "comment": "insert into ref; TODO(maxeng) is this a bug?", + "query": "insert into ref(col) values(1)", + "plan": { + "Instructions": { + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "MultiShardAutocommit": false, + "OperatorType": "Insert", + "Query": "insert into ref(col) values (1)", + "TableName": "ref", + "TargetTabletType": "PRIMARY", + "Variant": "Sharded" + }, + "Original": "insert into ref(col) values(1)", + "QueryType": "INSERT", + "TablesUsed": [ + "user.ref" + ] + } } ] diff --git a/go/vt/vtgate/planbuilder/testdata/reference_cases.json b/go/vt/vtgate/planbuilder/testdata/reference_cases.json new file mode 100644 index 00000000000..3f155eafc75 --- /dev/null +++ b/go/vt/vtgate/planbuilder/testdata/reference_cases.json @@ -0,0 +1,680 @@ +[ + { + "comment": "select from unqualified ambiguous reference routes to reference source", + "query": "select * from ambiguous_ref_with_source", + "v3-plan": { + "Instructions": { + "FieldQuery": "select * from ambiguous_ref_with_source where 1 != 1", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "OperatorType": "Route", + "Query": "select * from ambiguous_ref_with_source", + "Table": "ambiguous_ref_with_source", + "Variant": "Reference" + }, + "Original": "select * from ambiguous_ref_with_source", + "QueryType": "SELECT" + }, + "gen4-plan": { + "Instructions": { + "FieldQuery": "select * from ambiguous_ref_with_source where 1 != 1", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "OperatorType": "Route", + "Query": "select * from ambiguous_ref_with_source", + "Table": "ambiguous_ref_with_source", + "Variant": "Reference" + }, + "Original": "select * from ambiguous_ref_with_source", + "QueryType": "SELECT", + "TablesUsed": [ + "main.ambiguous_ref_with_source" + ] + } + }, + { + "comment": "join with unqualified ambiguous reference table routes to optimal keyspace", + "query": "select user.col from user join ambiguous_ref_with_source", + "v3-plan": { + "Instructions": { + "Inputs": [ + { + "FieldQuery": "select `user`.col from `user` where 1 != 1", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "OperatorType": "Route", + "Query": "select `user`.col from `user`", + "Table": "`user`", + "Variant": "Scatter" + }, + { + "FieldQuery": "select 1 from ambiguous_ref_with_source where 1 != 1", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "OperatorType": "Route", + "Query": "select 1 from ambiguous_ref_with_source", + "Table": "ambiguous_ref_with_source", + "Variant": "Reference" + } + ], + "JoinColumnIndexes": "L:0", + "OperatorType": "Join", + "TableName": "`user`_ambiguous_ref_with_source", + "Variant": "Join" + }, + "Original": "select user.col from user join ambiguous_ref_with_source", + "QueryType": "SELECT" + }, + "gen4-plan": { + "Instructions": { + "FieldQuery": "select `user`.col from `user`, ambiguous_ref_with_source where 1 != 1", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "OperatorType": "Route", + "Query": "select `user`.col from `user`, ambiguous_ref_with_source", + "Table": "`user`, ambiguous_ref_with_source", + "Variant": "Scatter" + }, + "Original": "select user.col from user join ambiguous_ref_with_source", + "QueryType": "SELECT", + "TablesUsed": [ + "user.ambiguous_ref_with_source", + "user.user" + ] + } + }, + { + "comment": "ambiguous unqualified reference table self-join routes to reference source", + "query": "select r1.col from ambiguous_ref_with_source r1 join ambiguous_ref_with_source", + "v3-plan": { + "Instructions": { + "FieldQuery": "select r1.col from ambiguous_ref_with_source as r1 join ambiguous_ref_with_source where 1 != 1", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "OperatorType": "Route", + "Query": "select r1.col from ambiguous_ref_with_source as r1 join ambiguous_ref_with_source", + "Table": "ambiguous_ref_with_source", + "Variant": "Reference" + }, + "Original": "select r1.col from ambiguous_ref_with_source r1 join ambiguous_ref_with_source", + "QueryType": "SELECT" + }, + "gen4-plan": { + "Instructions": { + "FieldQuery": "select r1.col from ambiguous_ref_with_source as r1, ambiguous_ref_with_source where 1 != 1", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "OperatorType": "Route", + "Query": "select r1.col from ambiguous_ref_with_source as r1, ambiguous_ref_with_source", + "Table": "ambiguous_ref_with_source", + "Variant": "Reference" + }, + "Original": "select r1.col from ambiguous_ref_with_source r1 join ambiguous_ref_with_source", + "QueryType": "SELECT", + "TablesUsed": [ + "main.ambiguous_ref_with_source" + ] + } + }, + { + "comment": "ambiguous unqualified reference table can merge with other opcodes left to right.", + "query": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source join user", + "v3-plan": { + "Instructions": { + "Inputs": [ + { + "FieldQuery": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source where 1 != 1", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "OperatorType": "Route", + "Query": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source", + "Table": "ambiguous_ref_with_source", + "Variant": "Reference" + }, + { + "FieldQuery": "select 1 from `user` where 1 != 1", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "OperatorType": "Route", + "Query": "select 1 from `user`", + "Table": "`user`", + "Variant": "Scatter" + } + ], + "JoinColumnIndexes": "L:0", + "OperatorType": "Join", + "TableName": "ambiguous_ref_with_source_`user`", + "Variant": "Join" + }, + "Original": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source join user", + "QueryType": "SELECT" + }, + "gen4-plan": { + "Instructions": { + "FieldQuery": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source, `user` where 1 != 1", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "OperatorType": "Route", + "Query": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source, `user`", + "Table": "`user`, ambiguous_ref_with_source", + "Variant": "Scatter" + }, + "Original": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source join user", + "QueryType": "SELECT", + "TablesUsed": [ + "user.ambiguous_ref_with_source", + "user.user" + ] + } + }, + { + "comment": "ambiguous unqualified reference table can merge with other opcodes left to right and vindex value is in the plan", + "query": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source join (select aa from user where user.id=1) user", + "v3-plan": { + "Instructions": { + "Inputs": [ + { + "FieldQuery": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source where 1 != 1", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "OperatorType": "Route", + "Query": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source", + "Table": "ambiguous_ref_with_source", + "Variant": "Reference" + }, + { + "FieldQuery": "select 1 from (select aa from `user` where 1 != 1) as `user` where 1 != 1", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "OperatorType": "Route", + "Query": "select 1 from (select aa from `user` where `user`.id = 1) as `user`", + "Table": "`user`", + "Values": [ + "INT64(1)" + ], + "Variant": "EqualUnique", + "Vindex": "user_index" + } + ], + "JoinColumnIndexes": "L:0", + "OperatorType": "Join", + "TableName": "ambiguous_ref_with_source_`user`", + "Variant": "Join" + }, + "Original": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source join (select aa from user where user.id=1) user", + "QueryType": "SELECT" + }, + "gen4-plan": { + "Instructions": { + "FieldQuery": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source, (select aa from `user` where 1 != 1) as `user` where 1 != 1", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "OperatorType": "Route", + "Query": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source, (select aa from `user` where `user`.id = 1) as `user`", + "Table": "`user`, ambiguous_ref_with_source", + "Values": [ + "INT64(1)" + ], + "Variant": "EqualUnique", + "Vindex": "user_index" + }, + "Original": "select ambiguous_ref_with_source.col from ambiguous_ref_with_source join (select aa from user where user.id=1) user", + "QueryType": "SELECT", + "TablesUsed": [ + "user.ambiguous_ref_with_source", + "user.user" + ] + } + }, + { + "comment": "qualified join to reference table routes to optimal keyspace", + "query": "select user.col from user join main.ambiguous_ref_with_source", + "v3-plan": { + "Instructions": { + "Inputs": [ + { + "FieldQuery": "select `user`.col from `user` where 1 != 1", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "OperatorType": "Route", + "Query": "select `user`.col from `user`", + "Table": "`user`", + "Variant": "Scatter" + }, + { + "FieldQuery": "select 1 from ambiguous_ref_with_source where 1 != 1", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "OperatorType": "Route", + "Query": "select 1 from ambiguous_ref_with_source", + "Table": "ambiguous_ref_with_source", + "Variant": "Reference" + } + ], + "JoinColumnIndexes": "L:0", + "OperatorType": "Join", + "TableName": "`user`_ambiguous_ref_with_source", + "Variant": "Join" + }, + "Original": "select user.col from user join main.ambiguous_ref_with_source", + "QueryType": "SELECT" + }, + "gen4-plan": { + "Instructions": { + "FieldQuery": "select `user`.col from `user`, ambiguous_ref_with_source where 1 != 1", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "OperatorType": "Route", + "Query": "select `user`.col from `user`, ambiguous_ref_with_source", + "Table": "`user`, ambiguous_ref_with_source", + "Variant": "Scatter" + }, + "Original": "select user.col from user join main.ambiguous_ref_with_source", + "QueryType": "SELECT", + "TablesUsed": [ + "user.ambiguous_ref_with_source", + "user.user" + ] + } + }, + { + "comment": "insert into ambiguous qualified reference table routes to source", + "query": "insert into ambiguous_ref_with_source(col) values(1)", + "v3-plan": { + "Instructions": { + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "MultiShardAutocommit": false, + "OperatorType": "Insert", + "Query": "insert into ambiguous_ref_with_source(col) values (1)", + "TableName": "ambiguous_ref_with_source", + "TargetTabletType": "PRIMARY", + "Variant": "Unsharded" + }, + "Original": "insert into ambiguous_ref_with_source(col) values(1)", + "QueryType": "INSERT", + "TablesUsed": [ + "main.ambiguous_ref_with_source" + ] + }, + "gen4-plan": { + "Instructions": { + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "MultiShardAutocommit": false, + "OperatorType": "Insert", + "Query": "insert into ambiguous_ref_with_source(col) values (1)", + "TableName": "ambiguous_ref_with_source", + "TargetTabletType": "PRIMARY", + "Variant": "Unsharded" + }, + "Original": "insert into ambiguous_ref_with_source(col) values(1)", + "QueryType": "INSERT", + "TablesUsed": [ + "main.ambiguous_ref_with_source" + ] + } + }, + { + "comment": "insert into qualified ambiguous reference table routes v3 to requested keyspace gen4 to source", + "query": "insert into user.ambiguous_ref_with_source(col) values(1)", + "v3-plan": { + "Instructions": { + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "MultiShardAutocommit": false, + "OperatorType": "Insert", + "Query": "insert into ambiguous_ref_with_source(col) values (1)", + "TableName": "ambiguous_ref_with_source", + "TargetTabletType": "PRIMARY", + "Variant": "Sharded" + }, + "Original": "insert into user.ambiguous_ref_with_source(col) values(1)", + "QueryType": "INSERT", + "TablesUsed": [ + "user.ambiguous_ref_with_source" + ] + }, + "gen4-plan": { + "Instructions": { + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "MultiShardAutocommit": false, + "OperatorType": "Insert", + "Query": "insert into ambiguous_ref_with_source(col) values (1)", + "TableName": "ambiguous_ref_with_source", + "TargetTabletType": "PRIMARY", + "Variant": "Unsharded" + }, + "Original": "insert into user.ambiguous_ref_with_source(col) values(1)", + "QueryType": "INSERT", + "TablesUsed": [ + "main.ambiguous_ref_with_source" + ] + } + }, + { + "comment": "update unqualified ambiguous reference table routes to source", + "query": "update ambiguous_ref_with_source set col = 1", + "v3-plan": { + "Instructions": { + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "MultiShardAutocommit": false, + "OperatorType": "Update", + "Query": "update ambiguous_ref_with_source set col = 1", + "Table": "ambiguous_ref_with_source", + "TargetTabletType": "PRIMARY", + "Variant": "Unsharded" + }, + "Original": "update ambiguous_ref_with_source set col = 1", + "QueryType": "UPDATE", + "TablesUsed": [ + "main.ambiguous_ref_with_source" + ] + }, + "gen4-plan": { + "Instructions": { + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "MultiShardAutocommit": false, + "OperatorType": "Update", + "Query": "update ambiguous_ref_with_source set col = 1", + "Table": "ambiguous_ref_with_source", + "TargetTabletType": "PRIMARY", + "Variant": "Unsharded" + }, + "Original": "update ambiguous_ref_with_source set col = 1", + "QueryType": "UPDATE", + "TablesUsed": [ + "main.ambiguous_ref_with_source" + ] + } + }, + { + "comment": "update qualified ambiguous reference table v3 error no primary vindex v4 route to source", + "query": "update user.ambiguous_ref_with_source set col = 1", + "v3-plan": "table 'ambiguous_ref_with_source' does not have a primary vindex", + "gen4-plan": { + "Instructions": { + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "MultiShardAutocommit": false, + "OperatorType": "Update", + "Query": "update ambiguous_ref_with_source set col = 1", + "Table": "ambiguous_ref_with_source", + "TargetTabletType": "PRIMARY", + "Variant": "Scatter" + }, + "Original": "update user.ambiguous_ref_with_source set col = 1", + "QueryType": "UPDATE", + "TablesUsed": [ + "main.ambiguous_ref_with_source" + ] + } + }, + { + "comment": "delete from unqualified ambiguous reference table routes to source", + "query": "delete from ambiguous_ref_with_source where col = 1", + "v3-plan": { + "Instructions": { + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "MultiShardAutocommit": false, + "OperatorType": "Delete", + "Query": "delete from ambiguous_ref_with_source where col = 1", + "Table": "ambiguous_ref_with_source", + "TargetTabletType": "PRIMARY", + "Variant": "Unsharded" + }, + "Original": "delete from ambiguous_ref_with_source where col = 1", + "QueryType": "DELETE", + "TablesUsed": [ + "main.ambiguous_ref_with_source" + ] + }, + "gen4-plan": { + "Instructions": { + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "MultiShardAutocommit": false, + "OperatorType": "Delete", + "Query": "delete from ambiguous_ref_with_source where col = 1", + "Table": "ambiguous_ref_with_source", + "TargetTabletType": "PRIMARY", + "Variant": "Unsharded" + }, + "Original": "delete from ambiguous_ref_with_source where col = 1", + "QueryType": "DELETE", + "TablesUsed": [ + "main.ambiguous_ref_with_source" + ] + } + }, + { + "comment": "delete from qualified ambiguous reference table v3 error no primary vindex v4 route to source", + "query": "delete from user.ambiguous_ref_with_source where col = 1", + "v3-plan": "table 'ambiguous_ref_with_source' does not have a primary vindex", + "gen4-plan": { + "Instructions": { + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "MultiShardAutocommit": false, + "OperatorType": "Delete", + "Query": "delete from ambiguous_ref_with_source where col = 1", + "Table": "ambiguous_ref_with_source", + "TargetTabletType": "PRIMARY", + "Variant": "Scatter" + }, + "Original": "delete from user.ambiguous_ref_with_source where col = 1", + "QueryType": "DELETE", + "TablesUsed": [ + "main.ambiguous_ref_with_source" + ] + } + }, + { + "comment": "join with unqualified unambiguous ref with source routes to requested table", + "query": "select user.col from user join ref_with_source", + "v3-plan": { + "Instructions": { + "FieldQuery": "select `user`.col from `user` join ref_with_source where 1 != 1", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "OperatorType": "Route", + "Query": "select `user`.col from `user` join ref_with_source", + "Table": "`user`, ref_with_source", + "Variant": "Scatter" + }, + "Original": "select user.col from user join ref_with_source", + "QueryType": "SELECT" + }, + "gen4-plan": { + "Instructions": { + "FieldQuery": "select `user`.col from `user`, ref_with_source where 1 != 1", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "OperatorType": "Route", + "Query": "select `user`.col from `user`, ref_with_source", + "Table": "`user`, ref_with_source", + "Variant": "Scatter" + }, + "Original": "select user.col from user join ref_with_source", + "QueryType": "SELECT", + "TablesUsed": [ + "user.ref_with_source", + "user.user" + ] + } + }, + { + "comment": "join with unqualified reference optimize routes when source & reference have different names", + "query": "select user.col from user join ref_in_source", + "v3-plan": { + "Instructions": { + "Inputs": [ + { + "FieldQuery": "select `user`.col from `user` where 1 != 1", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "OperatorType": "Route", + "Query": "select `user`.col from `user`", + "Table": "`user`", + "Variant": "Scatter" + }, + { + "FieldQuery": "select 1 from ref_in_source where 1 != 1", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "OperatorType": "Route", + "Query": "select 1 from ref_in_source", + "Table": "ref_in_source", + "Variant": "Reference" + } + ], + "JoinColumnIndexes": "L:0", + "OperatorType": "Join", + "TableName": "`user`_ref_in_source", + "Variant": "Join" + }, + "Original": "select user.col from user join ref_in_source", + "QueryType": "SELECT" + }, + "gen4-plan": { + "Instructions": { + "FieldQuery": "select `user`.col from `user`, ref_with_source as ref_in_source where 1 != 1", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "OperatorType": "Route", + "Query": "select `user`.col from `user`, ref_with_source as ref_in_source", + "Table": "`user`, ref_with_source", + "Variant": "Scatter" + }, + "Original": "select user.col from user join ref_in_source", + "QueryType": "SELECT", + "TablesUsed": [ + "user.ref_with_source", + "user.user" + ] + } + }, + { + "comment": "join with unqualified reference respects routing rules", + "query": "select user.col from user join rerouted_ref", + "v3-plan": { + "Instructions": { + "Inputs": [ + { + "FieldQuery": "select `user`.col from `user` where 1 != 1", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "OperatorType": "Route", + "Query": "select `user`.col from `user`", + "Table": "`user`", + "Variant": "Scatter" + }, + { + "FieldQuery": "select 1 from rerouted_ref where 1 != 1", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "OperatorType": "Route", + "Query": "select 1 from rerouted_ref", + "Table": "rerouted_ref", + "Variant": "Reference" + } + ], + "JoinColumnIndexes": "L:0", + "OperatorType": "Join", + "TableName": "`user`_rerouted_ref", + "Variant": "Join" + }, + "Original": "select user.col from user join rerouted_ref", + "QueryType": "SELECT" + }, + "gen4-plan": { + "Instructions": { + "FieldQuery": "select `user`.col from `user`, ref as rerouted_ref where 1 != 1", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "OperatorType": "Route", + "Query": "select `user`.col from `user`, ref as rerouted_ref", + "Table": "`user`, ref", + "Variant": "Scatter" + }, + "Original": "select user.col from user join rerouted_ref", + "QueryType": "SELECT", + "TablesUsed": [ + "user.ref", + "user.user" + ] + } + } +] diff --git a/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json b/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json index 497b3da2500..8c38997f06e 100644 --- a/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json +++ b/go/vt/vtgate/planbuilder/testdata/vschemas/schema.json @@ -40,6 +40,10 @@ { "from_table": "disabled", "to_tables": [] + }, + { + "from_table": "user.rerouted_ref", + "to_tables": ["user.ref"] } ] }, @@ -305,6 +309,18 @@ "ref": { "type": "reference" }, + "ambiguous_ref_with_source": { + "type": "reference", + "source": "main.ambiguous_ref_with_source" + }, + "ref_with_source": { + "type": "reference", + "source": "main.ref_in_source" + }, + "rerouted_ref": { + "type": "reference", + "source": "main.rerouted_ref" + }, "pin_test": { "pinned": "80" }, @@ -462,6 +478,15 @@ }, "unsharded_ref": { "type": "reference" + }, + "ambiguous_ref_with_source": { + "type": "reference" + }, + "ref_in_source": { + "type": "reference" + }, + "rerouted_ref": { + "type": "reference" } } }, diff --git a/go/vt/vtgate/vindexes/cached_size.go b/go/vt/vtgate/vindexes/cached_size.go index 00fc1d459d1..76fe7f4abf5 100644 --- a/go/vt/vtgate/vindexes/cached_size.go +++ b/go/vt/vtgate/vindexes/cached_size.go @@ -394,13 +394,27 @@ func (cached *ReverseBits) CachedSize(alloc bool) int64 { size += hack.RuntimeAllocSize(int64(len(cached.name))) return size } +func (cached *Source) CachedSize(alloc bool) int64 { + if cached == nil { + return int64(0) + } + size := int64(0) + if alloc { + size += int64(32) + } + // field TableName vitess.io/vitess/go/vt/sqlparser.TableName + size += cached.TableName.CachedSize(false) + return size +} + +//go:nocheckptr func (cached *Table) CachedSize(alloc bool) int64 { if cached == nil { return int64(0) } size := int64(0) if alloc { - size += int64(176) + size += int64(192) } // field Type string size += hack.RuntimeAllocSize(int64(len(cached.Type))) @@ -442,6 +456,23 @@ func (cached *Table) CachedSize(alloc bool) int64 { { size += hack.RuntimeAllocSize(int64(cap(cached.Pinned))) } + // field ReferencedBy map[string]*vitess.io/vitess/go/vt/vtgate/vindexes.Table + if cached.ReferencedBy != nil { + size += int64(48) + hmap := reflect.ValueOf(cached.ReferencedBy) + numBuckets := int(math.Pow(2, float64((*(*uint8)(unsafe.Pointer(hmap.Pointer() + uintptr(9))))))) + numOldBuckets := (*(*uint16)(unsafe.Pointer(hmap.Pointer() + uintptr(10)))) + size += hack.RuntimeAllocSize(int64(numOldBuckets * 208)) + if len(cached.ReferencedBy) > 0 || numBuckets > 1 { + size += hack.RuntimeAllocSize(int64(numBuckets * 208)) + } + for k, v := range cached.ReferencedBy { + size += hack.RuntimeAllocSize(int64(len(k))) + size += v.CachedSize(true) + } + } + // field Source *vitess.io/vitess/go/vt/vtgate/vindexes.Source + size += cached.Source.CachedSize(true) return size } func (cached *UnicodeLooseMD5) CachedSize(alloc bool) int64 { diff --git a/go/vt/vtgate/vindexes/vschema.go b/go/vt/vtgate/vindexes/vschema.go index 258a396f1c3..778f4e7b6b5 100644 --- a/go/vt/vtgate/vindexes/vschema.go +++ b/go/vt/vtgate/vindexes/vschema.go @@ -60,7 +60,7 @@ const ( // used for building routing plans. type VSchema struct { RoutingRules map[string]*RoutingRule `json:"routing_rules"` - uniqueTables map[string]*Table + globalTables map[string]*Table uniqueVindexes map[string]Vindex Keyspaces map[string]*KeyspaceSchema `json:"keyspaces"` ShardRoutingRules map[string]string `json:"shard_routing_rules"` @@ -79,7 +79,7 @@ func (rr *RoutingRule) MarshalJSON() ([]byte, error) { } tables := make([]string, 0, len(rr.Tables)) for _, t := range rr.Tables { - tables = append(tables, t.ToString()) + tables = append(tables, t.String()) } return json.Marshal(tables) @@ -97,6 +97,15 @@ type Table struct { Columns []Column `json:"columns,omitempty"` Pinned []byte `json:"pinned,omitempty"` ColumnListAuthoritative bool `json:"column_list_authoritative,omitempty"` + // ReferencedBy is an inverse mapping of tables in other keyspaces that + // reference this table via Source. + // + // This is useful in route-planning for quickly selecting the optimal route + // when JOIN-ing a reference table to a sharded table. + ReferencedBy map[string]*Table `json:"-"` + // Source is a keyspace-qualified table name that points to the source of a + // reference table. Only applicable for tables with Type set to "reference". + Source *Source `json:"source,omitempty"` } // Keyspace contains the keyspcae info for each Table. @@ -187,15 +196,27 @@ type AutoIncrement struct { Sequence *Table `json:"sequence"` } +type Source struct { + sqlparser.TableName +} + +func (source *Source) String() string { + buf := sqlparser.NewTrackedBuffer(nil) + source.Format(buf) + return buf.String() +} + // BuildVSchema builds a VSchema from a SrvVSchema. func BuildVSchema(source *vschemapb.SrvVSchema) (vschema *VSchema) { vschema = &VSchema{ RoutingRules: make(map[string]*RoutingRule), - uniqueTables: make(map[string]*Table), + globalTables: make(map[string]*Table), uniqueVindexes: make(map[string]Vindex), Keyspaces: make(map[string]*KeyspaceSchema), } buildKeyspaces(source, vschema) + buildReferences(source, vschema) + buildGlobalTables(source, vschema) resolveAutoIncrement(source, vschema) addDual(vschema) buildRoutingRule(source, vschema) @@ -216,7 +237,7 @@ func BuildKeyspaceSchema(input *vschemapb.Keyspace, keyspace string) (*KeyspaceS }, } vschema := &VSchema{ - uniqueTables: make(map[string]*Table), + globalTables: make(map[string]*Table), uniqueVindexes: make(map[string]Vindex), Keyspaces: make(map[string]*KeyspaceSchema), } @@ -247,6 +268,124 @@ func buildKeyspaces(source *vschemapb.SrvVSchema, vschema *VSchema) { } } +func buildGlobalTables(source *vschemapb.SrvVSchema, vschema *VSchema) { + for ksname, ks := range source.Keyspaces { + ksvschema := vschema.Keyspaces[ksname] + // If the keyspace requires explicit routing, don't include any of + // its tables in global tables. + if ks.RequireExplicitRouting { + continue + } + buildKeyspaceGlobalTables(ks, vschema, ksvschema) + } +} + +func buildKeyspaceGlobalTables(ks *vschemapb.Keyspace, vschema *VSchema, ksvschema *KeyspaceSchema) { + for tname, t := range ksvschema.Tables { + if gt, ok := vschema.globalTables[tname]; ok { + // There is already an entry table stored in global tables + // with this name. + if gt == nil { + // Table name is already marked ambiguous, nothing to do. + continue + } else if t.isReferencedInKeyspace(gt.Keyspace.Name) { + // If the stored table refers to this table, store this + // table instead. + vschema.globalTables[tname] = t + } else if gt.isReferencedInKeyspace(t.Keyspace.Name) { + // The source of this table is already stored. Do nothing. + continue + } else { + // Otherwise, mark this table name ambiguous. + vschema.globalTables[tname] = nil + } + } else { + vschema.globalTables[tname] = t + } + } +} + +func buildReferences(source *vschemapb.SrvVSchema, vschema *VSchema) { + for ksname, ks := range source.Keyspaces { + ksvschema := vschema.Keyspaces[ksname] + if err := buildKeyspaceReferences(ks, vschema, ksvschema); err != nil && ksvschema.Error == nil { + ksvschema.Error = err + } + } +} + +func buildKeyspaceReferences( + ks *vschemapb.Keyspace, + vschema *VSchema, + ksvschema *KeyspaceSchema, +) error { + keyspace := ksvschema.Keyspace + for tname, t := range ksvschema.Tables { + source := t.Source + + if t.Type != TypeReference || source == nil { + continue + } + + sourceKsname := source.Qualifier.String() + + // Prohibit self-references. + if sourceKsname == keyspace.Name { + return vterrors.Errorf( + vtrpcpb.Code_UNIMPLEMENTED, + "source %q may not reference a table in the same keyspace as table: %s", + source, + tname, + ) + } + + // Validate that reference can be resolved. + _, sourceT, err := vschema.findKeyspaceAndTableBySource(source) + if sourceT == nil { + return err + } + + // Validate source table types. + if !(sourceT.Type == "" || sourceT.Type == TypeReference) { + return vterrors.Errorf( + vtrpcpb.Code_UNIMPLEMENTED, + "source %q may not reference a table of type %q: %s", + source, + sourceT.Type, + tname, + ) + } + + // Update inverse reference table map. + if ot := sourceT.getReferenceInKeyspace(keyspace.Name); ot != nil { + names := []string{ot.Name.String(), tname} + sort.Strings(names) + return vterrors.Errorf( + vtrpcpb.Code_UNIMPLEMENTED, + "source %q may not be referenced more than once per keyspace: %s, %s", + source, + names[0], + names[1], + ) + } + sourceT.addReferenceInKeyspace(keyspace.Name, t) + + // Forbid reference chains. + for sourceT.Source != nil { + chain := fmt.Sprintf("%s => %s => %s", tname, sourceT, sourceT.Source) + + return vterrors.Errorf( + vtrpcpb.Code_UNIMPLEMENTED, + "reference chaining is not allowed %s: %s", + chain, + tname, + ) + } + } + + return nil +} + func buildTables(ks *vschemapb.Keyspace, vschema *VSchema, ksvschema *KeyspaceSchema) error { keyspace := ksvschema.Keyspace for vname, vindexInfo := range ks.Vindexes { @@ -255,7 +394,8 @@ func buildTables(ks *vschemapb.Keyspace, vschema *VSchema, ksvschema *KeyspaceSc return err } - // If the keyspace requires explicit routing, don't include it in global routing + // If the keyspace requires explicit routing, don't include its indexes + // in global routing. if !ks.RequireExplicitRouting { if _, ok := vschema.uniqueVindexes[vname]; ok { vschema.uniqueVindexes[vname] = nil @@ -272,27 +412,58 @@ func buildTables(ks *vschemapb.Keyspace, vschema *VSchema, ksvschema *KeyspaceSc ColumnListAuthoritative: table.ColumnListAuthoritative, } switch table.Type { - case "", TypeReference: + case "": + t.Type = table.Type + case TypeReference: + if table.Source != "" { + tableName, err := parseQualifiedTable(table.Source) + if err != nil { + return vterrors.Errorf( + vtrpcpb.Code_INVALID_ARGUMENT, + "invalid source %q for reference table: %s; %v", + table.Source, + tname, + err, + ) + } + t.Source = &Source{TableName: tableName} + } t.Type = table.Type case TypeSequence: if keyspace.Sharded && table.Pinned == "" { - return fmt.Errorf("sequence table has to be in an unsharded keyspace or must be pinned: %s", tname) + return vterrors.Errorf( + vtrpcpb.Code_FAILED_PRECONDITION, + "sequence table has to be in an unsharded keyspace or must be pinned: %s", + tname, + ) } t.Type = table.Type default: - return fmt.Errorf("unidentified table type %s", table.Type) + return vterrors.Errorf( + vtrpcpb.Code_NOT_FOUND, + "unidentified table type %s", + table.Type, + ) } if table.Pinned != "" { decoded, err := hex.DecodeString(table.Pinned) if err != nil { - return fmt.Errorf("could not decode the keyspace id for pin: %v", err) + return vterrors.Errorf( + vtrpcpb.Code_INVALID_ARGUMENT, + "could not decode the keyspace id for pin: %v", + err, + ) } t.Pinned = decoded } // If keyspace is sharded, then any table that's not a reference or pinned must have vindexes. if keyspace.Sharded && t.Type != TypeReference && table.Pinned == "" && len(table.ColumnVindexes) == 0 { - return fmt.Errorf("missing primary col vindex for table: %s", tname) + return vterrors.Errorf( + vtrpcpb.Code_NOT_FOUND, + "missing primary col vindex for table: %s", + tname, + ) } // Initialize Columns. @@ -300,7 +471,12 @@ func buildTables(ks *vschemapb.Keyspace, vschema *VSchema, ksvschema *KeyspaceSc for _, col := range table.Columns { name := sqlparser.NewIdentifierCI(col.Name) if colNames[name.Lowered()] { - return fmt.Errorf("duplicate column name '%v' for table: %s", name, tname) + return vterrors.Errorf( + vtrpcpb.Code_INVALID_ARGUMENT, + "duplicate column name '%v' for table: %s", + name, + tname, + ) } colNames[name.Lowered()] = true t.Columns = append(t.Columns, Column{Name: name, Type: col.Type}) @@ -310,7 +486,12 @@ func buildTables(ks *vschemapb.Keyspace, vschema *VSchema, ksvschema *KeyspaceSc for i, ind := range table.ColumnVindexes { vindexInfo, ok := ks.Vindexes[ind.Name] if !ok { - return fmt.Errorf("vindex %s not found for table %s", ind.Name, tname) + return vterrors.Errorf( + vtrpcpb.Code_NOT_FOUND, + "vindex %s not found for table %s", + ind.Name, + tname, + ) } vindex := ksvschema.Vindexes[ind.Name] owned := false @@ -320,12 +501,22 @@ func buildTables(ks *vschemapb.Keyspace, vschema *VSchema, ksvschema *KeyspaceSc var columns []sqlparser.IdentifierCI if ind.Column != "" { if len(ind.Columns) > 0 { - return fmt.Errorf("can't use column and columns at the same time in vindex (%s) and table (%s)", ind.Name, tname) + return vterrors.Errorf( + vtrpcpb.Code_INVALID_ARGUMENT, + "can't use column and columns at the same time in vindex (%s) and table (%s)", + ind.Name, + tname, + ) } columns = []sqlparser.IdentifierCI{sqlparser.NewIdentifierCI(ind.Column)} } else { if len(ind.Columns) == 0 { - return fmt.Errorf("must specify at least one column for vindex (%s) and table (%s)", ind.Name, tname) + return vterrors.Errorf( + vtrpcpb.Code_INVALID_ARGUMENT, + "must specify at least one column for vindex (%s) and table (%s)", + ind.Name, + tname, + ) } for _, indCol := range ind.Columns { columns = append(columns, sqlparser.NewIdentifierCI(indCol)) @@ -343,10 +534,20 @@ func buildTables(ks *vschemapb.Keyspace, vschema *VSchema, ksvschema *KeyspaceSc if i == 0 { // Perform Primary vindex check. if !columnVindex.Vindex.IsUnique() { - return fmt.Errorf("primary vindex %s is not Unique for table %s", ind.Name, tname) + return vterrors.Errorf( + vtrpcpb.Code_INVALID_ARGUMENT, + "primary vindex %s is not Unique for table %s", + ind.Name, + tname, + ) } if owned { - return fmt.Errorf("primary vindex %s cannot be owned for table %s", ind.Name, tname) + return vterrors.Errorf( + vtrpcpb.Code_INVALID_ARGUMENT, + "primary vindex %s cannot be owned for table %s", + ind.Name, + tname, + ) } } t.ColumnVindexes = append(t.ColumnVindexes, columnVindex) @@ -364,7 +565,12 @@ func buildTables(ks *vschemapb.Keyspace, vschema *VSchema, ksvschema *KeyspaceSc continue } if i != 0 { - return vterrors.Errorf(vtrpcpb.Code_UNIMPLEMENTED, "multi-column vindex %s should be a primary vindex for table %s", ind.Name, tname) + return vterrors.Errorf( + vtrpcpb.Code_UNIMPLEMENTED, + "multi-column vindex %s should be a primary vindex for table %s", + ind.Name, + tname, + ) } if !mcv.PartialVindex() { // Partial column selection not allowed. @@ -390,16 +596,9 @@ func buildTables(ks *vschemapb.Keyspace, vschema *VSchema, ksvschema *KeyspaceSc t.Ordered = colVindexSorted(t.ColumnVindexes) // Add the table to the map entries. - // If the keyspace requires explicit routing, don't include it in global routing - if !ks.RequireExplicitRouting { - if _, ok := vschema.uniqueTables[tname]; ok { - vschema.uniqueTables[tname] = nil - } else { - vschema.uniqueTables[tname] = t - } - } ksvschema.Tables[tname] = t } + return nil } @@ -419,8 +618,14 @@ func resolveAutoIncrement(source *vschemapb.SrvVSchema, vschema *VSchema) { if err != nil { // Better to remove the table than to leave it partially initialized. delete(ksvschema.Tables, tname) - delete(vschema.uniqueTables, tname) - ksvschema.Error = fmt.Errorf("cannot resolve sequence %s: %v", table.AutoIncrement.Sequence, err) + delete(vschema.globalTables, tname) + ksvschema.Error = vterrors.Errorf( + vtrpcpb.Code_NOT_FOUND, + "cannot resolve sequence %s: %s", + table.AutoIncrement.Sequence, + err.Error(), + ) + continue } t.AutoIncrement = &AutoIncrement{ @@ -448,26 +653,46 @@ func addDual(vschema *VSchema) { // the keyspaces. For consistency, we'll always use the // first keyspace by lexical ordering. first = ksname - vschema.uniqueTables["dual"] = t + vschema.globalTables["dual"] = t } } } // expects table name of the form . func escapeQualifiedTable(qualifiedTableName string) (string, error) { + keyspace, tableName, err := extractQualifiedTableParts(qualifiedTableName) + if err != nil { + return "", err + } + return fmt.Sprintf("%s.%s", + // unescape() first in case an already escaped string was passed + sqlescape.EscapeID(sqlescape.UnescapeID(keyspace)), + sqlescape.EscapeID(sqlescape.UnescapeID(tableName))), nil +} + +func extractQualifiedTableParts(qualifiedTableName string) (string, string, error) { // It's possible to have a database or table name with a dot in it, but that's not otherwise supported within vitess today arr := strings.Split(qualifiedTableName, ".") switch len(arr) { - case 1: - return "", fmt.Errorf("table %s must be qualified", qualifiedTableName) case 2: - keyspace, tableName := arr[0], arr[1] - return fmt.Sprintf("%s.%s", - // unescape() first in case an already escaped string was passed - sqlescape.EscapeID(sqlescape.UnescapeID(keyspace)), - sqlescape.EscapeID(sqlescape.UnescapeID(tableName))), nil + return arr[0], arr[1], nil } - return "", fmt.Errorf("invalid table name: %s, it must be of the qualified form . (dots are not allowed in either name)", qualifiedTableName) + // Using fmt.Errorf instead of vterrors here because this error is always wrapped in vterrors. + return "", "", fmt.Errorf( + "invalid table name: %s, it must be of the qualified form . (dots are not allowed in either name)", + qualifiedTableName, + ) +} + +func parseQualifiedTable(qualifiedTableName string) (sqlparser.TableName, error) { + keyspace, tableName, err := extractQualifiedTableParts(qualifiedTableName) + if err != nil { + return sqlparser.TableName{}, err + } + return sqlparser.TableName{ + Qualifier: sqlparser.NewIdentifierCS(keyspace), + Name: sqlparser.NewIdentifierCS(tableName), + }, nil } func buildRoutingRule(source *vschemapb.SrvVSchema, vschema *VSchema) { @@ -480,14 +705,23 @@ outer: rr := &RoutingRule{} if len(rule.ToTables) > 1 { vschema.RoutingRules[rule.FromTable] = &RoutingRule{ - Error: fmt.Errorf("table %v has more than one target: %v", rule.FromTable, rule.ToTables), + Error: vterrors.Errorf( + vtrpcpb.Code_INVALID_ARGUMENT, + "table %v has more than one target: %v", + rule.FromTable, + rule.ToTables, + ), } continue } for _, toTable := range rule.ToTables { if _, ok := vschema.RoutingRules[rule.FromTable]; ok { vschema.RoutingRules[rule.FromTable] = &RoutingRule{ - Error: fmt.Errorf("duplicate rule for entry %s", rule.FromTable), + Error: vterrors.Errorf( + vtrpcpb.Code_ALREADY_EXISTS, + "duplicate rule for entry %s", + rule.FromTable, + ), } continue outer } @@ -496,7 +730,10 @@ outer: toTable, err = escapeQualifiedTable(toTable) if err != nil { vschema.RoutingRules[rule.FromTable] = &RoutingRule{ - Error: err, + Error: vterrors.Errorf( + vtrpcpb.Code_INVALID_ARGUMENT, + err.Error(), + ), } continue outer } @@ -511,7 +748,11 @@ outer: } if toKeyspace == "" { vschema.RoutingRules[rule.FromTable] = &RoutingRule{ - Error: fmt.Errorf("table %s must be qualified", toTable), + Error: vterrors.Errorf( + vtrpcpb.Code_INVALID_ARGUMENT, + "table %s must be qualified", + toTable, + ), } continue outer } @@ -552,7 +793,12 @@ func (vschema *VSchema) FindTable(keyspace, tablename string) (*Table, error) { return nil, err } if t == nil { - return nil, fmt.Errorf("table %s not found", tablename) + return nil, vterrors.NewErrorf( + vtrpcpb.Code_NOT_FOUND, + vterrors.UnknownTable, + "table %s not found", + tablename, + ) } return t, nil } @@ -560,10 +806,14 @@ func (vschema *VSchema) FindTable(keyspace, tablename string) (*Table, error) { // findTable is like FindTable, but does not return an error if a table is not found. func (vschema *VSchema) findTable(keyspace, tablename string) (*Table, error) { if keyspace == "" { - table, ok := vschema.uniqueTables[tablename] + table, ok := vschema.globalTables[tablename] if table == nil { if ok { - return nil, fmt.Errorf("ambiguous table reference: %s", tablename) + return nil, vterrors.Errorf( + vtrpcpb.Code_FAILED_PRECONDITION, + "ambiguous table reference: %s", + tablename, + ) } if len(vschema.Keyspaces) != 1 { return nil, nil @@ -580,7 +830,12 @@ func (vschema *VSchema) findTable(keyspace, tablename string) (*Table, error) { } ks, ok := vschema.Keyspaces[keyspace] if !ok { - return nil, vterrors.NewErrorf(vtrpcpb.Code_NOT_FOUND, vterrors.BadDb, "Unknown database '%s' in vschema", keyspace) + return nil, vterrors.NewErrorf( + vtrpcpb.Code_NOT_FOUND, + vterrors.BadDb, + "Unknown database '%s' in vschema", + keyspace, + ) } table := ks.Tables[tablename] if table == nil { @@ -608,7 +863,11 @@ func (vschema *VSchema) FindRoutedTable(keyspace, tablename string, tabletType t return nil, rr.Error } if len(rr.Tables) == 0 { - return nil, fmt.Errorf("table %s has been disabled", tablename) + return nil, vterrors.Errorf( + vtrpcpb.Code_FAILED_PRECONDITION, + "table %s has been disabled", + tablename, + ) } return rr.Tables[0], nil } @@ -635,6 +894,33 @@ func (vschema *VSchema) FindTableOrVindex(keyspace, name string, tabletType topo return nil, nil, NotFoundError{TableName: name} } +func (vschema *VSchema) findKeyspaceAndTableBySource(source *Source) (*Keyspace, *Table, error) { + sourceKsname := source.Qualifier.String() + sourceTname := source.Name.String() + + sourceKs, ok := vschema.Keyspaces[sourceKsname] + if !ok { + return nil, nil, vterrors.NewErrorf( + vtrpcpb.Code_NOT_FOUND, + vterrors.BadDb, + "source %q references a non-existent keyspace %q", + source, + sourceKsname, + ) + } + + sourceT, ok := sourceKs.Tables[sourceTname] + if !ok { + return sourceKs.Keyspace, nil, vterrors.NewErrorf( + vtrpcpb.Code_NOT_FOUND, + vterrors.UnknownTable, + "source %q references a table %q that is not present in the VSchema of keyspace %q", source, sourceTname, sourceKsname, + ) + } + + return sourceKs.Keyspace, sourceT, nil +} + // NotFoundError represents the error where the table name was not found type NotFoundError struct { TableName string @@ -652,13 +938,22 @@ func (vschema *VSchema) FindVindex(keyspace, name string) (Vindex, error) { if keyspace == "" { vindex, ok := vschema.uniqueVindexes[name] if vindex == nil && ok { - return nil, fmt.Errorf("ambiguous vindex reference: %s", name) + return nil, vterrors.Errorf( + vtrpcpb.Code_FAILED_PRECONDITION, + "ambiguous vindex reference: %s", + name, + ) } return vindex, nil } ks, ok := vschema.Keyspaces[keyspace] if !ok { - return nil, vterrors.NewErrorf(vtrpcpb.Code_NOT_FOUND, vterrors.BadDb, "Unknown database '%s' in vschema", keyspace) + return nil, vterrors.NewErrorf( + vtrpcpb.Code_NOT_FOUND, + vterrors.BadDb, + "Unknown database '%s' in vschema", + keyspace, + ) } return ks.Vindexes[name], nil } @@ -737,13 +1032,21 @@ func ChooseVindexForType(typ querypb.Type) (string, error) { case sqltypes.IsBinary(typ): return "binary_md5", nil } - return "", fmt.Errorf("type %v is not recommended for a vindex", typ) + return "", vterrors.Errorf( + vtrpcpb.Code_INVALID_ARGUMENT, + "type %v is not recommended for a vindex", + typ, + ) } // FindBestColVindex finds the best ColumnVindex for VReplication. func FindBestColVindex(table *Table) (*ColumnVindex, error) { if table.ColumnVindexes == nil || len(table.ColumnVindexes) == 0 { - return nil, fmt.Errorf("table %s has no vindex", table.Name.String()) + return nil, vterrors.Errorf( + vtrpcpb.Code_INVALID_ARGUMENT, + "table %s has no vindex", + table.Name.String(), + ) } var result *ColumnVindex for _, cv := range table.ColumnVindexes { @@ -758,7 +1061,11 @@ func FindBestColVindex(table *Table) (*ColumnVindex, error) { } } if result == nil { - return nil, fmt.Errorf("could not find a vindex to compute keyspace id for table %v", table.Name.String()) + return nil, vterrors.Errorf( + vtrpcpb.Code_NOT_FOUND, + "could not find a vindex to compute keyspace id for table %v", + table.Name.String(), + ) } return result, nil } @@ -770,7 +1077,11 @@ func FindBestColVindex(table *Table) (*ColumnVindex, error) { // if the final result is too expensive, return nil func FindVindexForSharding(tableName string, colVindexes []*ColumnVindex) (*ColumnVindex, error) { if len(colVindexes) == 0 { - return nil, fmt.Errorf("no vindex definition for table %v", tableName) + return nil, vterrors.Errorf( + vtrpcpb.Code_NOT_FOUND, + "no vindex definition for table %v", + tableName, + ) } result := colVindexes[0] for _, colVindex := range colVindexes { @@ -783,16 +1094,45 @@ func FindVindexForSharding(tableName string, colVindexes []*ColumnVindex) (*Colu } } if result.Cost() > 1 || !result.IsUnique() { - return nil, fmt.Errorf("could not find a vindex to use for sharding table %v", tableName) + return nil, vterrors.Errorf( + vtrpcpb.Code_NOT_FOUND, + "could not find a vindex to use for sharding table %v", + tableName, + ) } return result, nil } -// ToString prints the table name -func (t *Table) ToString() string { +// String prints the (possibly qualified) table name +func (t *Table) String() string { res := "" + if t == nil { + return res + } if t.Keyspace != nil { res = t.Keyspace.Name + "." } return res + t.Name.String() } + +func (t *Table) addReferenceInKeyspace(keyspace string, table *Table) { + if t.ReferencedBy == nil { + t.ReferencedBy = make(map[string]*Table) + } + t.ReferencedBy[keyspace] = table +} + +func (t *Table) getReferenceInKeyspace(keyspace string) *Table { + if t.ReferencedBy == nil { + return nil + } + t, ok := t.ReferencedBy[keyspace] + if !ok { + return nil + } + return t +} + +func (t *Table) isReferencedInKeyspace(keyspace string) bool { + return t.getReferenceInKeyspace(keyspace) != nil +} diff --git a/go/vt/vtgate/vindexes/vschema_test.go b/go/vt/vtgate/vindexes/vschema_test.go index 4752b771d64..1c0f58b1f40 100644 --- a/go/vt/vtgate/vindexes/vschema_test.go +++ b/go/vt/vtgate/vindexes/vschema_test.go @@ -279,7 +279,7 @@ func TestUnshardedVSchema(t *testing.T) { } want := &VSchema{ RoutingRules: map[string]*RoutingRule{}, - uniqueTables: map[string]*Table{ + globalTables: map[string]*Table{ "t1": t1, "dual": dual, }, @@ -341,7 +341,7 @@ func TestVSchemaColumns(t *testing.T) { } want := &VSchema{ RoutingRules: map[string]*RoutingRule{}, - uniqueTables: map[string]*Table{ + globalTables: map[string]*Table{ "t1": t1, "dual": dual, }, @@ -405,7 +405,7 @@ func TestVSchemaColumnListAuthoritative(t *testing.T) { } want := &VSchema{ RoutingRules: map[string]*RoutingRule{}, - uniqueTables: map[string]*Table{ + globalTables: map[string]*Table{ "t1": t1, "dual": dual, }, @@ -484,7 +484,7 @@ func TestVSchemaPinned(t *testing.T) { } want := &VSchema{ RoutingRules: map[string]*RoutingRule{}, - uniqueTables: map[string]*Table{ + globalTables: map[string]*Table{ "t1": t1, "dual": dual, }, @@ -590,7 +590,7 @@ func TestShardedVSchemaOwned(t *testing.T) { } want := &VSchema{ RoutingRules: map[string]*RoutingRule{}, - uniqueTables: map[string]*Table{ + globalTables: map[string]*Table{ "t1": t1, "dual": dual, }, @@ -827,7 +827,7 @@ func TestVSchemaRoutingRules(t *testing.T) { Error: errors.New("invalid table name: t1.t2.t3, it must be of the qualified form . (dots are not allowed in either name)"), }, "unqualified": { - Error: errors.New("table t1 must be qualified"), + Error: errors.New("invalid table name: t1, it must be of the qualified form . (dots are not allowed in either name)"), }, "badkeyspace": { Error: errors.New("Unknown database 'ks3' in vschema"), @@ -836,7 +836,7 @@ func TestVSchemaRoutingRules(t *testing.T) { Error: errors.New("table t2 not found"), }, }, - uniqueTables: map[string]*Table{ + globalTables: map[string]*Table{ "t1": t1, "t2": t2, "dual": dual1, @@ -1267,7 +1267,7 @@ func TestShardedVSchemaMultiColumnVindex(t *testing.T) { } want := &VSchema{ RoutingRules: map[string]*RoutingRule{}, - uniqueTables: map[string]*Table{ + globalTables: map[string]*Table{ "t1": t1, "dual": dual, }, @@ -1369,7 +1369,7 @@ func TestShardedVSchemaNotOwned(t *testing.T) { } want := &VSchema{ RoutingRules: map[string]*RoutingRule{}, - uniqueTables: map[string]*Table{ + globalTables: map[string]*Table{ "t1": t1, "dual": dual, }, @@ -1501,7 +1501,7 @@ func TestBuildVSchemaDupSeq(t *testing.T) { } want := &VSchema{ RoutingRules: map[string]*RoutingRule{}, - uniqueTables: map[string]*Table{ + globalTables: map[string]*Table{ "t1": nil, "dual": duala, }, @@ -1574,7 +1574,7 @@ func TestBuildVSchemaDupTable(t *testing.T) { } want := &VSchema{ RoutingRules: map[string]*RoutingRule{}, - uniqueTables: map[string]*Table{ + globalTables: map[string]*Table{ "t1": nil, "dual": duala, }, @@ -1712,7 +1712,7 @@ func TestBuildVSchemaDupVindex(t *testing.T) { } want := &VSchema{ RoutingRules: map[string]*RoutingRule{}, - uniqueTables: map[string]*Table{ + globalTables: map[string]*Table{ "t1": nil, "dual": duala, }, @@ -1905,6 +1905,271 @@ func TestBuildVSchemaPrimaryCannotBeOwned(t *testing.T) { } } +func TestBuildVSchemaReferenceTableSourceMustBeQualified(t *testing.T) { + input := vschemapb.SrvVSchema{ + Keyspaces: map[string]*vschemapb.Keyspace{ + "unsharded": { + Sharded: false, + Tables: map[string]*vschemapb.Table{ + "src": {}, + }, + }, + "sharded": { + Sharded: true, + Tables: map[string]*vschemapb.Table{ + "ref": { + Type: "reference", + Source: "src", + }, + }, + }, + }, + } + vschema := BuildVSchema(&input) + require.NoError(t, vschema.Keyspaces["unsharded"].Error) + require.Error(t, vschema.Keyspaces["sharded"].Error) + require.EqualError(t, vschema.Keyspaces["sharded"].Error, + "invalid source \"src\" for reference table: ref; invalid table name: src, it must be of the qualified form . (dots are not allowed in either name)") +} + +func TestBuildVSchemaReferenceTableSourceMustBeInDifferentKeyspace(t *testing.T) { + input := vschemapb.SrvVSchema{ + Keyspaces: map[string]*vschemapb.Keyspace{ + "sharded": { + Sharded: true, + Vindexes: map[string]*vschemapb.Vindex{ + "hash": { + Type: "binary_md5", + }, + }, + Tables: map[string]*vschemapb.Table{ + "ref": { + Type: "reference", + Source: "sharded.src", + }, + "src": { + ColumnVindexes: []*vschemapb.ColumnVindex{ + { + Column: "c1", + Name: "hash", + }, + }, + }, + }, + }, + }, + } + vschema := BuildVSchema(&input) + require.Error(t, vschema.Keyspaces["sharded"].Error) + require.EqualError(t, vschema.Keyspaces["sharded"].Error, + "source \"sharded.src\" may not reference a table in the same keyspace as table: ref") +} + +func TestBuildVSchemaReferenceTableSourceKeyspaceMustExist(t *testing.T) { + input := vschemapb.SrvVSchema{ + Keyspaces: map[string]*vschemapb.Keyspace{ + "sharded": { + Sharded: true, + Tables: map[string]*vschemapb.Table{ + "ref": { + Type: "reference", + Source: "unsharded.src", + }, + }, + }, + }, + } + vschema := BuildVSchema(&input) + require.Error(t, vschema.Keyspaces["sharded"].Error) + require.EqualError(t, vschema.Keyspaces["sharded"].Error, + "source \"unsharded.src\" references a non-existent keyspace \"unsharded\"") +} + +func TestBuildVSchemaReferenceTableSourceTableMustExist(t *testing.T) { + input := vschemapb.SrvVSchema{ + Keyspaces: map[string]*vschemapb.Keyspace{ + "unsharded": { + Sharded: false, + Tables: map[string]*vschemapb.Table{ + "foo": {}, + }, + }, + "sharded": { + Sharded: true, + Tables: map[string]*vschemapb.Table{ + "ref": { + Type: "reference", + Source: "unsharded.src", + }, + }, + }, + }, + } + vschema := BuildVSchema(&input) + require.Error(t, vschema.Keyspaces["sharded"].Error) + require.EqualError(t, vschema.Keyspaces["sharded"].Error, + "source \"unsharded.src\" references a table \"src\" that is not present in the VSchema of keyspace \"unsharded\"") +} + +func TestBuildVSchemaReferenceTableSourceMayUseShardedKeyspace(t *testing.T) { + input := vschemapb.SrvVSchema{ + Keyspaces: map[string]*vschemapb.Keyspace{ + "sharded1": { + Sharded: true, + Vindexes: map[string]*vschemapb.Vindex{ + "hash": { + Type: "binary_md5", + }, + }, + Tables: map[string]*vschemapb.Table{ + "src": { + ColumnVindexes: []*vschemapb.ColumnVindex{ + { + Column: "c1", + Name: "hash", + }, + }, + }, + }, + }, + "sharded2": { + Sharded: true, + Tables: map[string]*vschemapb.Table{ + "ref": { + Type: "reference", + Source: "sharded1.src", + }, + }, + }, + }, + } + vschema := BuildVSchema(&input) + require.NoError(t, vschema.Keyspaces["sharded1"].Error) + require.NoError(t, vschema.Keyspaces["sharded2"].Error) +} + +func TestBuildVSchemaReferenceTableSourceTableMustBeBasicOrReference(t *testing.T) { + input := vschemapb.SrvVSchema{ + Keyspaces: map[string]*vschemapb.Keyspace{ + "unsharded1": { + Sharded: false, + Tables: map[string]*vschemapb.Table{ + "src1": { + Type: "sequence", + }, + }, + }, + "unsharded2": { + Sharded: false, + Tables: map[string]*vschemapb.Table{ + "src2": { + Type: "reference", + Source: "unsharded1.src1", + }, + }, + }, + "unsharded3": { + Tables: map[string]*vschemapb.Table{ + "src3": { + Type: "reference", + }, + }, + }, + "sharded1": { + Sharded: true, + Tables: map[string]*vschemapb.Table{ + "ref1": { + Type: "reference", + Source: "unsharded1.src1", + }, + }, + }, + "sharded2": { + Sharded: true, + Tables: map[string]*vschemapb.Table{ + "ref2": { + Type: "reference", + Source: "unsharded3.src3", + }, + }, + }, + }, + } + vschema := BuildVSchema(&input) + require.Error(t, vschema.Keyspaces["sharded1"].Error) + require.EqualError(t, vschema.Keyspaces["sharded1"].Error, + "source \"unsharded1.src1\" may not reference a table of type \"sequence\": ref1") + require.NoError(t, vschema.Keyspaces["sharded2"].Error) +} + +func TestBuildVSchemaSourceMayBeReferencedAtMostOncePerKeyspace(t *testing.T) { + input := vschemapb.SrvVSchema{ + Keyspaces: map[string]*vschemapb.Keyspace{ + "unsharded": { + Sharded: false, + Tables: map[string]*vschemapb.Table{ + "src": {}, + }, + }, + "sharded": { + Sharded: true, + Tables: map[string]*vschemapb.Table{ + "ref2": { + Type: "reference", + Source: "unsharded.src", + }, + "ref1": { + Type: "reference", + Source: "unsharded.src", + }, + }, + }, + }, + } + vschema := BuildVSchema(&input) + require.Error(t, vschema.Keyspaces["sharded"].Error) + require.EqualError(t, vschema.Keyspaces["sharded"].Error, + "source \"unsharded.src\" may not be referenced more than once per keyspace: ref1, ref2") +} + +func TestBuildVSchemaMayNotChainReferences(t *testing.T) { + input := vschemapb.SrvVSchema{ + Keyspaces: map[string]*vschemapb.Keyspace{ + "unsharded1": { + Sharded: false, + Tables: map[string]*vschemapb.Table{ + "ref": { + Type: TypeReference, + Source: "unsharded2.ref", + }, + }, + }, + "unsharded2": { + Sharded: false, + Tables: map[string]*vschemapb.Table{ + "ref": { + Type: TypeReference, + Source: "unsharded3.ref", + }, + }, + }, + "unsharded3": { + Sharded: false, + Tables: map[string]*vschemapb.Table{ + "ref": { + Type: "reference", + Source: "unsharded1.ref", + }, + }, + }, + }, + } + vschema := BuildVSchema(&input) + require.Error(t, vschema.Keyspaces["unsharded1"].Error) + require.EqualError(t, vschema.Keyspaces["unsharded1"].Error, + "reference chaining is not allowed ref => unsharded2.ref => unsharded3.ref: ref") +} + func TestSequence(t *testing.T) { good := vschemapb.SrvVSchema{ Keyspaces: map[string]*vschemapb.Keyspace{ @@ -2033,7 +2298,7 @@ func TestSequence(t *testing.T) { } want := &VSchema{ RoutingRules: map[string]*RoutingRule{}, - uniqueTables: map[string]*Table{ + globalTables: map[string]*Table{ "seq": seq, "t1": t1, "t2": t2, @@ -2204,6 +2469,7 @@ func TestFindTable(t *testing.T) { Tables: map[string]*vschemapb.Table{ "ta": {}, "t1": {}, + "t2": {}, }, }, "ksb": { @@ -2233,6 +2499,10 @@ func TestFindTable(t *testing.T) { }, }, }, + "t2": { + Type: "reference", + Source: "ksa.t2", + }, }, }, }, @@ -2254,6 +2524,32 @@ func TestFindTable(t *testing.T) { require.NoError(t, err) require.Equal(t, ta, got) + t2 := &Table{ + Name: sqlparser.NewIdentifierCS("t2"), + Keyspace: &Keyspace{ + Name: "ksa", + }, + ReferencedBy: map[string]*Table{ + "ksb": { + Type: "reference", + Name: sqlparser.NewIdentifierCS("t2"), + Keyspace: &Keyspace{ + Sharded: true, + Name: "ksb", + }, + Source: &Source{ + sqlparser.TableName{ + Qualifier: sqlparser.NewIdentifierCS("ksa"), + Name: sqlparser.NewIdentifierCS("t2"), + }, + }, + }, + }, + } + got, err = vschema.FindTable("", "t2") + require.NoError(t, err) + require.Equal(t, t2, got) + got, _ = vschema.FindTable("ksa", "ta") require.Equal(t, ta, got) @@ -2793,3 +3089,112 @@ func TestMultiColVindexPartialNotAllowed(t *testing.T) { require.True(t, table.ColumnVindexes[0].IsUnique()) require.EqualValues(t, 1, table.ColumnVindexes[0].Cost()) } + +func TestSourceTableHasReferencedBy(t *testing.T) { + input := vschemapb.SrvVSchema{ + Keyspaces: map[string]*vschemapb.Keyspace{ + "unsharded": { + Sharded: false, + Tables: map[string]*vschemapb.Table{ + "src": {}, + }, + }, + "sharded1": { + Sharded: true, + Tables: map[string]*vschemapb.Table{ + "ref": { + Type: "reference", + Source: "unsharded.src", + }, + }, + }, + "sharded2": { + Sharded: true, + Tables: map[string]*vschemapb.Table{ + "ref": { + Type: "reference", + Source: "unsharded.src", + }, + }, + }, + }, + } + vs := BuildVSchema(&input) + ref1, err := vs.FindTable("sharded1", "ref") + require.NoError(t, err) + ref2, err := vs.FindTable("sharded2", "ref") + require.NoError(t, err) + src, err := vs.FindTable("unsharded", "src") + require.NoError(t, err) + require.Equal(t, src.ReferencedBy, map[string]*Table{ + "sharded1": ref1, + "sharded2": ref2, + }) +} + +func TestReferenceTableAndSourceAreGloballyRoutable(t *testing.T) { + input := vschemapb.SrvVSchema{ + Keyspaces: map[string]*vschemapb.Keyspace{ + "unsharded": { + Sharded: false, + Tables: map[string]*vschemapb.Table{ + "t1": {}, + }, + }, + "sharded": { + Sharded: true, + Tables: map[string]*vschemapb.Table{ + "t1": { + Type: "reference", + Source: "unsharded.t1", + }, + }, + }, + }, + } + vs := BuildVSchema(&input) + t1, err := vs.FindTable("unsharded", "t1") + require.NoError(t, err) + globalT1, err := vs.FindTable("", "t1") + require.NoError(t, err) + require.Equal(t, t1, globalT1) + + input.Keyspaces["unsharded"].RequireExplicitRouting = true + vs = BuildVSchema(&input) + t1, err = vs.FindTable("sharded", "t1") + require.NoError(t, err) + globalT1, err = vs.FindTable("", "t1") + require.NoError(t, err) + require.Equal(t, t1, globalT1) +} + +func TestOtherTablesMakeReferenceTableAndSourceAmbiguous(t *testing.T) { + input := vschemapb.SrvVSchema{ + Keyspaces: map[string]*vschemapb.Keyspace{ + "unsharded1": { + Sharded: false, + Tables: map[string]*vschemapb.Table{ + "t1": {}, + }, + }, + "unsharded2": { + Sharded: false, + Tables: map[string]*vschemapb.Table{ + "t1": {}, + }, + }, + "sharded": { + Sharded: true, + Tables: map[string]*vschemapb.Table{ + "t1": { + Type: "reference", + Source: "unsharded1.t1", + }, + }, + }, + }, + } + vs := BuildVSchema(&input) + _, err := vs.FindTable("", "t1") + require.Error(t, err) +} diff --git a/go/vt/vtgate/vschema_manager_test.go b/go/vt/vtgate/vschema_manager_test.go index 9fd1fbb9052..fbbf7ba21c1 100644 --- a/go/vt/vtgate/vschema_manager_test.go +++ b/go/vt/vtgate/vschema_manager_test.go @@ -110,7 +110,7 @@ func TestVSchemaUpdate(t *testing.T) { vm.currentVschema = tcase.currentVSchema vm.VSchemaUpdate(tcase.srvVschema, nil) - utils.MustMatchFn(".uniqueTables", ".uniqueVindexes")(t, tcase.expected, vs) + utils.MustMatchFn(".globalTables", ".uniqueVindexes")(t, tcase.expected, vs) if tcase.srvVschema != nil { utils.MustMatch(t, vs, vm.currentVschema, "currentVschema should have same reference as Vschema") } @@ -210,7 +210,7 @@ func TestRebuildVSchema(t *testing.T) { vm.currentVschema = nil vm.Rebuild() - utils.MustMatchFn(".uniqueTables", ".uniqueVindexes")(t, tcase.expected, vs) + utils.MustMatchFn(".globalTables", ".uniqueVindexes")(t, tcase.expected, vs) if vs != nil { utils.MustMatch(t, vs, vm.currentVschema, "currentVschema should have same reference as Vschema") } diff --git a/proto/vschema.proto b/proto/vschema.proto index 7f06d720284..f618492693c 100644 --- a/proto/vschema.proto +++ b/proto/vschema.proto @@ -91,6 +91,9 @@ message Table { // an authoritative list for the table. This allows // us to expand 'select *' expressions. bool column_list_authoritative = 6; + + // reference tables may optionally indicate their source table. + string source = 7; } // ColumnVindex is used to associate a column to a vindex. diff --git a/test/config.json b/test/config.json index 00f562fc3e5..eac4b6e8dfe 100644 --- a/test/config.json +++ b/test/config.json @@ -639,6 +639,15 @@ "RetryMax": 2, "Tags": [] }, + "vtgate_queries_reference": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/queries/reference"], + "Command": [], + "Manual": false, + "Shard": "vtgate_queries", + "RetryMax": 1, + "Tags": [] + }, "vtgate_concurrentdml": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/vtgate/concurrentdml"], diff --git a/web/vtadmin/src/proto/vtadmin.d.ts b/web/vtadmin/src/proto/vtadmin.d.ts index 8f25e01f073..7336312a1ef 100644 --- a/web/vtadmin/src/proto/vtadmin.d.ts +++ b/web/vtadmin/src/proto/vtadmin.d.ts @@ -32548,6 +32548,9 @@ export namespace vschema { /** Table column_list_authoritative */ column_list_authoritative?: (boolean|null); + + /** Table source */ + source?: (string|null); } /** Represents a Table. */ @@ -32577,6 +32580,9 @@ export namespace vschema { /** Table column_list_authoritative. */ public column_list_authoritative: boolean; + /** Table source. */ + public source: string; + /** * Creates a new Table instance using the specified properties. * @param [properties] Properties to set diff --git a/web/vtadmin/src/proto/vtadmin.js b/web/vtadmin/src/proto/vtadmin.js index 59379ac4c7d..08977a9420d 100644 --- a/web/vtadmin/src/proto/vtadmin.js +++ b/web/vtadmin/src/proto/vtadmin.js @@ -77308,6 +77308,7 @@ $root.vschema = (function() { * @property {Array.|null} [columns] Table columns * @property {string|null} [pinned] Table pinned * @property {boolean|null} [column_list_authoritative] Table column_list_authoritative + * @property {string|null} [source] Table source */ /** @@ -77375,6 +77376,14 @@ $root.vschema = (function() { */ Table.prototype.column_list_authoritative = false; + /** + * Table source. + * @member {string} source + * @memberof vschema.Table + * @instance + */ + Table.prototype.source = ""; + /** * Creates a new Table instance using the specified properties. * @function create @@ -77413,6 +77422,8 @@ $root.vschema = (function() { writer.uint32(/* id 5, wireType 2 =*/42).string(message.pinned); if (message.column_list_authoritative != null && Object.hasOwnProperty.call(message, "column_list_authoritative")) writer.uint32(/* id 6, wireType 0 =*/48).bool(message.column_list_authoritative); + if (message.source != null && Object.hasOwnProperty.call(message, "source")) + writer.uint32(/* id 7, wireType 2 =*/58).string(message.source); return writer; }; @@ -77469,6 +77480,9 @@ $root.vschema = (function() { case 6: message.column_list_authoritative = reader.bool(); break; + case 7: + message.source = reader.string(); + break; default: reader.skipType(tag & 7); break; @@ -77536,6 +77550,9 @@ $root.vschema = (function() { if (message.column_list_authoritative != null && message.hasOwnProperty("column_list_authoritative")) if (typeof message.column_list_authoritative !== "boolean") return "column_list_authoritative: boolean expected"; + if (message.source != null && message.hasOwnProperty("source")) + if (!$util.isString(message.source)) + return "source: string expected"; return null; }; @@ -77582,6 +77599,8 @@ $root.vschema = (function() { message.pinned = String(object.pinned); if (object.column_list_authoritative != null) message.column_list_authoritative = Boolean(object.column_list_authoritative); + if (object.source != null) + message.source = String(object.source); return message; }; @@ -77607,6 +77626,7 @@ $root.vschema = (function() { object.auto_increment = null; object.pinned = ""; object.column_list_authoritative = false; + object.source = ""; } if (message.type != null && message.hasOwnProperty("type")) object.type = message.type; @@ -77626,6 +77646,8 @@ $root.vschema = (function() { object.pinned = message.pinned; if (message.column_list_authoritative != null && message.hasOwnProperty("column_list_authoritative")) object.column_list_authoritative = message.column_list_authoritative; + if (message.source != null && message.hasOwnProperty("source")) + object.source = message.source; return object; };