Skip to content

Commit d4e5cc7

Browse files
rohit-nayak-psrohit-nayak-ps
authored
[release-16.0] MoveTables: add flag to specify that routing rules should not be created when a movetables workflow is created (vitessio#13858)
Signed-off-by: Rohit Nayak <[email protected]>
1 parent c934b2b commit d4e5cc7

File tree

4 files changed

+74
-36
lines changed

4 files changed

+74
-36
lines changed

‎go/vt/vtctl/vtctl.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -467,7 +467,7 @@ var commands = []commandGroup{
467467
{
468468
name: "MoveTables",
469469
method: commandMoveTables,
470-
params: "[--source=<sourceKs>] [--tables=<tableSpecs>] [--cells=<cells>] [--tablet_types=<source_tablet_types>] [--all] [--exclude=<tables>] [--auto_start] [--stop_after_copy] [--defer-secondary-keys] [--on-ddl=<ddl-action>] [--source_shards=<source_shards>] <action> 'action must be one of the following: Create, Complete, Cancel, SwitchTraffic, ReverseTrafffic, Show, or Progress' <targetKs.workflow>",
470+
params: "[--source=<sourceKs>] [--tables=<tableSpecs>] [--cells=<cells>] [--tablet_types=<source_tablet_types>] [--all] [--exclude=<tables>] [--auto_start] [--stop_after_copy] [--defer-secondary-keys] [--on-ddl=<ddl-action>] [--source_shards=<source_shards>] [--no-routing-rules] <action> 'action must be one of the following: Create, Complete, Cancel, SwitchTraffic, ReverseTrafffic, Show, or Progress' <targetKs.workflow>",
471471
help: `Move table(s) to another keyspace, table_specs is a list of tables or the tables section of the vschema for the target keyspace. Example: '{"t1":{"column_vindexes": [{"column": "id1", "name": "hash"}]}, "t2":{"column_vindexes": [{"column": "id2", "name": "hash"}]}}'. In the case of an unsharded target keyspace the vschema for each table may be empty. Example: '{"t1":{}, "t2":{}}'.`,
472472
},
473473
{
@@ -2125,6 +2125,7 @@ func commandVRWorkflow(ctx context.Context, wr *wrangler.Wrangler, subFlags *pfl
21252125

21262126
// MoveTables-only params
21272127
renameTables := subFlags.Bool("rename_tables", false, "MoveTables only. Rename tables instead of dropping them. --rename_tables is only supported for Complete.")
2128+
noRoutingRules := subFlags.Bool("no-routing-rules", false, "(Advanced) MoveTables Create only. Do not create routing rules while creating the workflow. See the reference documentation for limitations if you use this flag.")
21282129

21292130
// MoveTables and Reshard params
21302131
sourceShards := subFlags.String("source_shards", "", "Source shards")
@@ -2262,6 +2263,7 @@ func commandVRWorkflow(ctx context.Context, wr *wrangler.Wrangler, subFlags *pfl
22622263
vrwp.ExternalCluster = externalClusterName
22632264
vrwp.SourceTimeZone = *sourceTimeZone
22642265
vrwp.DropForeignKeys = *dropForeignKeys
2266+
vrwp.NoRoutingRules = *noRoutingRules
22652267
if *sourceShards != "" {
22662268
vrwp.SourceShards = strings.Split(*sourceShards, ",")
22672269
}

‎go/vt/wrangler/materializer.go

Lines changed: 31 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,8 @@ func shouldInclude(table string, excludes []string) bool {
122122
// MoveTables initiates moving table(s) over to another keyspace
123123
func (wr *Wrangler) MoveTables(ctx context.Context, workflow, sourceKeyspace, targetKeyspace, tableSpecs,
124124
cell, tabletTypes string, allTables bool, excludeTables string, autoStart, stopAfterCopy bool,
125-
externalCluster string, dropForeignKeys, deferSecondaryKeys bool, sourceTimeZone, onDDL string, sourceShards []string) error {
125+
externalCluster string, dropForeignKeys, deferSecondaryKeys bool, sourceTimeZone, onDDL string,
126+
sourceShards []string, noRoutingRules bool) error {
126127
//FIXME validate tableSpecs, allTables, excludeTables
127128
var tables []string
128129
var externalTopo *topo.Server
@@ -206,33 +207,37 @@ func (wr *Wrangler) MoveTables(ctx context.Context, workflow, sourceKeyspace, ta
206207
}
207208
}
208209
if externalTopo == nil {
209-
// Save routing rules before vschema. If we save vschema first, and routing rules
210-
// fails to save, we may generate duplicate table errors.
211-
rules, err := topotools.GetRoutingRules(ctx, wr.ts)
212-
if err != nil {
213-
return err
214-
}
215-
for _, table := range tables {
216-
toSource := []string{sourceKeyspace + "." + table}
217-
rules[table] = toSource
218-
rules[table+"@replica"] = toSource
219-
rules[table+"@rdonly"] = toSource
220-
rules[targetKeyspace+"."+table] = toSource
221-
rules[targetKeyspace+"."+table+"@replica"] = toSource
222-
rules[targetKeyspace+"."+table+"@rdonly"] = toSource
223-
rules[targetKeyspace+"."+table] = toSource
224-
rules[sourceKeyspace+"."+table+"@replica"] = toSource
225-
rules[sourceKeyspace+"."+table+"@rdonly"] = toSource
226-
}
227-
if err := topotools.SaveRoutingRules(ctx, wr.ts, rules); err != nil {
228-
return err
229-
}
230-
231-
if vschema != nil {
232-
// We added to the vschema.
233-
if err := wr.ts.SaveVSchema(ctx, targetKeyspace, vschema); err != nil {
210+
if noRoutingRules {
211+
log.Warningf("Found --no-routing-rules flag, not creating routing rules for workflow %s.%s", targetKeyspace, workflow)
212+
} else {
213+
// Save routing rules before vschema. If we save vschema first, and routing rules
214+
// fails to save, we may generate duplicate table errors.
215+
rules, err := topotools.GetRoutingRules(ctx, wr.ts)
216+
if err != nil {
217+
return err
218+
}
219+
for _, table := range tables {
220+
toSource := []string{sourceKeyspace + "." + table}
221+
rules[table] = toSource
222+
rules[table+"@replica"] = toSource
223+
rules[table+"@rdonly"] = toSource
224+
rules[targetKeyspace+"."+table] = toSource
225+
rules[targetKeyspace+"."+table+"@replica"] = toSource
226+
rules[targetKeyspace+"."+table+"@rdonly"] = toSource
227+
rules[targetKeyspace+"."+table] = toSource
228+
rules[sourceKeyspace+"."+table+"@replica"] = toSource
229+
rules[sourceKeyspace+"."+table+"@rdonly"] = toSource
230+
}
231+
if err := topotools.SaveRoutingRules(ctx, wr.ts, rules); err != nil {
234232
return err
235233
}
234+
235+
if vschema != nil {
236+
// We added to the vschema.
237+
if err := wr.ts.SaveVSchema(ctx, targetKeyspace, vschema); err != nil {
238+
return err
239+
}
240+
}
236241
}
237242
}
238243
if err := wr.ts.RebuildSrvVSchema(ctx, nil); err != nil {

‎go/vt/wrangler/materializer_test.go

Lines changed: 36 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,34 @@ const mzCheckJournal = "/select val from _vt.resharding_journal where id="
4646

4747
var defaultOnDDL = binlogdatapb.OnDDLAction_name[int32(binlogdatapb.OnDDLAction_IGNORE)]
4848

49+
// TestMoveTablesNoRoutingRules confirms that MoveTables does not create routing rules if --no-routing-rules is specified.
50+
func TestMoveTablesNoRoutingRules(t *testing.T) {
51+
ms := &vtctldatapb.MaterializeSettings{
52+
Workflow: "workflow",
53+
SourceKeyspace: "sourceks",
54+
TargetKeyspace: "targetks",
55+
TableSettings: []*vtctldatapb.TableMaterializeSettings{{
56+
TargetTable: "t1",
57+
SourceExpression: "select * from t1",
58+
}},
59+
}
60+
env := newTestMaterializerEnv(t, ms, []string{"0"}, []string{"0"})
61+
defer env.close()
62+
63+
env.tmc.expectVRQuery(100, mzCheckJournal, &sqltypes.Result{})
64+
env.tmc.expectVRQuery(200, mzSelectFrozenQuery, &sqltypes.Result{})
65+
env.tmc.expectVRQuery(200, insertPrefix, &sqltypes.Result{})
66+
env.tmc.expectVRQuery(200, mzSelectIDQuery, &sqltypes.Result{})
67+
env.tmc.expectVRQuery(200, mzUpdateQuery, &sqltypes.Result{})
68+
69+
ctx := context.Background()
70+
err := env.wr.MoveTables(ctx, "workflow", "sourceks", "targetks", "t1", "", "", false, "", true, false, "", false, false, "", defaultOnDDL, nil, true)
71+
require.NoError(t, err)
72+
rr, err := env.wr.ts.GetRoutingRules(ctx)
73+
require.NoError(t, err)
74+
require.Equal(t, 0, len(rr.Rules))
75+
}
76+
4977
func TestMigrateTables(t *testing.T) {
5078
ms := &vtctldatapb.MaterializeSettings{
5179
Workflow: "workflow",
@@ -66,7 +94,7 @@ func TestMigrateTables(t *testing.T) {
6694
env.tmc.expectVRQuery(200, mzUpdateQuery, &sqltypes.Result{})
6795

6896
ctx := context.Background()
69-
err := env.wr.MoveTables(ctx, "workflow", "sourceks", "targetks", "t1", "", "", false, "", true, false, "", false, false, "", defaultOnDDL, nil)
97+
err := env.wr.MoveTables(ctx, "workflow", "sourceks", "targetks", "t1", "", "", false, "", true, false, "", false, false, "", defaultOnDDL, nil, false)
7098
require.NoError(t, err)
7199
vschema, err := env.wr.ts.GetSrvVSchema(ctx, env.cell)
72100
require.NoError(t, err)
@@ -107,11 +135,11 @@ func TestMissingTables(t *testing.T) {
107135
env.tmc.expectVRQuery(200, mzUpdateQuery, &sqltypes.Result{})
108136

109137
ctx := context.Background()
110-
err := env.wr.MoveTables(ctx, "workflow", "sourceks", "targetks", "t1,tyt", "", "", false, "", true, false, "", false, false, "", defaultOnDDL, nil)
138+
err := env.wr.MoveTables(ctx, "workflow", "sourceks", "targetks", "t1,tyt", "", "", false, "", true, false, "", false, false, "", defaultOnDDL, nil, false)
111139
require.EqualError(t, err, "table(s) not found in source keyspace sourceks: tyt")
112-
err = env.wr.MoveTables(ctx, "workflow", "sourceks", "targetks", "t1,tyt,t2,txt", "", "", false, "", true, false, "", false, false, "", defaultOnDDL, nil)
140+
err = env.wr.MoveTables(ctx, "workflow", "sourceks", "targetks", "t1,tyt,t2,txt", "", "", false, "", true, false, "", false, false, "", defaultOnDDL, nil, false)
113141
require.EqualError(t, err, "table(s) not found in source keyspace sourceks: tyt,txt")
114-
err = env.wr.MoveTables(ctx, "workflow", "sourceks", "targetks", "t1", "", "", false, "", true, false, "", false, false, "", defaultOnDDL, nil)
142+
err = env.wr.MoveTables(ctx, "workflow", "sourceks", "targetks", "t1", "", "", false, "", true, false, "", false, false, "", defaultOnDDL, nil, false)
115143
require.NoError(t, err)
116144
}
117145

@@ -167,7 +195,7 @@ func TestMoveTablesAllAndExclude(t *testing.T) {
167195
env.tmc.expectVRQuery(200, insertPrefix, &sqltypes.Result{})
168196
env.tmc.expectVRQuery(200, mzSelectIDQuery, &sqltypes.Result{})
169197
env.tmc.expectVRQuery(200, mzUpdateQuery, &sqltypes.Result{})
170-
err = env.wr.MoveTables(ctx, "workflow", "sourceks", "targetks", "", "", "", tcase.allTables, tcase.excludeTables, true, false, "", false, false, "", defaultOnDDL, nil)
198+
err = env.wr.MoveTables(ctx, "workflow", "sourceks", "targetks", "", "", "", tcase.allTables, tcase.excludeTables, true, false, "", false, false, "", defaultOnDDL, nil, false)
171199
require.NoError(t, err)
172200
require.EqualValues(t, tcase.want, targetTables(env))
173201
})
@@ -201,7 +229,7 @@ func TestMoveTablesStopFlags(t *testing.T) {
201229
env.tmc.expectVRQuery(200, mzSelectIDQuery, &sqltypes.Result{})
202230
// -auto_start=false is tested by NOT expecting the update query which sets state to RUNNING
203231
err = env.wr.MoveTables(ctx, "workflow", "sourceks", "targetks", "t1", "",
204-
"", false, "", false, true, "", false, false, "", defaultOnDDL, nil)
232+
"", false, "", false, true, "", false, false, "", defaultOnDDL, nil, false)
205233
require.NoError(t, err)
206234
env.tmc.verifyQueries(t)
207235
})
@@ -227,7 +255,7 @@ func TestMigrateVSchema(t *testing.T) {
227255
env.tmc.expectVRQuery(200, mzUpdateQuery, &sqltypes.Result{})
228256

229257
ctx := context.Background()
230-
err := env.wr.MoveTables(ctx, "workflow", "sourceks", "targetks", `{"t1":{}}`, "", "", false, "", true, false, "", false, false, "", defaultOnDDL, nil)
258+
err := env.wr.MoveTables(ctx, "workflow", "sourceks", "targetks", `{"t1":{}}`, "", "", false, "", true, false, "", false, false, "", defaultOnDDL, nil, false)
231259
require.NoError(t, err)
232260
vschema, err := env.wr.ts.GetSrvVSchema(ctx, env.cell)
233261
require.NoError(t, err)
@@ -2828,7 +2856,7 @@ func TestMoveTablesDDLFlag(t *testing.T) {
28282856
env.tmc.expectVRQuery(200, mzUpdateQuery, &sqltypes.Result{})
28292857

28302858
err := env.wr.MoveTables(ctx, "workflow", "sourceks", "targetks", "t1", "",
2831-
"", false, "", false, true, "", false, false, "", onDDLAction, nil)
2859+
"", false, "", false, true, "", false, false, "", onDDLAction, nil, false)
28322860
require.NoError(t, err)
28332861
})
28342862
}

‎go/vt/wrangler/workflow.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,9 @@ type VReplicationWorkflowParams struct {
7171

7272
// Migrate specific
7373
ExternalCluster string
74+
75+
// MoveTables only
76+
NoRoutingRules bool
7477
}
7578

7679
// VReplicationWorkflow stores various internal objects for a workflow
@@ -433,7 +436,7 @@ func (vrw *VReplicationWorkflow) initMoveTables() error {
433436
return vrw.wr.MoveTables(vrw.ctx, vrw.params.Workflow, vrw.params.SourceKeyspace, vrw.params.TargetKeyspace,
434437
vrw.params.Tables, vrw.params.Cells, vrw.params.TabletTypes, vrw.params.AllTables, vrw.params.ExcludeTables,
435438
vrw.params.AutoStart, vrw.params.StopAfterCopy, vrw.params.ExternalCluster, vrw.params.DropForeignKeys,
436-
vrw.params.DeferSecondaryKeys, vrw.params.SourceTimeZone, vrw.params.OnDDL, vrw.params.SourceShards)
439+
vrw.params.DeferSecondaryKeys, vrw.params.SourceTimeZone, vrw.params.OnDDL, vrw.params.SourceShards, vrw.params.NoRoutingRules)
437440
}
438441

439442
func (vrw *VReplicationWorkflow) initReshard() error {

0 commit comments

Comments
 (0)