Skip to content

Commit d86da8b

Browse files
gouhongshengouhongshen
authored
fix clone indexes when alter table. (#23257)
Fixing the ALTER TABLE with a full-text index will cause index data duplication. Approved by: @heni02, @ouyuanning
1 parent 5c109d4 commit d86da8b

File tree

4 files changed

+226
-4
lines changed

4 files changed

+226
-4
lines changed

‎pkg/sql/plan/bind_insert.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -521,7 +521,7 @@ func (builder *QueryBuilder) appendDedupAndMultiUpdateNodesForBindInsert(
521521
continue
522522
}
523523

524-
// clone mode
524+
// we will clone this index data to new index table, skip insert.
525525
if option != nil && option.SkipIndexesCopy[idxDef.IndexName] {
526526
continue
527527
}

‎pkg/sql/plan/build_dml_util.go‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,7 @@ func buildInsertPlans(
159159
ifNeedCheckPkDup := !builder.qry.LoadTag
160160
var indexSourceColTypes []*plan.Type
161161
var fuzzymessage *OriginTableMessageForFuzzy
162+
var skipIndexesCoyp map[string]bool
162163

163164
if v := builder.compCtx.GetContext().Value(defines.AlterCopyOpt{}); v != nil {
164165
dedupOpt := v.(*plan.AlterCopyOpt)
@@ -179,10 +180,13 @@ func buildInsertPlans(
179180
}
180181
}
181182
}
183+
skipIndexesCoyp = dedupOpt.SkipIndexesCopy
182184
}
183185
return buildInsertPlansWithRelatedHiddenTable(stmt, ctx, builder, insertBindCtx, objRef, tableDef,
184186
updateColLength, sourceStep, addAffectedRows, isFkRecursionCall, updatePkCol, pkFilterExpr,
185-
newPartitionExpr, ifExistAutoPkCol, ifNeedCheckPkDup, indexSourceColTypes, fuzzymessage, insertWithoutUniqueKeyMap, ifInsertFromUniqueColMap, nil)
187+
newPartitionExpr, ifExistAutoPkCol, ifNeedCheckPkDup, indexSourceColTypes, fuzzymessage,
188+
insertWithoutUniqueKeyMap, ifInsertFromUniqueColMap, nil, skipIndexesCoyp,
189+
)
186190
}
187191

188192
// buildUpdatePlans build update plan.
@@ -267,7 +271,7 @@ func buildUpdatePlans(ctx CompilerContext, builder *QueryBuilder, bindCtx *BindC
267271
return buildInsertPlansWithRelatedHiddenTable(nil, ctx, builder, insertBindCtx, updatePlanCtx.objRef, updatePlanCtx.tableDef,
268272
updatePlanCtx.updateColLength, sourceStep, addAffectedRows, updatePlanCtx.isFkRecursionCall, updatePlanCtx.updatePkCol,
269273
updatePlanCtx.pkFilterExprs, partitionExpr, ifExistAutoPkCol, ifNeedCheckPkDup, indexSourceColTypes, fuzzymessage, nil, nil,
270-
updatePlanCtx.updateColPosMap)
274+
updatePlanCtx.updateColPosMap, nil)
271275
}
272276

273277
func getStepByNodeId(builder *QueryBuilder, nodeId int32) int {
@@ -855,7 +859,7 @@ func buildInsertPlansWithRelatedHiddenTable(
855859
updatePkCol bool, pkFilterExprs []*Expr, partitionExpr *Expr, ifExistAutoPkCol bool,
856860
checkInsertPkDupForHiddenIndexTable bool, indexSourceColTypes []*plan.Type, fuzzymessage *OriginTableMessageForFuzzy,
857861
insertWithoutUniqueKeyMap map[string]bool, ifInsertFromUniqueColMap map[string]bool,
858-
updateColPosMap map[string]int,
862+
updateColPosMap map[string]int, skipIndexesCopy map[string]bool,
859863
) error {
860864
//var lastNodeId int32
861865
var err error
@@ -872,6 +876,11 @@ func buildInsertPlansWithRelatedHiddenTable(
872876
continue
873877
}
874878

879+
// we will clone this index data to new index table, skip insert.
880+
if skipIndexesCopy != nil && skipIndexesCopy[indexdef.IndexName] {
881+
continue
882+
}
883+
875884
// append plan for the hidden tables of unique/secondary keys
876885
if indexdef.TableExist && catalog.IsRegularIndexAlgo(indexdef.IndexAlgo) {
877886
/********

‎test/distributed/cases/snapshot/clone/clone_in_alter_table_2.result‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
SET experimental_fulltext_index = 1;
2+
SET experimental_ivf_index = 1;
3+
SET experimental_hnsw_index = 1;
4+
drop database if exists db;
5+
create database db;
6+
use db;
7+
DROP TABLE IF EXISTS stress_alter_table;
8+
CREATE TABLE `stress_alter_table` (
9+
`md5_id` int NOT NULL,
10+
`b` varchar(65535) DEFAULT NULL,
11+
`delete_flag` int DEFAULT NULL,
12+
PRIMARY KEY (`md5_id`),
13+
FULLTEXT `fb`(`b`) WITH PARSER ngram,
14+
KEY `delete_flag_idx` (`delete_flag`)
15+
);
16+
INSERT INTO stress_alter_table VALUES (1, 'this is a one', 0);
17+
DELETE FROM `stress_alter_table` WHERE delete_flag = 0;
18+
INSERT INTO `stress_alter_table` (`md5_id`,`b`,`delete_flag`) VALUES (1,'this is a one',0);
19+
ALTER TABLE `stress_alter_table` MODIFY COLUMN delete_flag INT DEFAULT 0;
20+
select mo_ctl('dn','checkpoint','');
21+
mo_ctl(dn, checkpoint, )
22+
{\n "method": "Checkpoint",\n "result": [\n {\n "returnStr": "OK"\n }\n ]\n}\n
23+
SELECT COUNT(*) FROM `stress_alter_table`;
24+
COUNT(*)
25+
1
26+
UPDATE `stress_alter_table` SET delete_flag = 1 WHERE md5_id = '1';
27+
SELECT COUNT(*) FROM `stress_alter_table`;
28+
COUNT(*)
29+
1
30+
drop table stress_alter_table;
31+
create table t1 (id bigint primary key, delete_flag int, a int, body varchar(10), title text, FULLTEXT (title, body), key(delete_flag));
32+
insert into t1 values(1,1,1, "body", "title");
33+
ALTER TABLE t1 MODIFY COLUMN a int DEFAULT NULL;
34+
SET @inner_sql = (
35+
SELECT GROUP_CONCAT(
36+
DISTINCT CONCAT(
37+
'SELECT ''', mi.index_table_name,
38+
''' AS index_table_name, COUNT(*) AS cnt FROM `',
39+
mi.index_table_name, '`'
40+
) SEPARATOR ' UNION ALL '
41+
)
42+
FROM mo_catalog.mo_indexes mi
43+
JOIN mo_catalog.mo_tables mt ON mi.table_id = mt.rel_id
44+
WHERE mt.relname = 't1'
45+
AND mt.reldatabase = 'db'
46+
AND mi.type IN ('MULTIPLE', 'UNIQUE')
47+
AND mi.index_table_name IS NOT NULL
48+
AND mi.index_table_name != ''
49+
AND mi.column_name <> 'question'
50+
AND mi.column_name <> 'keyword'
51+
);
52+
SET @sql = CONCAT(
53+
'SELECT * FROM (',
54+
@inner_sql,
55+
') AS t ORDER BY cnt DESC'
56+
);
57+
PREPARE stmt FROM @sql;
58+
EXECUTE stmt;
59+
index_table_name cnt
60+
__mo_index_secondary_019b0be5-956f-77a1-b31b-b1db0d19540e 3
61+
__mo_index_secondary_019b0be5-956f-77a8-923c-2ce77693d21a 1
62+
DEALLOCATE PREPARE stmt;
63+
drop table t1;
64+
create table t2 (id int primary key, a int, b int, c int, key(a), key(b));
65+
insert into t2 values(1,1,1,1);
66+
ALTER TABLE t2 MODIFY COLUMN c int DEFAULT NULL;
67+
SET @inner_sql = (
68+
SELECT GROUP_CONCAT(
69+
DISTINCT CONCAT(
70+
'SELECT ''', mi.index_table_name,
71+
''' AS index_table_name, COUNT(*) AS cnt FROM `',
72+
mi.index_table_name, '`'
73+
) SEPARATOR ' UNION ALL '
74+
)
75+
FROM mo_catalog.mo_indexes mi
76+
JOIN mo_catalog.mo_tables mt ON mi.table_id = mt.rel_id
77+
WHERE mt.relname = 't2'
78+
AND mt.reldatabase = 'db'
79+
AND mi.type IN ('MULTIPLE', 'UNIQUE')
80+
AND mi.index_table_name IS NOT NULL
81+
AND mi.index_table_name != ''
82+
AND mi.column_name <> 'question'
83+
AND mi.column_name <> 'keyword'
84+
);
85+
SET @sql = CONCAT(
86+
'SELECT * FROM (',
87+
@inner_sql,
88+
') AS t ORDER BY cnt DESC'
89+
);
90+
PREPARE stmt FROM @sql;
91+
EXECUTE stmt;
92+
index_table_name cnt
93+
__mo_index_secondary_019b0be5-95ad-764d-b6f8-03e4c66741c5 1
94+
__mo_index_secondary_019b0be5-95ad-7645-8b2b-2cac0abec44d 1
95+
DEALLOCATE PREPARE stmt;
96+
drop table t2;
97+
SET experimental_fulltext_index = 0;
98+
SET experimental_ivf_index = 0;
99+
SET experimental_hnsw_index = 0;
100+
drop database db;

‎test/distributed/cases/snapshot/clone/clone_in_alter_table_2.sql‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
SET experimental_fulltext_index = 1;
2+
SET experimental_ivf_index = 1;
3+
SET experimental_hnsw_index = 1;
4+
5+
drop database if exists db;
6+
create database db;
7+
use db;
8+
9+
-- case 1: primary table duplicate
10+
DROP TABLE IF EXISTS stress_alter_table;
11+
CREATE TABLE `stress_alter_table` (
12+
`md5_id` int NOT NULL,
13+
`b` varchar(65535) DEFAULT NULL,
14+
`delete_flag` int DEFAULT NULL,
15+
PRIMARY KEY (`md5_id`),
16+
FULLTEXT `fb`(`b`) WITH PARSER ngram,
17+
KEY `delete_flag_idx` (`delete_flag`)
18+
);
19+
20+
INSERT INTO stress_alter_table VALUES (1, 'this is a one', 0);
21+
22+
DELETE FROM `stress_alter_table` WHERE delete_flag = 0;
23+
INSERT INTO `stress_alter_table` (`md5_id`,`b`,`delete_flag`) VALUES (1,'this is a one',0);
24+
ALTER TABLE `stress_alter_table` MODIFY COLUMN delete_flag INT DEFAULT 0;
25+
-- @ignore:0
26+
select mo_ctl('dn','checkpoint','');
27+
SELECT COUNT(*) FROM `stress_alter_table`;
28+
UPDATE `stress_alter_table` SET delete_flag = 1 WHERE md5_id = '1';
29+
SELECT COUNT(*) FROM `stress_alter_table`;
30+
31+
drop table stress_alter_table;
32+
33+
-- case 2
34+
create table t1 (id bigint primary key, delete_flag int, a int, body varchar(10), title text, FULLTEXT (title, body), key(delete_flag));
35+
insert into t1 values(1,1,1, "body", "title");
36+
ALTER TABLE t1 MODIFY COLUMN a int DEFAULT NULL;
37+
38+
SET @inner_sql = (
39+
SELECT GROUP_CONCAT(
40+
DISTINCT CONCAT(
41+
'SELECT ''', mi.index_table_name,
42+
''' AS index_table_name, COUNT(*) AS cnt FROM `',
43+
mi.index_table_name, '`'
44+
) SEPARATOR ' UNION ALL '
45+
)
46+
FROM mo_catalog.mo_indexes mi
47+
JOIN mo_catalog.mo_tables mt ON mi.table_id = mt.rel_id
48+
WHERE mt.relname = 't1'
49+
AND mt.reldatabase = 'db'
50+
AND mi.type IN ('MULTIPLE', 'UNIQUE')
51+
AND mi.index_table_name IS NOT NULL
52+
AND mi.index_table_name != ''
53+
AND mi.column_name <> 'question'
54+
AND mi.column_name <> 'keyword'
55+
);
56+
57+
SET @sql = CONCAT(
58+
'SELECT * FROM (',
59+
@inner_sql,
60+
') AS t ORDER BY cnt DESC'
61+
);
62+
63+
PREPARE stmt FROM @sql;
64+
-- @ignore:0
65+
EXECUTE stmt;
66+
DEALLOCATE PREPARE stmt;
67+
68+
69+
drop table t1;
70+
71+
-- case 3
72+
73+
create table t2 (id int primary key, a int, b int, c int, key(a), key(b));
74+
insert into t2 values(1,1,1,1);
75+
ALTER TABLE t2 MODIFY COLUMN c int DEFAULT NULL;
76+
77+
SET @inner_sql = (
78+
SELECT GROUP_CONCAT(
79+
DISTINCT CONCAT(
80+
'SELECT ''', mi.index_table_name,
81+
''' AS index_table_name, COUNT(*) AS cnt FROM `',
82+
mi.index_table_name, '`'
83+
) SEPARATOR ' UNION ALL '
84+
)
85+
FROM mo_catalog.mo_indexes mi
86+
JOIN mo_catalog.mo_tables mt ON mi.table_id = mt.rel_id
87+
WHERE mt.relname = 't2'
88+
AND mt.reldatabase = 'db'
89+
AND mi.type IN ('MULTIPLE', 'UNIQUE')
90+
AND mi.index_table_name IS NOT NULL
91+
AND mi.index_table_name != ''
92+
AND mi.column_name <> 'question'
93+
AND mi.column_name <> 'keyword'
94+
);
95+
96+
SET @sql = CONCAT(
97+
'SELECT * FROM (',
98+
@inner_sql,
99+
') AS t ORDER BY cnt DESC'
100+
);
101+
102+
PREPARE stmt FROM @sql;
103+
-- @ignore:0
104+
EXECUTE stmt;
105+
DEALLOCATE PREPARE stmt;
106+
107+
drop table t2;
108+
109+
SET experimental_fulltext_index = 0;
110+
SET experimental_ivf_index = 0;
111+
SET experimental_hnsw_index = 0;
112+
113+
drop database db;

0 commit comments

Comments
 (0)