Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

load duplicate key support checking/nochecking/ignore/replace #20612

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

load duplicate key support checking/nochecking/ignore/replace #20612

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b65451a
feature: load duplicate key checking/nochecking syntax
huby2358 Dec 5, 2024
57eb743
support load duplicate opt
huby2358 Dec 6, 2024
5633ac4
fix: determineShuffleForJoin
huby2358 Dec 6, 2024
9972de7
fix
huby2358 Dec 19, 2024
266d196
test
huby2358 Jan 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion etc/launch-with-proxy/cn1.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,4 +42,7 @@ type = "distributed-tae"

[cn.frontend]
port = 16001
proxy-enabled = true
proxy-enabled = true

[cn.txn]
enable-leak-check = 1
5 changes: 4 additions & 1 deletion etc/launch-with-proxy/cn2.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,4 +43,7 @@ type = "distributed-tae"
[cn.frontend]
port = 16002
unix-socket = "/tmp/mysql2.sock"
proxy-enabled = true
proxy-enabled = true

[cn.txn]
enable-leak-check = 1
1 change: 1 addition & 0 deletions pkg/pb/plan/plan.pb.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 12 additions & 3 deletions pkg/sql/compile/compile.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1682,6 +1682,9 @@ func (c *Compile) compileExternScanParallelReadWrite(n *plan.Node, param *tree.E
}

parallelSize := GetExternParallelSize(fileSize[0], mcpu)
if parallelSize > 4 {
parallelSize = 4
}

var fileOffset [][]int64
for i := 0; i < len(fileList); i++ {
Expand All @@ -1697,11 +1700,17 @@ func (c *Compile) compileExternScanParallelReadWrite(n *plan.Node, param *tree.E
pre := 0
currentFirstFlag := c.anal.isFirst
for i := 0; i < len(c.cnList); i++ {
scope := c.constructScopeForExternal(c.cnList[i].Addr, param.Parallel)
count := min(parallelSize, ID2Addr[i])
var addr string
if i == 0 && parallelSize <= count {
addr = c.addr
} else {
addr = c.cnList[i].Addr
}
scope := c.constructScopeForExternal(addr, param.Parallel)
ss = append(ss, scope)
scope.IsLoad = true
count := min(parallelSize, ID2Addr[i])
scope.NodeInfo.Mcpu = count

fileOffsetTmp := make([]*pipeline.FileOffset, len(fileList))
for j := range fileOffsetTmp {
preIndex := pre
Expand Down
2 changes: 2 additions & 0 deletions pkg/sql/parsers/dialect/mysql/keywords.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -649,5 +649,7 @@ func init() {
"apply": APPLY,
"dedup": DEDUP,
"savepoint": SAVEPOINT,
"checking": CHECKING,
"nochecking": NOCHECKING,
}
}
Loading
Loading