Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix load large data duplicates #148 #154

Merged
merged 1 commit into from
Jan 16, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions bin/data/csv10.ctl
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
output = import_duplicate_data_test
WRITER = DIRECT
TYPE = CSV
QUOTE = "\""
ESCAPE = \

CHECK_CONSTRAINTS = YES
PARSE_ERRORS = INFINITE
DUPLICATE_ERRORS = INFINITE
300 changes: 300 additions & 0 deletions bin/data/data11.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,300 @@
1,2,3
2,3,4
3,4,5
4,5,6
5,6,7
6,7,8
7,8,9
8,9,10
9,10,11
10,11,12
11,12,13
12,13,14
13,14,15
14,15,16
15,16,17
16,17,18
17,18,19
18,19,20
19,20,21
20,21,22
21,22,23
22,23,24
23,24,25
24,25,26
25,26,27
26,27,28
27,28,29
28,29,30
29,30,31
30,31,32
31,32,33
32,33,34
33,34,35
34,35,36
35,36,37
36,37,38
37,38,39
38,39,40
39,40,41
40,41,42
41,42,43
42,43,44
43,44,45
44,45,46
45,46,47
46,47,48
47,48,49
48,49,50
49,50,51
50,51,52
51,52,53
52,53,54
53,54,55
54,55,56
55,56,57
56,57,58
57,58,59
58,59,60
59,60,61
60,61,62
61,62,63
62,63,64
63,64,65
64,65,66
65,66,67
66,67,68
67,68,69
68,69,70
69,70,71
70,71,72
71,72,73
72,73,74
73,74,75
74,75,76
75,76,77
76,77,78
77,78,79
78,79,80
79,80,81
80,81,82
81,82,83
82,83,84
83,84,85
84,85,86
85,86,87
86,87,88
87,88,89
88,89,90
89,90,91
90,91,92
91,92,93
92,93,94
93,94,95
94,95,96
95,96,97
96,97,98
97,98,99
98,99,100
99,100,101
100,101,102
101,102,103
102,103,104
103,104,105
104,105,106
105,106,107
106,107,108
107,108,109
108,109,110
109,110,111
110,111,112
111,112,113
112,113,114
113,114,115
114,115,116
115,116,117
116,117,118
117,118,119
118,119,120
119,120,121
120,121,122
121,122,123
122,123,124
123,124,125
124,125,126
125,126,127
126,127,128
127,128,129
128,129,130
129,130,131
130,131,132
131,132,133
132,133,134
133,134,135
134,135,136
135,136,137
136,137,138
137,138,139
138,139,140
139,140,141
140,141,142
141,142,143
142,143,144
143,144,145
144,145,146
145,146,147
146,147,148
147,148,149
148,149,150
149,150,151
150,151,152
151,152,153
152,153,154
153,154,155
154,155,156
155,156,157
156,157,158
157,158,159
158,159,160
159,160,161
160,161,162
161,162,163
162,163,164
163,164,165
164,165,166
165,166,167
166,167,168
167,168,169
168,169,170
169,170,171
170,171,172
171,172,173
172,173,174
173,174,175
174,175,176
175,176,177
176,177,178
177,178,179
178,179,180
179,180,181
180,181,182
181,182,183
182,183,184
183,184,185
184,185,186
185,186,187
186,187,188
187,188,189
188,189,190
189,190,191
190,191,192
191,192,193
192,193,194
193,194,195
194,195,196
195,196,197
196,197,198
197,198,199
198,199,200
199,200,201
200,201,202
201,202,203
202,203,204
203,204,205
204,205,206
205,206,207
206,207,208
207,208,209
208,209,210
209,210,211
210,211,212
211,212,213
212,213,214
213,214,215
214,215,216
215,216,217
216,217,218
217,218,219
218,219,220
219,220,221
220,221,222
221,222,223
222,223,224
223,224,225
224,225,226
225,226,227
226,227,228
227,228,229
228,229,230
229,230,231
230,231,232
231,232,233
232,233,234
233,234,235
234,235,236
235,236,237
236,237,238
237,238,239
238,239,240
239,240,241
240,241,242
241,242,243
242,243,244
243,244,245
244,245,246
245,246,247
246,247,248
247,248,249
248,249,250
249,250,251
250,251,252
251,252,253
252,253,254
253,254,255
254,255,256
255,256,257
256,257,258
257,258,259
258,259,260
259,260,261
260,261,262
261,262,263
262,263,264
263,264,265
264,265,266
265,266,267
266,267,268
267,268,269
268,269,270
269,270,271
270,271,272
271,272,273
272,273,274
273,274,275
274,275,276
275,276,277
276,277,278
277,278,279
278,279,280
279,280,281
280,281,282
281,282,283
282,283,284
283,284,285
284,285,286
285,286,287
286,287,288
287,288,289
288,289,290
289,290,291
290,291,292
291,292,293
292,293,294
293,294,295
294,295,296
295,296,297
296,297,298
297,298,299
298,299,300
299,300,301
300,301,302
7 changes: 7 additions & 0 deletions bin/expected/init.out
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,13 @@ WARNING: hash indexes are not WAL-logged and their use is discouraged
CREATE INDEX idx_hash_fn ON customer USING hash ((abs(c_w_id) + c_d_id));
WARNING: hash indexes are not WAL-logged and their use is discouraged
---------------------------------------------------------------------------
-- load_check Import duplicate data
CREATE TABLE import_duplicate_data_test (
id1 int,
id2 int,
id3 int, UNIQUE(id1, id2, id3)
);
---------------------------------------------------------------------------
-- load_check test
CREATE TABLE master (
id int PRIMARY KEY,
Expand Down
7 changes: 7 additions & 0 deletions bin/expected/init_1.out
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,13 @@ CREATE INDEX idx_btree_fn ON customer USING btree ((abs(c_w_id) + c_d_id));
CREATE INDEX idx_hash ON customer USING hash (c_d_id);
CREATE INDEX idx_hash_fn ON customer USING hash ((abs(c_w_id) + c_d_id));
---------------------------------------------------------------------------
-- load_check Import duplicate data
CREATE TABLE import_duplicate_data_test (
id1 int,
id2 int,
id3 int, UNIQUE(id1, id2, id3)
);
---------------------------------------------------------------------------
-- load_check test
CREATE TABLE master (
id int PRIMARY KEY,
Expand Down
7 changes: 7 additions & 0 deletions bin/expected/init_2.out
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,13 @@ CREATE INDEX idx_btree_fn ON customer USING btree ((abs(c_w_id) + c_d_id));
CREATE INDEX idx_hash ON customer USING hash (c_d_id);
CREATE INDEX idx_hash_fn ON customer USING hash ((abs(c_w_id) + c_d_id));
---------------------------------------------------------------------------
-- load_check Import duplicate data
CREATE TABLE import_duplicate_data_test (
id1 int,
id2 int,
id3 int, UNIQUE(id1, id2, id3)
);
---------------------------------------------------------------------------
-- load_check test
CREATE TABLE master (
id int PRIMARY KEY,
Expand Down
7 changes: 7 additions & 0 deletions bin/expected/init_3.out
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,13 @@ CREATE INDEX idx_btree_fn ON customer USING btree ((abs(c_w_id) + c_d_id));
CREATE INDEX idx_hash ON customer USING hash (c_d_id);
CREATE INDEX idx_hash_fn ON customer USING hash ((abs(c_w_id) + c_d_id));
---------------------------------------------------------------------------
-- load_check Import duplicate data
CREATE TABLE import_duplicate_data_test (
id1 int,
id2 int,
id3 int, UNIQUE(id1, id2, id3)
);
---------------------------------------------------------------------------
-- load_csv test csv9
CREATE TABLE unique_tbl1 (i int, t text);
-- default is NULLS DISTINCT
Expand Down
Loading
Loading