Skip to content

Commit b994f3f

Browse files
gh-98820: Fix quadratic time in csv.Sniffer for quoted fields
The regular expressions which look for a quoted field matched its body lazily, so a closing quote which was not followed by a delimiter was retried with every following quote, to the end of the sample. Match the body possessively instead: it ends at the first quote which is not doubled, as it does for a reader.
1 parent 1088266 commit b994f3f

3 files changed

Lines changed: 19 additions & 5 deletions

File tree

Lib/csv.py

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -280,12 +280,16 @@ def _guess_quote_and_delimiter(self, data, delimiters):
280280
"""
281281
import re
282282

283+
# The body of a quoted field ends at the first quote which is
284+
# not doubled, as it does for a reader. A lazy ".*?" scans to
285+
# the end of the sample instead, from every start: quadratically.
286+
body = r'(?:(?P=quote){2}|(?!(?P=quote)).)*+'
283287
matches = []
284-
for restr in (r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?P=delim)', # ,".*?",
285-
r'(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?P<delim>[^\w\n"\'])(?P<space> ?)', # ".*?",
286-
r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\']).*?(?P=quote)(?:$|\r|\n)', # ,".*?"
287-
r'(?:^|\n)(?P<quote>["\']).*?(?P=quote)(?:$|\r|\n)'): # ".*?" (no delim, no space)
288-
regexp = re.compile(restr, re.DOTALL | re.MULTILINE)
288+
for restr in (r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\'])%s(?P=quote)(?P=delim)', # ,"...",
289+
r'(?:^|\n)(?P<quote>["\'])%s(?P=quote)(?P<delim>[^\w\n"\'])(?P<space> ?)', # "...",
290+
r'(?P<delim>[^\w\n"\'])(?P<space> ?)(?P<quote>["\'])%s(?P=quote)(?:$|\r|\n)', # ,"..."
291+
r'(?:^|\n)(?P<quote>["\'])%s(?P=quote)(?:$|\r|\n)'): # "..." (no delim, no space)
292+
regexp = re.compile(restr % body, re.DOTALL | re.MULTILINE)
289293
matches = regexp.findall(data)
290294
if matches:
291295
break

Lib/test/test_csv.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1564,6 +1564,14 @@ def test_zero_mode_tie_order_colon_first(self):
15641564
sniffer.sniff(sample)
15651565

15661566

1567+
def test_sniff_quoted_single_column(self):
1568+
# gh-98820: this sample used to take minutes.
1569+
sniffer = csv.Sniffer()
1570+
sample = '"abcdefghijklmnopqrstuvwxyz"\n' * 10000
1571+
with self.assertRaisesRegex(csv.Error, "Could not determine delimiter"):
1572+
sniffer.sniff(sample, delimiters=',:|\t')
1573+
1574+
15671575
class NUL:
15681576
def write(s, *args):
15691577
pass
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix quadratic time in :meth:`csv.Sniffer.sniff` for a sample which contains
2+
quoted fields, in particular for a single column of quoted fields.

0 commit comments

Comments
 (0)