Skip to content

Commit b517280

Browse files
committed
gh-111487: Fix csv.Sniffer never testing the minimum delimiter consistency
1 parent 23b2b40 commit b517280

3 files changed

Lines changed: 29 additions & 8 deletions

File tree

Lib/csv.py

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -404,18 +404,19 @@ def _guess_delimiter(self, data, delimiters):
404404

405405
# build a list of possible delimiters
406406
modeList = modes.items()
407-
total = float(min(chunkLength * iteration, len(data)))
408-
# (rows of consistent data) / (number of rows) = 100%
409-
consistency = 1.0
410-
# minimum consistency threshold
411-
threshold = 0.9
412-
while len(delims) == 0 and consistency >= threshold:
407+
total = min(chunkLength * iteration, len(data))
408+
# (rows of consistent data) / (number of rows) = 100%, counted down
409+
# in whole percent to a minimum consistency threshold of 90%.
410+
# Integer arithmetic: subtracting 0.01 from a float stopped at
411+
# 0.9099999999999999, so the pass at the threshold never ran.
412+
for consistency in range(100, 89, -1):
413+
if delims:
414+
break
413415
for k, v in modeList:
414416
if v[0] > 0 and v[1] > 0:
415-
if ((v[1]/total) >= consistency and
417+
if (v[1] * 100 >= consistency * total and
416418
(delimiters is None or k in delimiters)):
417419
delims[k] = v
418-
consistency -= 0.01
419420

420421
if len(delims) == 1:
421422
delim = list(delims.keys())[0]

Lib/test/test_csv.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1521,6 +1521,21 @@ def test_guess_delimiter_crlf_not_chosen(self):
15211521
self.assertEqual(sniffer.sniff(sample).delimiter, "|")
15221522
self.assertNotEqual(sniffer.sniff(sample).delimiter, "\r")
15231523

1524+
def test_guess_delimiter_at_minimum_consistency(self):
1525+
# gh-111487: the consistency counter was stepped down by repeatedly
1526+
# subtracting 0.01 from a float, so it stopped at 0.9099999999999999
1527+
# and the pass at the documented 0.9 threshold never ran. Here ";" is
1528+
# modal on 19 of the 20 rows, which scores (19 - 1) / 20 == 90% once
1529+
# the non-matching row is subtracted -- exactly the threshold.
1530+
# Every row uses a different letter so nothing else is consistent, and
1531+
# the odd row sits in the first chunk, which keeps that chunk below the
1532+
# threshold and defers the decision to the full 20-row chunk.
1533+
rows = [(chr(ord('a') + i) + ';') * 3 + chr(ord('a') + i)
1534+
for i in range(20)]
1535+
rows[4] = 'e;e;e'
1536+
sniffer = csv.Sniffer()
1537+
self.assertEqual(sniffer.sniff('\n'.join(rows)).delimiter, ';')
1538+
15241539
def test_zero_mode_tie_order_independence(self):
15251540
sniffer = csv.Sniffer()
15261541
# ":" appears in half the rows (1, 0, 1, 0) - a tie between
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Fix :meth:`csv.Sniffer.sniff` never testing the documented minimum delimiter
2+
consistency of 90%. The consistency counter was stepped down by repeatedly
3+
subtracting ``0.01`` from a float, so it stopped at ``0.9099999999999999`` and
4+
the final pass at the threshold was skipped; it now counts down in whole
5+
percent using exact integer arithmetic.

0 commit comments

Comments
 (0)