Skip to content

Commit e176062

Browse files
pablogsalpablogsal
committed
gh-153569: Unify decoded tokenizer storage ownership
1 parent b5addc5 commit e176062

8 files changed

Lines changed: 191 additions & 96 deletions

File tree

‎Lib/test/test_capi/test_tokenizer.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@ class TokenizerTests(unittest.TestCase):
99
def test_source(self):
1010
_testinternalcapi.test_tokenizer_source()
1111

12+
def test_source_discard(self):
13+
_testinternalcapi.test_tokenizer_source_discard()
14+
1215
def test_cursor(self):
1316
_testinternalcapi.test_tokenizer_cursor()
1417

‎Modules/_testinternalcapi/tokenizer.c‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,28 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
195195
goto error;
196196
}
197197

198+
_PyTok_SourceDiscard(&source);
199+
if (check(_PyTok_SourceAppendLine(&source, "a\n", 2, 0) == 4,
200+
"wrong retained source offset") < 0 ||
201+
_PyTok_SourceLine(&source, 1, &line) < 0 ||
202+
check(line.start == 4 && line.end == 6,
203+
"wrong retained source line") < 0 ||
204+
_PyTok_SourceLocation(
205+
&source, 4, _PYTOK_AFFINITY_LEFT, &loc) < 0 ||
206+
check(loc.lineno == 1 && loc.byte_col == 0,
207+
"wrong retained source location") < 0) {
208+
goto error;
209+
}
210+
view = _PyTok_SourceSpanView(
211+
&source, _PyTok_SpanFromBounds(4, 5), &view_len);
212+
if (check(view != NULL && view_len == 1 && view[0] == 'a',
213+
"wrong retained source span") < 0 ||
214+
check_system_error(_PyTok_SourceSpanView(
215+
&source, _PyTok_SpanFromBounds(0, 1), &view_len) == NULL,
216+
"accepted discarded source span") < 0) {
217+
goto error;
218+
}
219+
198220
_PyTok_SourceClear(&source);
199221
Py_RETURN_NONE;
200222

@@ -296,6 +318,88 @@ test_tokenizer_cursor(PyObject *Py_UNUSED(module),
296318
}
297319
#endif
298320

321+
_PyTok_Off base = source.len;
322+
_PyTok_SourceDiscard(&source);
323+
if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 ||
324+
_PyTok_SourceAppendLine(&source, "cd", 2, 0) < 0) {
325+
goto error;
326+
}
327+
_PyTok_CursorInit(&cursor, &source);
328+
if (_PyTok_CursorSetLine(&cursor, 1) < 0 ||
329+
check(cursor.pos == base && _PyTok_CursorPeek(&cursor, 1) == 'b',
330+
"wrong retained cursor line") < 0 ||
331+
_PyTok_CursorSetLine(&cursor, 2) < 0 ||
332+
check(_PyTok_CursorAdvance(&cursor) == 'c',
333+
"wrong retained cursor byte") < 0 ||
334+
_PyTok_CursorSetOffset(&cursor, base + 5) < 0 ||
335+
check(cursor.lineno == 2 && _PyTok_CursorAdvance(&cursor) == EOF,
336+
"wrong retained cursor EOF") < 0) {
337+
goto error;
338+
}
339+
340+
_PyTok_SourceClear(&source);
341+
Py_RETURN_NONE;
342+
343+
error:
344+
_PyTok_SourceClear(&source);
345+
return NULL;
346+
}
347+
348+
static PyObject *
349+
test_tokenizer_source_discard(PyObject *Py_UNUSED(module),
350+
PyObject *Py_UNUSED(args))
351+
{
352+
_PyTok_SourceText source;
353+
_PyTok_SourceInit(&source);
354+
for (int i = 0; i < 260; i++) {
355+
if (_PyTok_SourceAppendLine(&source, "x\n", 2, 1) < 0) {
356+
goto error;
357+
}
358+
}
359+
char *bytes = source.bytes;
360+
_PyTok_Off capacity = source.cap;
361+
_PyTok_SourceDiscard(&source);
362+
if (check(source.base_offset == 520 && source.len == 0 &&
363+
source.nlines == 0 && source.bytes == bytes &&
364+
source.cap == capacity && source.bytes[0] == '\0',
365+
"discard did not preserve source allocation") < 0) {
366+
goto error;
367+
}
368+
for (int i = 0; i < 260; i++) {
369+
if (check(_PyTok_SourceAppendLine(&source, "y\n", 2, 0) == 520 + 2 * i,
370+
"wrong source offset after discard") < 0 ||
371+
check(!_PyTok_SourceLineIsImplicit(&source, i + 1),
372+
"discard preserved implicit newline flag") < 0) {
373+
goto error;
374+
}
375+
}
376+
if (check(source.bytes == bytes && source.cap == capacity,
377+
"discarded allocation was not reused") < 0) {
378+
goto error;
379+
}
380+
_PyTok_SourceDiscard(&source);
381+
if (check(_PyTok_SourceAppendLine(&source, "tail", 4, 0) == 1040,
382+
"wrong source offset after repeated discard") < 0) {
383+
goto error;
384+
}
385+
_PyTok_SourceDiscard(&source);
386+
if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) == 1044,
387+
"cannot append after discarding unterminated line") < 0) {
388+
goto error;
389+
}
390+
_PyTok_SourceDiscard(&source);
391+
source.base_offset = PY_SSIZE_T_MAX - 1;
392+
if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) < 0 &&
393+
PyErr_ExceptionMatches(PyExc_MemoryError),
394+
"accepted overflowing logical source offset") < 0) {
395+
goto error;
396+
}
397+
PyErr_Clear();
398+
if (check(source.len == 0 && source.nlines == 0 &&
399+
source.base_offset == PY_SSIZE_T_MAX - 1,
400+
"overflow changed retained source") < 0) {
401+
goto error;
402+
}
299403
_PyTok_SourceClear(&source);
300404
Py_RETURN_NONE;
301405

@@ -307,6 +411,7 @@ test_tokenizer_cursor(PyObject *Py_UNUSED(module),
307411
static PyMethodDef test_methods[] = {
308412
{"test_tokenizer_source", test_tokenizer_source, METH_NOARGS},
309413
{"test_tokenizer_cursor", test_tokenizer_cursor, METH_NOARGS},
414+
{"test_tokenizer_source_discard", test_tokenizer_source_discard, METH_NOARGS},
310415
{NULL},
311416
};
312417

‎Parser/tokenizer/cursor.c‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ _PyTok_CursorSetLine(_PyTok_Cursor *cursor, int lineno)
2323
if (lineno > 0 && cursor->lineno == lineno - 1 &&
2424
lineno <= source->nlines) {
2525
_PyTok_Off start = cursor->line_end;
26-
_PyTok_Off end = source->len;
26+
_PyTok_Off end = source->base_offset + source->len;
2727
if (lineno < source->nlines) {
2828
end = _PyTok_SourceFindLineEnd(source, start);
2929
if (end < 0) {
@@ -53,8 +53,9 @@ _PyTok_CursorSetOffset(_PyTok_Cursor *cursor, _PyTok_Off offset)
5353
int stays_on_line = cursor->lineno > 0 &&
5454
offset >= cursor->line_start && offset < cursor->line_end;
5555
if (!stays_on_line && cursor->lineno > 0 &&
56-
offset == cursor->line_end && offset == source->len &&
57-
(offset == 0 || source->bytes[offset - 1] != '\n')) {
56+
offset == cursor->line_end &&
57+
offset - source->base_offset == source->len &&
58+
(source->len == 0 || source->bytes[source->len - 1] != '\n')) {
5859
stays_on_line = 1;
5960
}
6061
if (stays_on_line) {
@@ -68,7 +69,7 @@ _PyTok_CursorSetOffset(_PyTok_Cursor *cursor, _PyTok_Off offset)
6869
return -1;
6970
}
7071
_PyTok_Off start = offset - loc.byte_col;
71-
_PyTok_Off end = source->len;
72+
_PyTok_Off end = source->base_offset + source->len;
7273
if (loc.lineno < source->nlines) {
7374
end = _PyTok_SourceFindLineEnd(source, start);
7475
if (end < 0) {

‎Parser/tokenizer/cursor.h‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,12 @@ PyAPI_FUNC(int) _PyTok_CursorSetOffset(_PyTok_Cursor *, _PyTok_Off);
2121
static inline void
2222
_PyTok_CursorInit(_PyTok_Cursor *cursor, const _PyTok_SourceText *source)
2323
{
24+
_PyTok_Off base = source != NULL ? source->base_offset : 0;
2425
*cursor = (_PyTok_Cursor){
2526
.source = source,
27+
.pos = base,
28+
.line_start = base,
29+
.line_end = base,
2630
};
2731
}
2832

@@ -35,14 +39,16 @@ _PyTok_CursorAdvance(_PyTok_Cursor *cursor)
3539
assert(cursor->source != NULL);
3640
assert(cursor->pos >= cursor->line_start);
3741
assert(cursor->pos <= cursor->line_end);
38-
assert(cursor->line_end <= cursor->source->len);
42+
assert(cursor->line_start >= cursor->source->base_offset);
43+
assert(cursor->line_end - cursor->source->base_offset <= cursor->source->len);
3944
if (cursor->pos >= cursor->line_end) {
4045
return EOF;
4146
}
4247
if (cursor->pos - cursor->line_start >= INT_MAX) {
4348
return EOF;
4449
}
45-
return Py_CHARMASK(cursor->source->bytes[cursor->pos++]);
50+
return Py_CHARMASK(cursor->source->bytes[
51+
cursor->pos++ - cursor->source->base_offset]);
4652
}
4753

4854
/* Return the byte at a nonnegative distance within the current line, or EOF
@@ -53,13 +59,15 @@ _PyTok_CursorPeek(const _PyTok_Cursor *cursor, int distance)
5359
assert(cursor->source != NULL);
5460
assert(cursor->pos >= cursor->line_start);
5561
assert(cursor->pos <= cursor->line_end);
56-
assert(cursor->line_end <= cursor->source->len);
62+
assert(cursor->line_start >= cursor->source->base_offset);
63+
assert(cursor->line_end - cursor->source->base_offset <= cursor->source->len);
5764
assert(distance >= 0);
5865
if (distance < 0 ||
5966
distance >= cursor->line_end - cursor->pos) {
6067
return EOF;
6168
}
62-
return Py_CHARMASK(cursor->source->bytes[cursor->pos + distance]);
69+
return Py_CHARMASK(cursor->source->bytes[
70+
cursor->pos - cursor->source->base_offset + distance]);
6371
}
6472

6573
#endif

‎Parser/tokenizer/reader.c‎

Lines changed: 14 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,6 @@ _PyTok_ReaderFree(struct tok_state *tok)
3434
}
3535
PyMem_Free(reader->file_buffer);
3636
PyMem_Free(reader->decoded);
37-
if (reader_is_streaming(reader->kind)) {
38-
PyMem_Free(tok->buf);
39-
}
4037
tok->buf = NULL;
4138
PyMem_Free(reader);
4239
tok->reader = NULL;
@@ -66,39 +63,6 @@ reserve_buffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed)
6663
return 0;
6764
}
6865

69-
static int
70-
reserve_input_buffer(struct tok_state *tok, Py_ssize_t needed)
71-
{
72-
_PyTok_Reader *reader = tok->reader;
73-
if (needed <= reader->input_buffer_cap) {
74-
return 0;
75-
}
76-
assert(tok->buf != NULL);
77-
assert(tok->cur >= tok->buf && tok->cur <= tok->inp);
78-
assert(tok->inp - tok->buf <= reader->input_buffer_cap);
79-
_PyLexer_BufferPointers pointers;
80-
_PyLexer_SaveBufferPointers(tok, tok->buf, &pointers);
81-
#ifdef Py_DEBUG
82-
char *buffer = NULL;
83-
Py_ssize_t capacity = reader->input_buffer_cap;
84-
if (reserve_buffer(&buffer, &capacity, needed) < 0) {
85-
return -1;
86-
}
87-
memcpy(buffer, tok->buf, (size_t)(tok->inp - tok->buf) + 1);
88-
memset(tok->buf, 0xDD, reader->input_buffer_cap);
89-
PyMem_Free(tok->buf);
90-
tok->buf = buffer;
91-
reader->input_buffer_cap = capacity;
92-
#else
93-
if (reserve_buffer(
94-
&tok->buf, &reader->input_buffer_cap, needed) < 0) {
95-
return -1;
96-
}
97-
#endif
98-
_PyLexer_RestoreBufferPointers(tok, tok->buf, &pointers);
99-
return 0;
100-
}
101-
10266
static int
10367
append_decoded(_PyTok_Reader *reader, const char *data, Py_ssize_t len)
10468
{
@@ -573,10 +537,10 @@ reset_streaming_buffer(struct tok_state *tok)
573537
{
574538
assert(tok->buf != NULL);
575539
assert(tok->cur >= tok->buf && tok->cur <= tok->inp);
576-
Py_ssize_t consumed = tok->inp - tok->buf;
577-
assert(tok->buf_offset <= PY_SSIZE_T_MAX - consumed);
578-
tok->buf_offset += consumed;
579-
tok->cur = tok->inp = tok->buf;
540+
_PyTok_SourceDiscard(&tok->source);
541+
tok->buf_offset = tok->source.base_offset;
542+
tok->buf = tok->cur = tok->inp = (char *)_PyTok_SourceData(&tok->source);
543+
tok->line_start = tok->buf;
580544
}
581545

582546
int
@@ -621,27 +585,10 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
621585
chunk.implicit_newline) {
622586
scan_len--;
623587
}
624-
if (streaming) {
625-
if (reset_buffer) {
588+
if (!prepared) {
589+
if (streaming && reset_buffer) {
626590
reset_streaming_buffer(tok);
627591
}
628-
Py_ssize_t used = tok->inp - tok->buf;
629-
int overflow = scan_len > PY_SSIZE_T_MAX - used - 1 ||
630-
tok->buf_offset > PY_SSIZE_T_MAX - used - scan_len;
631-
if (overflow) {
632-
PyErr_NoMemory();
633-
}
634-
if (overflow || reserve_input_buffer(tok, used + scan_len + 1) < 0) {
635-
_PyTok_ChunkClear(&chunk);
636-
tok->done = E_NOMEM;
637-
tok->input_error = 1;
638-
return 0;
639-
}
640-
memcpy(tok->inp, chunk.data, (size_t)scan_len);
641-
tok->inp += scan_len;
642-
*tok->inp = '\0';
643-
}
644-
else if (!prepared) {
645592
_PyLexer_BufferPointers pointers;
646593
if (!reset_buffer) {
647594
_PyLexer_SaveBufferPointers(
@@ -658,7 +605,8 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
658605
return 0;
659606
}
660607
if (reset_buffer) {
661-
tok->buf = tok->cur = tok->source.bytes + source_start;
608+
tok->buf = tok->cur =
609+
tok->source.bytes + (source_start - tok->source.base_offset);
662610
tok->buf_offset = source_start;
663611
tok->line_start = tok->buf;
664612
tok->start = NULL;
@@ -668,7 +616,8 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
668616
_PyLexer_RestoreBufferPointers(
669617
tok, tok->source.bytes, &pointers);
670618
}
671-
tok->inp = tok->source.bytes + source_start + scan_len;
619+
tok->inp = tok->source.bytes +
620+
(source_start - tok->source.base_offset) + scan_len;
672621
}
673622
if (tok->fp_interactive) {
674623
tok->interactive_src_start = tok->source.bytes;
@@ -677,7 +626,8 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
677626
if (prepared) {
678627
if (tok->start == NULL) {
679628
tok->buf = tok->cur;
680-
tok->buf_offset = chunk.data - tok->source.bytes;
629+
tok->buf_offset = tok->source.base_offset +
630+
(chunk.data - tok->source.bytes);
681631
}
682632
tok->inp = chunk.data + chunk.len;
683633
}
@@ -719,13 +669,8 @@ tokenizer_new_with_reader(_PyTok_ReaderKind kind)
719669
return tok;
720670
}
721671
if (reader_is_streaming(kind)) {
722-
if (reserve_buffer(
723-
&tok->buf, &tok->reader->input_buffer_cap, BUFSIZ) < 0) {
724-
_PyTokenizer_Free(tok);
725-
return NULL;
726-
}
727-
tok->cur = tok->inp = tok->buf;
728-
tok->buf[0] = '\0';
672+
tok->buf = tok->cur = tok->inp =
673+
(char *)_PyTok_SourceData(&tok->source);
729674
}
730675
return tok;
731676
}

‎Parser/tokenizer/reader_internal.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,6 @@ typedef struct _PyTok_Reader {
4444
PyObject *decoder;
4545
const char *nextprompt;
4646

47-
Py_ssize_t input_buffer_cap;
48-
4947
char *file_buffer;
5048
Py_ssize_t file_buffer_cap;
5149
_PyTok_Chunk prefetched_lines[2];

0 commit comments

Comments
 (0)