gh-153569: enforce tokenizer buffer view lifetimes

Author: pablogsal

Lib/test/test_repl.py

@@ -198,6 +198,22 @@ def test_lexer_buffer_realloc_with_null_start(self):
         self.assertEqual(p.returncode, 0)
         self.assertIn(long_value, output)
 
+    @cpython_only
+    def test_multiline_fstring_source_reallocation(self):
+        long_line = " " * 9000 + "+ 2"
+        user_input = (
+            'value = f"""{(\n'
+            '1\n'
+            f'{long_line}\n'
+            ')}"""\n'
+            'print(value)\n'
+        )
+        p = spawn_repl()
+        p.stdin.write(user_input)
+        output = kill_python(p)
+        self.assertEqual(p.returncode, 0)
+        self.assertIn(">>> 3\n>>> ", output)
+
     def test_close_stdin(self):
         user_input = dedent('''
             import os

Lib/test/test_tokenize.py

@@ -2428,29 +2428,28 @@ def test_stop_iteration_skips_encoded_readline_codec_lookup(self):
         )
 
     def test_fstring_offsets_survive_buffer_reallocation(self):
-        padding = " " * 9000
-        expression_line = ")=:>{2}}\n"
-        physical_lines = [
-            'f"""\n',
-            "{(\n",
-            padding + "1\n",
-            expression_line,
-            '"""\n',
-        ]
-        source = "".join(physical_lines)
-        chunks = iter([
-            "".join(physical_lines[:2]),
-            "".join(physical_lines[2:4]),
-            physical_lines[4],
-            "",
-        ])
-
-        expected = self._get_tokens(source, extra_tokens=True)
-        tokens = list(tokenize._generate_tokens_from_c_tokenizer(
-            chunks.__next__,
-            extra_tokens=True,
-        ))
-        self.assertEqual(tokens, expected)
+        for prefix in ("f", "t"):
+            for extra_tokens in (False, True):
+                with self.subTest(prefix=prefix, extra_tokens=extra_tokens):
+                    physical_lines = [
+                        prefix + '"""\n',
+                        "{(\n",
+                        " " * 9000 + "1\n",
+                        ")=:>{2}}\n",
+                        '"""\n',
+                    ]
+                    source = "".join(physical_lines)
+                    chunks = iter([
+                        "".join(physical_lines[:2]),
+                        "".join(physical_lines[2:4]),
+                        physical_lines[4],
+                        "",
+                    ])
+                    expected = self._get_tokens(
+                        source, extra_tokens=extra_tokens)
+                    tokens = list(tokenize._generate_tokens_from_c_tokenizer(
+                        chunks.__next__, extra_tokens=extra_tokens))
+                    self.assertEqual(tokens, expected)
 
     def test_extra_tokens_relaxes_lexer_errors(self):
         cases = [

Parser/tokenizer/reader.c

@@ -78,10 +78,23 @@ reserve_input_buffer(struct tok_state *tok, Py_ssize_t needed)
     assert(tok->inp - tok->buf <= reader->input_buffer_cap);
     _PyLexer_BufferPointers pointers;
     _PyLexer_SaveBufferPointers(tok, tok->buf, &pointers);
+#ifdef Py_DEBUG
+    char *buffer = NULL;
+    Py_ssize_t capacity = reader->input_buffer_cap;
+    if (reserve_buffer(&buffer, &capacity, needed) < 0) {
+        return -1;
+    }
+    memcpy(buffer, tok->buf, (size_t)(tok->inp - tok->buf) + 1);
+    memset(tok->buf, 0xDD, reader->input_buffer_cap);
+    PyMem_Free(tok->buf);
+    tok->buf = buffer;
+    reader->input_buffer_cap = capacity;
+#else
     if (reserve_buffer(
             &tok->buf, &reader->input_buffer_cap, needed) < 0) {
         return -1;
     }
+#endif
     _PyLexer_RestoreBufferPointers(tok, tok->buf, &pointers);
     return 0;
 }
@@ -629,10 +642,8 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
         *tok->inp = '\0';
     }
     else if (!prepared) {
-        int source_will_grow =
-            chunk.len > tok->source.cap - tok->source.len - 1;
         _PyLexer_BufferPointers pointers;
-        if (!reset_buffer && source_will_grow) {
+        if (!reset_buffer) {
             _PyLexer_SaveBufferPointers(
                 tok, tok->source.bytes, &pointers);
         }
@@ -653,7 +664,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
             tok->start = NULL;
             tok->multi_line_start = NULL;
         }
-        else if (source_will_grow) {
+        else {
             _PyLexer_RestoreBufferPointers(
                 tok, tok->source.bytes, &pointers);
         }