11#include "parts.h"
22
3- #include "../../Parser/tokenizer/cursor .h"
3+ #include "../../Parser/tokenizer/source .h"
44
55static int
66check (int condition , const char * message )
@@ -23,97 +23,31 @@ check_system_error(int failed, const char *message)
2323 return 0 ;
2424}
2525
26- static int
27- same_cursor (const _PyTok_Cursor * left , const _PyTok_Cursor * right )
28- {
29- return left -> source == right -> source &&
30- left -> pos == right -> pos &&
31- left -> line_start == right -> line_start &&
32- left -> line_end == right -> line_end &&
33- left -> lineno == right -> lineno ;
34- }
35-
3626static PyObject *
3727test_tokenizer_source (PyObject * Py_UNUSED (module ),
3828 PyObject * Py_UNUSED (args ))
3929{
4030 _PyTok_SourceText source ;
4131 _PyTok_SourceInit (& source );
4232
43- _PyTok_Loc loc ;
44- _PyTok_Line line ;
45- if (check (_PyTok_SourceLocation (
46- & source , 0 , _PYTOK_AFFINITY_RIGHT , & loc ) == 0 ,
47- "cannot locate empty source" ) < 0 ||
48- check (loc .lineno == 1 && loc .byte_col == 0 ,
49- "wrong empty source location" ) < 0 ||
50- check (_PyTok_SourceLine (& source , 1 , & line ) == 0 ,
51- "cannot find empty source line" ) < 0 ||
52- check (line .start == 0 && line .end == 0 ,
53- "wrong empty source line" ) < 0 ||
54- check_system_error (
55- _PyTok_SourceAppendLine (& source , "" , 0 , 0 ) < 0 ,
56- "accepted empty source line" ) < 0 ||
33+ if (check_system_error (
34+ _PyTok_SourceAppendLine (& source , "" , 0 , 0 ) < 0 ,
35+ "accepted empty source line" ) < 0 ||
5736 check_system_error (
5837 _PyTok_SourceAppendLine (& source , "a\nb\n" , 4 , 0 ) < 0 ,
5938 "accepted multiple source lines" ) < 0 ||
6039 check_system_error (
6140 _PyTok_SourceAppendLine (& source , "a" , 1 , 1 ) < 0 ,
62- "accepted missing implicit newline" ) < 0 ) {
63- goto error ;
64- }
65-
66- if (check (_PyTok_SourceAppendLine (& source , "alpha\n" , 6 , 0 ) == 0 ,
67- "wrong first source offset" ) < 0 ||
41+ "accepted missing implicit newline" ) < 0 ||
42+ check (_PyTok_SourceAppendLine (
43+ & source , "alpha\n" , 6 , 0 ) == 0 ,
44+ "wrong first source offset" ) < 0 ||
6845 check (_PyTok_SourceAppendLine (
6946 & source , "\xce\xb2\n" , 3 , 1 ) == 6 ,
7047 "wrong second source offset" ) < 0 ||
71- check (_PyTok_SourceAppendLine (
72- & source , "nul\0x\n" , 6 , 0 ) == 9 ,
73- "wrong third source offset" ) < 0 ) {
74- goto error ;
75- }
76-
77- int marker_line = 257 ;
78- int final_line = 300 ;
79- _PyTok_Off marker_start = -1 ;
80- for (int lineno = 4 ; lineno <= final_line ; lineno ++ ) {
81- const char * text = lineno == marker_line ? "marker\n" : "x\n" ;
82- Py_ssize_t len = (Py_ssize_t )strlen (text );
83- _PyTok_Off start = _PyTok_SourceAppendLine (
84- & source , text , len , lineno == final_line );
85- if (start < 0 ) {
86- goto error ;
87- }
88- if (lineno == marker_line ) {
89- marker_start = start ;
90- }
91- }
92-
93- if (check (source .nlines == final_line , "wrong source line count" ) < 0 ||
94- check (_PyTok_SourceLine (& source , marker_line , & line ) == 0 ,
95- "cannot find late source line" ) < 0 ||
96- check (line .start == marker_start &&
97- line .end == marker_start + 7 ,
98- "wrong late source line" ) < 0 ||
99- check (!line .implicit_newline && !line .contains_nul ,
100- "wrong late source flags" ) < 0 ||
101- check (_PyTok_SourceLine (& source , 2 , & line ) == 0 ,
102- "cannot find second source line" ) < 0 ||
103- check (line .start == 6 && line .end == 9 &&
104- line .implicit_newline && !line .contains_nul ,
105- "wrong second source line" ) < 0 ||
10648 check (!_PyTok_SourceLineIsImplicit (& source , 1 ) &&
10749 _PyTok_SourceLineIsImplicit (& source , 2 ),
108- "wrong early implicit newline flags" ) < 0 ||
109- check (_PyTok_SourceLine (& source , 3 , & line ) == 0 ,
110- "cannot find third source line" ) < 0 ||
111- check (line .contains_nul , "missing null byte flag" ) < 0 ||
112- check (_PyTok_SourceLine (& source , final_line , & line ) == 0 ,
113- "cannot find final source line" ) < 0 ||
114- check (line .implicit_newline &&
115- _PyTok_SourceLineIsImplicit (& source , final_line ),
116- "missing late implicit newline flag" ) < 0 ) {
50+ "wrong implicit newline flags" ) < 0 ) {
11751 goto error ;
11852 }
11953
@@ -123,178 +57,21 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
12357 if (check (view != NULL && view_len == 2 &&
12458 memcmp (view , "\xce\xb2" , 2 ) == 0 ,
12559 "wrong source span view" ) < 0 ||
126- check (_PyTok_SourceLocation (
127- & source , marker_start ,
128- _PYTOK_AFFINITY_LEFT , & loc ) == 0 ,
129- "cannot locate left line boundary" ) < 0 ||
130- check (loc .lineno == marker_line - 1 && loc .byte_col == 2 ,
131- "wrong left boundary location" ) < 0 ||
132- check (_PyTok_SourceLocation (
133- & source , marker_start ,
134- _PYTOK_AFFINITY_RIGHT , & loc ) == 0 ,
135- "cannot locate right line boundary" ) < 0 ||
136- check (loc .lineno == marker_line && loc .byte_col == 0 ,
137- "wrong right boundary location" ) < 0 ||
138- check (_PyTok_SourceLocation (
139- & source , marker_start + 1 ,
140- _PYTOK_AFFINITY_RIGHT , & loc ) == 0 ,
141- "cannot locate late source byte" ) < 0 ||
142- check (loc .lineno == marker_line && loc .byte_col == 1 ,
143- "wrong late source location" ) < 0 ) {
144- goto error ;
145- }
146-
147- if (check (_PyTok_SourceLocation (
148- & source , source .len , _PYTOK_AFFINITY_LEFT , & loc ) == 0 ,
149- "cannot locate left EOF" ) < 0 ||
150- check (loc .lineno == final_line && loc .byte_col == 2 ,
151- "wrong left EOF location" ) < 0 ||
152- check (_PyTok_SourceLocation (
153- & source , source .len ,
154- _PYTOK_AFFINITY_RIGHT , & loc ) == 0 ,
155- "cannot locate right EOF" ) < 0 ||
156- check (loc .lineno == final_line + 1 && loc .byte_col == 0 ,
157- "wrong right EOF location" ) < 0 ||
158- check (_PyTok_SourceLine (& source , final_line + 1 , & line ) == 0 ,
159- "cannot find virtual EOF line" ) < 0 ||
160- check (line .start == source .len && line .end == source .len ,
161- "wrong virtual EOF line" ) < 0 ||
162- check (!_PyTok_SourceLineIsImplicit (& source , 0 ) &&
163- !_PyTok_SourceLineIsImplicit (
164- & source , final_line + 1 ),
165- "virtual or invalid line is implicit" ) < 0 ) {
166- goto error ;
167- }
168-
169- view = _PyTok_SourceSpanView (
170- & source , _PyTok_SpanFromBounds (0 , source .len + 1 ), & view_len );
171- if (check_system_error (view == NULL , "accepted invalid source span" ) < 0 ||
17260 check_system_error (
173- _PyTok_SourceLocation (
174- & source , source .len + 1 ,
175- _PYTOK_AFFINITY_RIGHT , & loc ) < 0 ,
176- "accepted invalid source offset" ) < 0 ||
177- check_system_error (
178- _PyTok_SourceLine (& source , final_line + 2 , & line ) < 0 ,
179- "accepted invalid source line" ) < 0 ) {
61+ _PyTok_SourceSpanView (
62+ & source , _PyTok_SpanFromBounds (0 , source .len + 1 ),
63+ & view_len ) == NULL ,
64+ "accepted invalid source span" ) < 0 ) {
18065 goto error ;
18166 }
18267
18368 _PyTok_SourceClear (& source );
184- _PyTok_SourceInit (& source );
18569 if (_PyTok_SourceAppendLine (& source , "tail" , 4 , 0 ) < 0 ||
18670 check_system_error (
18771 _PyTok_SourceAppendLine (& source , "x\n" , 2 , 0 ) < 0 ,
188- "appended after unterminated source line" ) < 0 ||
189- check (_PyTok_SourceLocation (
190- & source , source .len ,
191- _PYTOK_AFFINITY_RIGHT , & loc ) == 0 ,
192- "cannot locate unterminated EOF" ) < 0 ||
193- check (loc .lineno == 1 && loc .byte_col == 4 ,
194- "wrong unterminated EOF location" ) < 0 ) {
195- goto error ;
196- }
197-
198- _PyTok_SourceClear (& source );
199- Py_RETURN_NONE ;
200-
201- error :
202- _PyTok_SourceClear (& source );
203- return NULL ;
204- }
205-
206- static PyObject *
207- test_tokenizer_cursor (PyObject * Py_UNUSED (module ),
208- PyObject * Py_UNUSED (args ))
209- {
210- _PyTok_SourceText source ;
211- _PyTok_SourceInit (& source );
212- if (_PyTok_SourceAppendLine (& source , "ab\n" , 3 , 0 ) < 0 ||
213- _PyTok_SourceAppendLine (& source , "cd\n" , 3 , 0 ) < 0 ) {
214- goto error ;
215- }
216-
217- _PyTok_Cursor cursor ;
218- _PyTok_CursorInit (& cursor , & source );
219- if (_PyTok_CursorSetOffset (& cursor , source .len ) < 0 ||
220- check (cursor .lineno == 3 && cursor .pos == source .len ,
221- "wrong cursor at virtual EOF" ) < 0 ||
222- _PyTok_CursorSetLine (& cursor , 1 ) < 0 ) {
223- goto error ;
224- }
225-
226- char large [BUFSIZ + 1 ];
227- memset (large , 'z' , sizeof (large ));
228- large [sizeof (large ) - 1 ] = '\n' ;
229- if (_PyTok_SourceAppendLine (& source , large , sizeof (large ), 0 ) < 0 ) {
230- goto error ;
231- }
232-
233- if (check (_PyTok_CursorPeek (& cursor , 0 ) == 'a' ,
234- "wrong cursor peek after relocation" ) < 0 ||
235- check (_PyTok_CursorPeek (& cursor , 1 ) == 'b' ,
236- "wrong distant cursor peek" ) < 0 ||
237- check (_PyTok_CursorAdvance (& cursor ) == 'a' ,
238- "wrong first cursor byte" ) < 0 ||
239- check (_PyTok_CursorAdvance (& cursor ) == 'b' ,
240- "wrong second cursor byte" ) < 0 ||
241- check (_PyTok_CursorAdvance (& cursor ) == '\n' ,
242- "wrong final cursor byte" ) < 0 ||
243- check (_PyTok_CursorAdvance (& cursor ) == EOF ,
244- "cursor advanced past line" ) < 0 ||
245- check (_PyTok_CursorSetOffset (& cursor , 2 ) == 0 ,
246- "cannot seek cursor offset" ) < 0 ||
247- check (_PyTok_CursorAdvance (& cursor ) == '\n' ,
248- "wrong cursor byte after seek" ) < 0 ||
249- check (_PyTok_CursorSetOffset (& cursor , 3 ) == 0 ,
250- "cannot seek line boundary" ) < 0 ||
251- check (cursor .lineno == 2 && cursor .line_start == 3 &&
252- _PyTok_CursorAdvance (& cursor ) == 'c' ,
253- "wrong cursor at line boundary" ) < 0 ||
254- check (_PyTok_CursorSetLine (& cursor , 3 ) == 0 ,
255- "cannot advance cursor to final line" ) < 0 ||
256- check (cursor .line_start == 6 &&
257- _PyTok_CursorAdvance (& cursor ) == 'z' ,
258- "wrong cursor byte on final line" ) < 0 ) {
259- goto error ;
260- }
261-
262- _PyTok_Cursor saved = cursor ;
263- if (check_system_error (
264- _PyTok_CursorSetOffset (& cursor , source .len + 1 ) < 0 ,
265- "accepted invalid cursor offset" ) < 0 ||
266- check (same_cursor (& cursor , & saved ),
267- "invalid offset changed cursor" ) < 0 ||
268- check_system_error (
269- _PyTok_CursorSetLine (& cursor , source .nlines + 2 ) < 0 ,
270- "accepted invalid cursor line" ) < 0 ||
271- check (same_cursor (& cursor , & saved ),
272- "invalid line changed cursor" ) < 0 ||
273- check (_PyTok_CursorSetOffset (& cursor , source .len ) == 0 ,
274- "cannot set cursor to EOF" ) < 0 ||
275- check (cursor .lineno == 4 && cursor .pos == source .len ,
276- "wrong cursor at EOF" ) < 0 ) {
277- goto error ;
278- }
279-
280- #if SIZEOF_VOID_P > 4
281- char byte = 0 ;
282- _PyTok_SourceText huge_source = {
283- .bytes = & byte ,
284- .len = (_PyTok_Off )INT_MAX + 1 ,
285- };
286- _PyTok_Cursor huge_cursor = {
287- .source = & huge_source ,
288- .pos = INT_MAX ,
289- .line_end = (_PyTok_Off )INT_MAX + 1 ,
290- .lineno = 1 ,
291- };
292- if (check (_PyTok_CursorAdvance (& huge_cursor ) == EOF &&
293- huge_cursor .pos == INT_MAX ,
294- "cursor advanced past maximum column" ) < 0 ) {
72+ "appended after unterminated source line" ) < 0 ) {
29573 goto error ;
29674 }
297- #endif
29875
29976 _PyTok_SourceClear (& source );
30077 Py_RETURN_NONE ;
@@ -306,7 +83,6 @@ test_tokenizer_cursor(PyObject *Py_UNUSED(module),
30683
30784static PyMethodDef test_methods [] = {
30885 {"test_tokenizer_source" , test_tokenizer_source , METH_NOARGS },
309- {"test_tokenizer_cursor" , test_tokenizer_cursor , METH_NOARGS },
31086 {NULL },
31187};
31288
0 commit comments