Skip to content

Commit 962c0c5

Browse files
pablogsalpablogsal
committed
gh-153569: simplify f-string span handling
1 parent e5f9a6c commit 962c0c5

6 files changed

Lines changed: 86 additions & 74 deletions

File tree

‎Parser/lexer/lexer.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -570,7 +570,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
570570
if (cursor_valid) {
571571
_PyLexer_update_ftstring_expr(tok, c);
572572
}
573-
if ((cursor_valid) && c != '{' && _PyLexer_set_ftstring_expr(tok, token, c)) {
573+
if (cursor_valid && c != '{' &&
574+
_PyLexer_set_ftstring_expr_metadata(tok, token)) {
574575
return MAKE_TOKEN(ERRORTOKEN);
575576
}
576577

‎Parser/lexer/lexer.h‎

Lines changed: 6 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,6 @@
55

66
int _PyTokenizer_Get(struct tok_state *, struct token *);
77

8-
static inline const char *
9-
_PyLexer_SpanView(const struct tok_state *tok, _PyTok_Span span,
10-
Py_ssize_t *length)
11-
{
12-
assert(length != NULL);
13-
assert(_PyTok_SpanIsValid(span));
14-
assert(tok->buf != NULL);
15-
assert(tok->inp >= tok->buf);
16-
if (span.start >= tok->buf_offset &&
17-
span.end - tok->buf_offset <= tok->inp - tok->buf) {
18-
*length = span.end - span.start;
19-
return tok->buf + (span.start - tok->buf_offset);
20-
}
21-
return _PyTok_SourceSpanView(&tok->source, span, length);
22-
}
23-
248
/* The view points into the current input window. The next
259
_PyTokenizer_Get() call may discard it. */
2610
static inline const char *
@@ -32,7 +16,12 @@ _PyToken_TextView(const struct tok_state *tok, const struct token *token,
3216
*length = 0;
3317
return "";
3418
}
35-
return _PyLexer_SpanView(tok, token->span, length);
19+
assert(tok->buf != NULL);
20+
assert(tok->inp >= tok->buf);
21+
assert(token->span.start >= tok->buf_offset);
22+
assert(token->span.end - tok->buf_offset <= tok->inp - tok->buf);
23+
*length = token->span.end - token->span.start;
24+
return tok->buf + (token->span.start - tok->buf_offset);
3625
}
3726

3827
#endif

‎Parser/lexer/lexer_internal.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ void _PyLexer_backup(struct tok_state *, int);
4949
void _PyLexer_update_ftstring_expr(struct tok_state *, char);
5050
int _PyLexer_record_ftstring_comment(
5151
struct tok_state *, const char *, const char *);
52-
int _PyLexer_set_ftstring_expr(struct tok_state *, struct token *, char);
52+
int _PyLexer_set_ftstring_expr_metadata(struct tok_state *, struct token *);
5353
int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int);
5454
int _PyLexer_scan_number(struct tok_state *, struct token *, int, int);
5555
int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int);

‎Parser/lexer/state.c‎

Lines changed: 1 addition & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -95,31 +95,11 @@ _PyToken_Init(struct token *token) {
9595
token->metadata = NULL;
9696
}
9797

98-
static inline _PyTok_Span
99-
buffer_span(const struct tok_state *tok, const char *start, const char *end)
100-
{
101-
if (start == NULL) {
102-
assert(end == NULL);
103-
return (_PyTok_Span){-1, -1};
104-
}
105-
assert(end != NULL);
106-
const char *base = tok->buf;
107-
assert(base != NULL);
108-
assert(tok->inp >= base);
109-
Py_ssize_t start_offset = start - base;
110-
Py_ssize_t end_offset = end - base;
111-
assert(start_offset >= 0 && start_offset <= end_offset);
112-
assert(end_offset <= tok->inp - base);
113-
assert(tok->buf_offset <= PY_SSIZE_T_MAX - end_offset);
114-
return _PyTok_SpanFromBounds(
115-
tok->buf_offset + start_offset, tok->buf_offset + end_offset);
116-
}
117-
11898
int
11999
_PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end)
120100
{
121101
token->level = tok->level;
122-
token->span = buffer_span(tok, start, end);
102+
token->span = _PyLexer_BufferSpan(tok, start, end);
123103
int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno;
124104
token->start_loc = (_PyTok_Loc){lineno, -1};
125105
token->end_loc = (_PyTok_Loc){tok->lineno, -1};

‎Parser/lexer/state.h‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ typedef struct _tokenizer_mode {
5454
_PyTok_Off multi_line_start;
5555
int first_line;
5656

57-
_PyTok_Span debug_expr;
57+
_PyTok_Span expr_span;
5858
int in_debug;
5959
int in_format_spec;
6060

@@ -65,7 +65,7 @@ typedef struct _tokenizer_comments {
6565
struct _tokenizer_comments *previous;
6666
Py_ssize_t count;
6767
Py_ssize_t capacity;
68-
int mode;
68+
int mode_index;
6969
_PyTok_Span spans[];
7070
} tokenizer_comments;
7171

@@ -133,6 +133,32 @@ struct tok_state {
133133
#endif
134134
};
135135

136+
static inline _PyTok_Off
137+
_PyLexer_BufferOffset(const struct tok_state *tok, const char *position)
138+
{
139+
assert(tok->buf != NULL);
140+
assert(tok->inp >= tok->buf);
141+
assert(position >= tok->buf && position <= tok->inp);
142+
Py_ssize_t offset = position - tok->buf;
143+
assert(tok->buf_offset <= PY_SSIZE_T_MAX - offset);
144+
return tok->buf_offset + offset;
145+
}
146+
147+
static inline _PyTok_Span
148+
_PyLexer_BufferSpan(const struct tok_state *tok, const char *start,
149+
const char *end)
150+
{
151+
if (start == NULL) {
152+
assert(end == NULL);
153+
return (_PyTok_Span){-1, -1};
154+
}
155+
assert(end != NULL);
156+
assert(start <= end);
157+
return _PyTok_SpanFromBounds(
158+
_PyLexer_BufferOffset(tok, start),
159+
_PyLexer_BufferOffset(tok, end));
160+
}
161+
136162
int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end);
137163

138164
struct tok_state *_PyTokenizer_tok_new(void);

‎Parser/lexer/string.c‎

Lines changed: 48 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,6 @@
77

88
#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
99

10-
static _PyTok_Off
11-
current_offset(const struct tok_state *tok, const char *position)
12-
{
13-
assert(position >= tok->buf && position <= tok->inp);
14-
return tok->buf_offset + (position - tok->buf);
15-
}
16-
1710
static char *
1811
offset_pointer(const struct tok_state *tok, _PyTok_Off offset)
1912
{
@@ -25,11 +18,25 @@ offset_pointer(const struct tok_state *tok, _PyTok_Off offset)
2518
return tok->source.bytes + offset;
2619
}
2720

21+
static const char *
22+
span_view(const struct tok_state *tok, _PyTok_Span span, Py_ssize_t *length)
23+
{
24+
assert(length != NULL);
25+
assert(_PyTok_SpanIsValid(span));
26+
if (span.start >= tok->buf_offset &&
27+
span.end - tok->buf_offset <= tok->inp - tok->buf) {
28+
*length = span.end - span.start;
29+
return tok->buf + (span.start - tok->buf_offset);
30+
}
31+
return _PyTok_SourceSpanView(&tok->source, span, length);
32+
}
33+
2834
static tokenizer_comments *
2935
current_comments(const struct tok_state *tok)
3036
{
3137
tokenizer_comments *comments = tok->ftstring_comments;
32-
return comments != NULL && comments->mode == tok->tok_mode_stack_index
38+
return comments != NULL &&
39+
comments->mode_index == tok->tok_mode_stack_index
3340
? comments : NULL;
3441
}
3542

@@ -38,10 +45,10 @@ _PyLexer_record_ftstring_comment(struct tok_state *tok, const char *start,
3845
const char *end)
3946
{
4047
tokenizer_mode *mode = TOK_GET_MODE(tok);
41-
if (mode->debug_expr.end >= 0) {
48+
if (mode->expr_span.end >= 0) {
4249
return 0;
4350
}
44-
assert(mode->debug_expr.start >= 0);
51+
assert(mode->expr_span.start >= 0);
4552
tokenizer_comments *comments = current_comments(tok);
4653
if (comments == NULL || comments->count == comments->capacity) {
4754
int create = comments == NULL;
@@ -64,55 +71,62 @@ _PyLexer_record_ftstring_comment(struct tok_state *tok, const char *start,
6471
if (create) {
6572
comments->previous = tok->ftstring_comments;
6673
comments->count = 0;
67-
comments->mode = tok->tok_mode_stack_index;
74+
comments->mode_index = tok->tok_mode_stack_index;
6875
}
6976
comments->capacity = capacity;
7077
tok->ftstring_comments = comments;
7178
}
72-
comments->spans[comments->count++] = (_PyTok_Span){
73-
current_offset(tok, start), current_offset(tok, end)};
79+
comments->spans[comments->count++] =
80+
_PyLexer_BufferSpan(tok, start, end);
7481
return 0;
7582
}
7683

7784
int
78-
_PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
85+
_PyLexer_set_ftstring_expr_metadata(struct tok_state *tok, struct token *token)
86+
{
7987
assert(token != NULL);
80-
assert(c == '}' || c == ':' || c == '!');
8188
tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
8289

8390
if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) {
8491
return 0;
8592
}
8693
Py_ssize_t expr_len;
87-
const char *expr = _PyLexer_SpanView(
88-
tok, tok_mode->debug_expr, &expr_len);
94+
const char *expr = span_view(tok, tok_mode->expr_span, &expr_len);
8995
if (expr == NULL) {
9096
return -1;
9197
}
9298
tokenizer_comments *comments = current_comments(tok);
9399
PyObject *res;
94100
if (comments != NULL && comments->count > 0) {
95-
char *stripped = PyMem_Malloc((size_t)expr_len);
101+
Py_ssize_t stripped_size = expr_len;
102+
_PyTok_Off previous_end = tok_mode->expr_span.start;
103+
for (Py_ssize_t i = 0; i < comments->count; i++) {
104+
_PyTok_Span comment = comments->spans[i];
105+
assert(_PyTok_SpanIsValid(comment));
106+
assert(comment.start >= previous_end);
107+
assert(comment.end <= tok_mode->expr_span.end);
108+
stripped_size -= comment.end - comment.start;
109+
previous_end = comment.end;
110+
}
111+
char *stripped = PyMem_Malloc((size_t)stripped_size);
96112
if (stripped == NULL) {
97113
PyErr_NoMemory();
98114
return -1;
99115
}
100-
_PyTok_Off copied_to = tok_mode->debug_expr.start;
116+
_PyTok_Off copied_to = tok_mode->expr_span.start;
101117
Py_ssize_t stripped_len = 0;
102118
for (Py_ssize_t i = 0; i < comments->count; i++) {
103119
_PyTok_Span comment = comments->spans[i];
104-
assert(comment.start >= copied_to);
105-
assert(comment.end <= tok_mode->debug_expr.end);
106120
Py_ssize_t length = comment.start - copied_to;
107121
memcpy(stripped + stripped_len,
108-
expr + copied_to - tok_mode->debug_expr.start,
122+
expr + copied_to - tok_mode->expr_span.start,
109123
(size_t)length);
110124
stripped_len += length;
111125
copied_to = comment.end;
112126
}
113-
Py_ssize_t length = tok_mode->debug_expr.end - copied_to;
127+
Py_ssize_t length = tok_mode->expr_span.end - copied_to;
114128
memcpy(stripped + stripped_len,
115-
expr + copied_to - tok_mode->debug_expr.start,
129+
expr + copied_to - tok_mode->expr_span.start,
116130
(size_t)length);
117131
stripped_len += length;
118132
res = PyUnicode_DecodeUTF8(stripped, stripped_len, NULL);
@@ -136,20 +150,21 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
136150

137151
switch (cur) {
138152
case '{':
139-
tok_mode->debug_expr = (_PyTok_Span){
140-
current_offset(tok, tok->cur), -1};
153+
tok_mode->expr_span = (_PyTok_Span){
154+
_PyLexer_BufferOffset(tok, tok->cur), -1};
141155
tokenizer_comments *comments = current_comments(tok);
142156
if (comments != NULL) {
143157
comments->count = 0;
144158
}
145159
break;
146160
case '}':
147161
case '!':
148-
tok_mode->debug_expr.end = current_offset(tok, tok->start);
162+
tok_mode->expr_span.end = _PyLexer_BufferOffset(tok, tok->start);
149163
break;
150164
case ':':
151-
if (tok_mode->debug_expr.end < 0) {
152-
tok_mode->debug_expr.end = current_offset(tok, tok->start);
165+
if (tok_mode->expr_span.end < 0) {
166+
tok_mode->expr_span.end =
167+
_PyLexer_BufferOffset(tok, tok->start);
153168
}
154169
break;
155170
default:
@@ -245,10 +260,11 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c)
245260
the_current_tok->kind = TOK_FSTRING_MODE;
246261
the_current_tok->quote = quote;
247262
the_current_tok->quote_size = quote_size;
248-
the_current_tok->start = current_offset(tok, tok->start);
249-
the_current_tok->multi_line_start = current_offset(tok, tok->line_start);
263+
the_current_tok->start = _PyLexer_BufferOffset(tok, tok->start);
264+
the_current_tok->multi_line_start =
265+
_PyLexer_BufferOffset(tok, tok->line_start);
250266
the_current_tok->first_line = tok->lineno;
251-
the_current_tok->debug_expr = (_PyTok_Span){-1, -1};
267+
the_current_tok->expr_span = (_PyTok_Span){-1, -1};
252268
the_current_tok->in_format_spec = 0;
253269
the_current_tok->in_debug = 0;
254270

0 commit comments

Comments
 (0)