py/lexer: Support raw f-strings.

Support for raw str/bytes already exists, and extending that to raw
f-strings is easy.  It also reduces code size because it eliminates an
error message.

Signed-off-by: Damien George <[email protected]>

Author: dpgeorge

py/lexer.c

@@ -661,21 +661,19 @@ void mp_lexer_to_next(mp_lexer_t *lex) {
                 }
                 #if MICROPY_PY_FSTRINGS
                 if (is_char_following(lex, 'f')) {
-                    // raw-f-strings unsupported, immediately return (invalid) token.
-                    lex->tok_kind = MP_TOKEN_FSTRING_RAW;
-                    break;
+                    is_fstring = true;
+                    n_char = 2;
                 }
                 #endif
             }
             #if MICROPY_PY_FSTRINGS
             else if (is_char(lex, 'f')) {
+                is_fstring = true;
+                n_char = 1;
                 if (is_char_following(lex, 'r')) {
-                    // raw-f-strings unsupported, immediately return (invalid) token.
-                    lex->tok_kind = MP_TOKEN_FSTRING_RAW;
-                    break;
+                    is_raw = true;
+                    n_char = 2;
                 }
-                n_char = 1;
-                is_fstring = true;
             }
             #endif
 

py/lexer.h

@@ -46,7 +46,6 @@ typedef enum _mp_token_kind_t {
     MP_TOKEN_LONELY_STRING_OPEN,
     #if MICROPY_PY_FSTRINGS
     MP_TOKEN_MALFORMED_FSTRING,
-    MP_TOKEN_FSTRING_RAW,
     #endif
 
     MP_TOKEN_NEWLINE,

py/parse.c

@@ -1351,9 +1351,6 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
         } else if (lex->tok_kind == MP_TOKEN_MALFORMED_FSTRING) {
             exc = mp_obj_new_exception_msg(&mp_type_SyntaxError,
                 MP_ERROR_TEXT("malformed f-string"));
-        } else if (lex->tok_kind == MP_TOKEN_FSTRING_RAW) {
-            exc = mp_obj_new_exception_msg(&mp_type_SyntaxError,
-                MP_ERROR_TEXT("raw f-strings are not supported"));
         #endif
         } else {
             exc = mp_obj_new_exception_msg(&mp_type_SyntaxError,

tests/basics/string_fstring.py

@@ -75,3 +75,7 @@ def foo(a, b):
     f"cd---------------------------------"
     f"e{y}f---------------------------------"
 )
+
+# Raw f-strings.
+print(rf"\r\a\w {'f'} \s\t\r\i\n\g")
+print(fr"\r{x}")

tests/cmdline/cmd_parsetree.py.exp

@@ -1,6 +1,6 @@
 ----------------
 [   1] file_input_2(1) (n=10)
-         tok(6)
+         tok(5)
 [   4]   \(rule\|for_stmt\)(22) (n=4)
            id(i)
 [   4]     \(rule\|atom_paren\)(45) (n=1)
@@ -9,7 +9,7 @@
            NULL
 [   6]   \(rule\|expr_stmt\)(5) (n=2)
            id(a)
-           tok(16)
+           tok(15)
 [   7]   \(rule\|expr_stmt\)(5) (n=2)
            id(b)
            str(str)