From 13eb851b39380c448c3c68473c61d92ac654b51e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 2 Jan 2026 13:35:04 +0900 Subject: [PATCH 01/10] first implementation of d-string --- Lib/test/test_dstring.py | 47 ++++++++++ Lib/test/test_tokenize.py | 4 +- Lib/tokenize.py | 3 +- Parser/action_helpers.c | 180 ++++++++++++++++++++++++++++++++++++-- Parser/lexer/lexer.c | 59 ++++++++----- Parser/string_parser.c | 119 ++++++++++++++++++++++++- 6 files changed, 376 insertions(+), 36 deletions(-) create mode 100644 Lib/test/test_dstring.py diff --git a/Lib/test/test_dstring.py b/Lib/test/test_dstring.py new file mode 100644 index 000000000000000..09592980dee8c51 --- /dev/null +++ b/Lib/test/test_dstring.py @@ -0,0 +1,47 @@ +import unittest + + +class DStringTestCase(unittest.TestCase): + def assertAllRaise(self, exception_type, regex, error_strings): + for str in error_strings: + with self.subTest(str=str): + with self.assertRaisesRegex(exception_type, regex) as cm: + eval(str) + # print("Testing expression:", repr(str)) + # print(repr(cm.exception)) + # print(repr(cm.exception.text)) + + def test_single_quote(self): + exprs = [ + "d'hello'", + 'D"hello"', + "d'hello\\nworld'", + ] + self.assertAllRaise(SyntaxError, "d-string must be triple-quoted", exprs) + + def test_empty_dstring(self): + exprs = [ + "d''''''", + 'D""""""', + ] + self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + + def test_no_last_newline(self): + exprs = [ + "d'''\nhello world'''", + 'D"""\nhello world"""', + "df'''\nhello {42}'''", + ] + self.assertAllRaise(SyntaxError, "d-string must end with an indent line", exprs) + + def test_simple_dstring(self): + self.assertEqual(eval('d"""\n hello world\n """'), "hello world\n") + self.assertEqual(eval('d"""\n hello world\n """'), " hello world\n") + self.assertEqual(eval('d"""\n hello world\n"""'), " hello world\n") + self.assertEqual(eval('d"""\n hello world\\\n """'), " hello world") + self.assertEqual(eval('dr"""\n hello world\\\n """'), " hello world\\\n") + + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index ab53a20cff55392..9a878164489681a 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -3421,7 +3421,7 @@ def determine_valid_prefixes(): # some uppercase-only prefix is added. for letter in itertools.chain(string.ascii_lowercase, string.ascii_uppercase): try: - eval(f'{letter}""') + eval(f'{letter}"""\n"""') # d-string needs multiline single_char_valid_prefixes.add(letter.lower()) except SyntaxError: pass @@ -3445,7 +3445,7 @@ def determine_valid_prefixes(): # because it's a valid expression: not "" continue try: - eval(f'{p}""') + eval(f'{p}"""\n"""') # d-string needs multiline # No syntax error, so p is a valid string # prefix. diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 3545d92c4f5d7ff..3ea6c5b986ea78a 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -87,7 +87,8 @@ def _all_string_prefixes(): # The valid string prefixes. Only contain the lower case versions, # and don't contain any permutations (include 'fr', but not # 'rf'). The various permutations will be generated. - _valid_string_prefixes = ['b', 'r', 'u', 'f', 't', 'br', 'fr', 'tr'] + _valid_string_prefixes = ['b', 'r', 'u', 'f', 't', 'd', 'br', 'fr', 'tr', + 'bd', 'rd', 'fd', 'td', 'brd', 'frd', 'trd'] # if we add binary f-strings, add: ['fb', 'fbr'] result = {''} for prefix in _valid_string_prefixes: diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index c74eb34deb7c35c..7ea3b6136e032e1 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1295,24 +1295,124 @@ _PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq // Fstring stuff +static int +unicodewriter_write_line(Parser *p, PyUnicodeWriter *w, const char *line_start, const char *line_end, + int is_raw, Token* token) +{ + if (is_raw || memchr(line_start, '\\', line_end - line_start) == NULL) { + return PyUnicodeWriter_WriteUTF8(w, line_start, line_end - line_start); + } + else { + PyObject *line = _PyPegen_decode_string(p, 1, line_start, line_end - line_start, token); + if (line == NULL || PyUnicodeWriter_WriteStr(w, line) < 0) { + Py_XDECREF(line); + return -1; + } + Py_DECREF(line); + } + return 0; +} + +static PyObject* +_PyPegen_dedent_string_part(Parser *p, const char *s, size_t len, int indent_char, Py_ssize_t dedent_count, + int is_raw, int is_first, expr_ty constant, Token* token) +{ + Py_ssize_t lineno = constant->lineno; + const char *line_start = s; + const char *s_end = s + len; + + PyUnicodeWriter *w = PyUnicodeWriter_Create(len); + if (w == NULL) { + return NULL; + } + if (is_first) { + assert (line_start[0] == '\n'); + line_start++; // skip the first newline + } + else { + // Example: df""" + // first part {param} second part + // next line + // """" + // We don't need to dedent the first line in the non-first parts. + const char *line_end = memchr(line_start, '\n', s_end - line_start); + if (line_end) { + line_end++; // include the newline + } + else { + line_end = s_end; + } + if (unicodewriter_write_line(p, w, line_start, line_end, is_raw, token) < 0) { + PyUnicodeWriter_Discard(w); + return NULL; + } + line_start = line_end; + } + + while (line_start < s + len) { + lineno++; + + Py_ssize_t i = 0; + while (line_start + i < s_end && i < dedent_count && line_start[i] == indent_char) { + i++; + } + + if (line_start[i] == '\0') { // found an empty line without newline. + break; + } + if (line_start[i] == '\n') { // found an empty line with newline. + if (PyUnicodeWriter_WriteChar(w, '\n') < 0) { + PyUnicodeWriter_Discard(w); + return NULL; + } + line_start += i+1; + continue; + } + if (i < dedent_count) { // found an invalid indent. + assert(line_start[i] != indent_char); + PyUnicodeWriter_Discard(w); + RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, lineno, i, lineno, i+1, + "d-string line missing valid indentation"); + return NULL; + } + + // found a indented line. let's dedent it. + line_start += i; + const char *line_end = memchr(line_start, '\n', s_end - line_start); + if (line_end) { + line_end++; // include the newline + } + else { + line_end = s_end; + } + if (unicodewriter_write_line(p, w, line_start, line_end, is_raw, token) < 0) { + PyUnicodeWriter_Discard(w); + return NULL; + } + line_start = line_end; + } + return PyUnicodeWriter_Finish(w); +} + static expr_ty -_PyPegen_decode_fstring_part(Parser* p, int is_raw, expr_ty constant, Token* token) { +_PyPegen_decode_fstring_part(Parser* p, int is_first, int is_raw, int indent_char, Py_ssize_t dedent_count, expr_ty constant, Token* token) { assert(PyUnicode_CheckExact(constant->v.Constant.value)); const char* bstr = PyUnicode_AsUTF8(constant->v.Constant.value); if (bstr == NULL) { return NULL; } + is_raw = is_raw || strchr(bstr, '\\') == NULL; - size_t len; - if (strcmp(bstr, "{{") == 0 || strcmp(bstr, "}}") == 0) { - len = 1; - } else { - len = strlen(bstr); + PyObject *str = NULL; + if (dedent_count > 0) { + str = _PyPegen_dedent_string_part(p, bstr, strlen(bstr), indent_char, dedent_count, + is_raw, is_first, constant, token); + } + else { + str = _PyPegen_decode_string(p, is_raw, bstr, strlen(bstr), token); } - is_raw = is_raw || strchr(bstr, '\\') == NULL; - PyObject *str = _PyPegen_decode_string(p, is_raw, bstr, len, token); if (str == NULL) { _Pypegen_raise_decode_error(p); return NULL; @@ -1343,12 +1443,74 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b return NULL; } int is_raw = strpbrk(quote_str, "rR") != NULL; + int is_dedent = strpbrk(quote_str, "dD") != NULL; + int indent_char = 0; + Py_ssize_t indent_count = 0; asdl_expr_seq *seq = _Py_asdl_expr_seq_new(total_items, p->arena); if (seq == NULL) { return NULL; } + if (is_dedent) { + expr_ty first_item = asdl_seq_GET(raw_expressions, 0); + if (first_item->kind != Constant_kind + || PyUnicode_ReadChar(first_item->v.Constant.value, 0) != '\n') { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + first_item, + "d-string must start with a newline" + ); + return NULL; + } + + expr_ty last_item = asdl_seq_GET(raw_expressions, n_items - 1); + if (last_item->kind != Constant_kind) { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + last_item, + "d-string must end with an indent line" + ); + return NULL; + } + + Py_ssize_t blen; + const char *bstr = PyUnicode_AsUTF8AndSize(last_item->v.Constant.value, &blen); + if (bstr == NULL) { + return NULL; + } + + // memrchr is GNU extension; use manual loop for portability. + const char *lastline = bstr + blen; + while (bstr < lastline) { + if (lastline[-1] == '\n') { + break; + } + lastline--; + if (*lastline != ' ' && *lastline != '\t') { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + last_item, + "d-string must end with an indent line" + ); + return NULL; + } + } + + // checks indent of the last line. + indent_count = bstr + blen - lastline; + if (indent_count > 0) { + indent_char = lastline[0]; + + for (Py_ssize_t i = 1; i < indent_count; i++) { + if (lastline[i] != indent_char) { + RAISE_ERROR_KNOWN_LOCATION( + p, PyExc_TabError, last_item->end_lineno, i, last_item->end_lineno, i+1, + "inconsistent use of tabs and spaces in indentation" + ); + return NULL; + } + } + } + } + Py_ssize_t index = 0; for (Py_ssize_t i = 0; i < n_items; i++) { expr_ty item = asdl_seq_GET(raw_expressions, i); @@ -1380,7 +1542,7 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b } if (item->kind == Constant_kind) { - item = _PyPegen_decode_fstring_part(p, is_raw, item, b); + item = _PyPegen_decode_fstring_part(p, i == 0, is_raw, indent_char, indent_count, item, b); if (item == NULL) { return NULL; } diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 7f25afec302c225..dc3d369c023119e 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -455,7 +455,7 @@ tok_continuation_line(struct tok_state *tok) { static int maybe_raise_syntax_error_for_string_prefixes(struct tok_state *tok, int saw_b, int saw_r, int saw_u, - int saw_f, int saw_t) { + int saw_f, int saw_t, int saw_d) { // Supported: rb, rf, rt (in any order) // Unsupported: ub, ur, uf, ut, bf, bt, ft (in any order) @@ -480,6 +480,9 @@ maybe_raise_syntax_error_for_string_prefixes(struct tok_state *tok, if (saw_u && saw_t) { RETURN_SYNTAX_ERROR("u", "t"); } + if (saw_u && saw_d) { + RETURN_SYNTAX_ERROR("u", "d"); + } if (saw_b && saw_f) { RETURN_SYNTAX_ERROR("b", "f"); @@ -487,6 +490,9 @@ maybe_raise_syntax_error_for_string_prefixes(struct tok_state *tok, if (saw_b && saw_t) { RETURN_SYNTAX_ERROR("b", "t"); } + if (saw_b && saw_d) { + RETURN_SYNTAX_ERROR("b", "d"); + } if (saw_f && saw_t) { RETURN_SYNTAX_ERROR("f", "t"); @@ -741,8 +747,8 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t /* Identifier (most frequent token!) */ nonascii = 0; if (is_potential_identifier_start(c)) { - /* Process the various legal combinations of b"", r"", u"", and f"". */ - int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0, saw_t = 0; + /* Process the various legal combinations of b"", r"", u"", f"", and d"". */ + int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0, saw_t = 0, saw_d = 0; while (1) { if (!saw_b && (c == 'b' || c == 'B')) { saw_b = 1; @@ -762,6 +768,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t else if (!saw_t && (c == 't' || c == 'T')) { saw_t = 1; } + else if (!saw_d && (c == 'd' || c == 'D')) { + saw_d = 1; + } else { break; } @@ -769,7 +778,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t if (c == '"' || c == '\'') { // Raise error on incompatible string prefixes: int status = maybe_raise_syntax_error_for_string_prefixes( - tok, saw_b, saw_r, saw_u, saw_f, saw_t); + tok, saw_b, saw_r, saw_u, saw_f, saw_t, saw_d); if (status < 0) { return MAKE_TOKEN(ERRORTOKEN); } @@ -1049,7 +1058,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t } f_string_quote: - if (((Py_TOLOWER(*tok->start) == 'f' || Py_TOLOWER(*tok->start) == 'r' || Py_TOLOWER(*tok->start) == 't') + if (((Py_TOLOWER(*tok->start) == 'f' || Py_TOLOWER(*tok->start) == 'r' || Py_TOLOWER(*tok->start) == 't' || Py_TOLOWER(*tok->start) == 'd') && (c == '\'' || c == '"'))) { int quote = c; @@ -1089,6 +1098,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t the_current_tok->kind = TOK_FSTRING_MODE; the_current_tok->quote = quote; the_current_tok->quote_size = quote_size; + the_current_tok->raw = 0; the_current_tok->start = tok->start; the_current_tok->multi_line_start = tok->line_start; the_current_tok->first_line = tok->lineno; @@ -1101,25 +1111,28 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t the_current_tok->in_debug = 0; enum string_kind_t string_kind = FSTRING; - switch (*tok->start) { - case 'T': - case 't': - the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r'; - string_kind = TSTRING; - break; - case 'F': - case 'f': - the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r'; - break; - case 'R': - case 'r': - the_current_tok->raw = 1; - if (Py_TOLOWER(*(tok->start + 1)) == 't') { + for (const char *p = tok->start; *p != c; p++) { + switch (*p) { + case 'f': + case 'F': + break; + case 't': + case 'T': string_kind = TSTRING; - } - break; - default: - Py_UNREACHABLE(); + break; + case 'r': + case 'R': + the_current_tok->raw = 1; + break; + case 'd': + case 'D': + if (quote_size != 3) { + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "d-string must be a multiline string")); + } + break; + default: + Py_UNREACHABLE(); + } } the_current_tok->string_kind = string_kind; diff --git a/Parser/string_parser.c b/Parser/string_parser.c index b164dfbc81a9339..99f73c099ff7a29 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -247,6 +247,107 @@ _PyPegen_decode_string(Parser *p, int raw, const char *s, size_t len, Token *t) return decode_unicode_with_escapes(p, s, len, t); } +static PyObject* +_PyPegen_dedent_string(Parser *p, int is_raw, const char *s, Py_ssize_t len, Token* token) +{ + // this function is for d-string without t/f-string. + // dt/df-string are processed in action_helper.c:_get_resized_exprs + Py_ssize_t lineno = token->lineno; + + if (len == 0 || s[0] != '\n') { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + token, + "d-string must start with a newline at line %d", + lineno + ); + return NULL; + } + + // find the last newline and check all chars after it are spaces or tabs. + const char *endline = s + len; + while (endline[-1] != '\n') { + assert(endline > s); // we know at least the first char is a newline. + endline--; + if (*endline != ' ' && *endline != '\t') { + RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, + // specify the location of just before closing triple quotes. + token->end_lineno, token->end_col_offset - 3, token->end_lineno, token->end_col_offset-2, + "d-string must end with an indent line"); + return NULL; + } + } + + // Now, prefix is both the dedent indentation and the end of the d-string body. + Py_ssize_t indent_len = s + len - endline; + int indent_char = endline[0]; // ' ', '\t', or '\0'. + + // checks the prefix is consistant. + for (Py_ssize_t i = 1; i < indent_len; i++) { + if (endline[i] != indent_char) { + RAISE_ERROR_KNOWN_LOCATION( + p, PyExc_TabError, token->end_lineno, token->end_col_offset - 3, token->end_lineno, token->end_col_offset -2, + "inconsistent use of tabs and spaces in indentation"); + return NULL; + } + } + + PyUnicodeWriter *w = PyUnicodeWriter_Create(endline - s); + if (w == NULL) { + return NULL; + } + const char *line_start = s + 1; // skip the first newline + + while (line_start < endline) { + lineno++; + + Py_ssize_t i; + for (i = 0; i < indent_len && line_start + i < endline; i++) { + if (line_start[i] != indent_char) { + if (line_start[i] == '\n') { + break; // empty line + } + PyUnicodeWriter_Discard(w); + RAISE_ERROR_KNOWN_LOCATION(p, PyExc_IndentationError, lineno, i, lineno, i+1, + "d-string missing valid indentation"); + return NULL; + } + } + + if (line_start[i] == '\n') { // found an empty line with newline. + if (PyUnicodeWriter_WriteChar(w, '\n') < 0) { + PyUnicodeWriter_Discard(w); + return NULL; + } + line_start += i+1; + continue; + } + + // found a indented line. let's dedent it. + line_start += i; + const char *line_end = memchr(line_start, '\n', endline - line_start); + assert(line_end != NULL); // we know there is at least one newline before endline. + line_end++; // include the newline in the line + + if (is_raw || memchr(line_start, '\\', line_end - line_start) == NULL) { + if (PyUnicodeWriter_WriteUTF8(w, line_start, line_end - line_start) < 0) { + PyUnicodeWriter_Discard(w); + return NULL; + } + } + else { + PyObject *line = _PyPegen_decode_string(p, 1, line_start, line_end - line_start, token); + if (line == NULL || PyUnicodeWriter_WriteStr(w, line) < 0) { + Py_XDECREF(line); + return NULL; + } + Py_DECREF(line); + } + + line_start = line_end; + } + return PyUnicodeWriter_Finish(w); +} + /* s must include the bracketing quote characters, and r, b &/or f prefixes (if any), and embedded escape sequences (if any). (f-strings are handled by the parser) _PyPegen_parse_string parses it, and returns the decoded Python string object. */ @@ -262,9 +363,10 @@ _PyPegen_parse_string(Parser *p, Token *t) int quote = Py_CHARMASK(*s); int bytesmode = 0; int rawmode = 0; + int dedentmode = 0; if (Py_ISALPHA(quote)) { - while (!bytesmode || !rawmode) { + while (!bytesmode || !rawmode || !dedentmode) { if (quote == 'b' || quote == 'B') { quote =(unsigned char)*++s; bytesmode = 1; @@ -276,6 +378,10 @@ _PyPegen_parse_string(Parser *p, Token *t) quote = (unsigned char)*++s; rawmode = 1; } + else if (quote == 'd' || quote == 'D') { + quote =(unsigned char)*++s; + dedentmode = 1; + } else { break; } @@ -315,10 +421,17 @@ _PyPegen_parse_string(Parser *p, Token *t) return NULL; } } + else if (dedentmode) { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + t, + "d-string must be triple-quoted"); + return NULL; + } /* Avoid invoking escape decoding routines if possible. */ rawmode = rawmode || strchr(s, '\\') == NULL; if (bytesmode) { + assert(!dedentmode); /* Disallow non-ASCII characters. */ const char *ch; for (ch = s; *ch; ch++) { @@ -335,5 +448,9 @@ _PyPegen_parse_string(Parser *p, Token *t) } return decode_bytes_with_escapes(p, s, (Py_ssize_t)len, t); } + if (dedentmode) { + return _PyPegen_dedent_string(p, rawmode, s, len, t); + } return _PyPegen_decode_string(p, rawmode, s, len, t); } + From 8724f45c7401e34b56cc94a3dc664e0ba4ece807 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 21 Jan 2026 01:29:30 +0900 Subject: [PATCH 02/10] use least indent instead of closing quote indent --- Lib/test/test_dstring.py | 9 --- Objects/unicodeobject.c | 6 +- Parser/action_helpers.c | 110 ++++++++++++++++++++----------------- Parser/lexer/lexer.c | 3 - Parser/string_parser.c | 116 +++++++++++++++++++-------------------- 5 files changed, 120 insertions(+), 124 deletions(-) diff --git a/Lib/test/test_dstring.py b/Lib/test/test_dstring.py index 09592980dee8c51..7927877e8bb088a 100644 --- a/Lib/test/test_dstring.py +++ b/Lib/test/test_dstring.py @@ -26,14 +26,6 @@ def test_empty_dstring(self): ] self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) - def test_no_last_newline(self): - exprs = [ - "d'''\nhello world'''", - 'D"""\nhello world"""', - "df'''\nhello {42}'''", - ] - self.assertAllRaise(SyntaxError, "d-string must end with an indent line", exprs) - def test_simple_dstring(self): self.assertEqual(eval('d"""\n hello world\n """'), "hello world\n") self.assertEqual(eval('d"""\n hello world\n """'), " hello world\n") @@ -42,6 +34,5 @@ def test_simple_dstring(self): self.assertEqual(eval('dr"""\n hello world\\\n """'), " hello world\\\n") - if __name__ == '__main__': unittest.main() diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 3c689761de9b199..f26f9062793d2f4 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -13507,8 +13507,8 @@ of all lines in the [src, end). It returns the length of the common leading whitespace and sets `output` to point to the beginning of the common leading whitespace if length > 0. */ -static Py_ssize_t -search_longest_common_leading_whitespace( +Py_ssize_t +_Py_search_longest_common_leading_whitespace( const char *const src, const char *const end, const char **output) @@ -13604,7 +13604,7 @@ _PyUnicode_Dedent(PyObject *unicode) // [whitespace_start, whitespace_start + whitespace_len) // describes the current longest common leading whitespace const char *whitespace_start = NULL; - Py_ssize_t whitespace_len = search_longest_common_leading_whitespace( + Py_ssize_t whitespace_len = _Py_search_longest_common_leading_whitespace( src, end, &whitespace_start); if (whitespace_len == 0) { diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 7ea3b6136e032e1..f85a8ddc1c4bca2 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1314,8 +1314,8 @@ unicodewriter_write_line(Parser *p, PyUnicodeWriter *w, const char *line_start, } static PyObject* -_PyPegen_dedent_string_part(Parser *p, const char *s, size_t len, int indent_char, Py_ssize_t dedent_count, - int is_raw, int is_first, expr_ty constant, Token* token) +_PyPegen_dedent_string_part(Parser *p, const char *s, size_t len, const char *indent, Py_ssize_t indent_len, + int is_first, int is_raw, expr_ty constant, Token* token) { Py_ssize_t lineno = constant->lineno; const char *line_start = s; @@ -1353,7 +1353,7 @@ _PyPegen_dedent_string_part(Parser *p, const char *s, size_t len, int indent_cha lineno++; Py_ssize_t i = 0; - while (line_start + i < s_end && i < dedent_count && line_start[i] == indent_char) { + while (line_start + i < s_end && i < indent_len && line_start[i] == indent[i]) { i++; } @@ -1368,8 +1368,8 @@ _PyPegen_dedent_string_part(Parser *p, const char *s, size_t len, int indent_cha line_start += i+1; continue; } - if (i < dedent_count) { // found an invalid indent. - assert(line_start[i] != indent_char); + if (i < indent_len) { // found an invalid indent. + assert(line_start[i] != indent[i]); PyUnicodeWriter_Discard(w); RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, lineno, i, lineno, i+1, "d-string line missing valid indentation"); @@ -1395,7 +1395,10 @@ _PyPegen_dedent_string_part(Parser *p, const char *s, size_t len, int indent_cha } static expr_ty -_PyPegen_decode_fstring_part(Parser* p, int is_first, int is_raw, int indent_char, Py_ssize_t dedent_count, expr_ty constant, Token* token) { +_PyPegen_decode_fstring_part(Parser* p, int is_first, int is_raw, + const char *indent, Py_ssize_t indent_len, + expr_ty constant, Token* token) +{ assert(PyUnicode_CheckExact(constant->v.Constant.value)); const char* bstr = PyUnicode_AsUTF8(constant->v.Constant.value); @@ -1405,9 +1408,9 @@ _PyPegen_decode_fstring_part(Parser* p, int is_first, int is_raw, int indent_cha is_raw = is_raw || strchr(bstr, '\\') == NULL; PyObject *str = NULL; - if (dedent_count > 0) { - str = _PyPegen_dedent_string_part(p, bstr, strlen(bstr), indent_char, dedent_count, - is_raw, is_first, constant, token); + if (indent_len > 0) { + str = _PyPegen_dedent_string_part(p, bstr, strlen(bstr), indent, indent_len, + is_first, is_raw, constant, token); } else { str = _PyPegen_decode_string(p, is_raw, bstr, strlen(bstr), token); @@ -1426,6 +1429,14 @@ _PyPegen_decode_fstring_part(Parser* p, int is_first, int is_raw, int indent_cha p->arena); } +/* defined in unicodeobject.c */ +extern Py_ssize_t +_Py_search_longest_common_leading_whitespace( + const char *const src, + const char *const end, + const char **output + ); + static asdl_expr_seq * _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b, enum string_kind_t string_kind) { @@ -1444,14 +1455,15 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b } int is_raw = strpbrk(quote_str, "rR") != NULL; int is_dedent = strpbrk(quote_str, "dD") != NULL; - int indent_char = 0; - Py_ssize_t indent_count = 0; asdl_expr_seq *seq = _Py_asdl_expr_seq_new(total_items, p->arena); if (seq == NULL) { return NULL; } + const char *common_indent_start = NULL; + Py_ssize_t common_indent_len = 0; + if (is_dedent) { expr_ty first_item = asdl_seq_GET(raw_expressions, 0); if (first_item->kind != Constant_kind @@ -1463,52 +1475,52 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b return NULL; } - expr_ty last_item = asdl_seq_GET(raw_expressions, n_items - 1); - if (last_item->kind != Constant_kind) { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION( - last_item, - "d-string must end with an indent line" - ); + // Instead of calculating common indent from all parts, + // build temporary string and calculate common indent from it. + PyBytesWriter *w = PyBytesWriter_Create(0); + if (w == NULL) { return NULL; } - Py_ssize_t blen; - const char *bstr = PyUnicode_AsUTF8AndSize(last_item->v.Constant.value, &blen); - if (bstr == NULL) { - return NULL; - } + for (Py_ssize_t i = 0; i < n_items; i++) { + expr_ty item = asdl_seq_GET(raw_expressions, i); - // memrchr is GNU extension; use manual loop for portability. - const char *lastline = bstr + blen; - while (bstr < lastline) { - if (lastline[-1] == '\n') { - break; - } - lastline--; - if (*lastline != ' ' && *lastline != '\t') { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION( - last_item, - "d-string must end with an indent line" - ); - return NULL; + if (item->kind == JoinedStr_kind) { + // Write a placeholder. + if (PyBytesWriter_WriteBytes(w, "X", 1) < 0) { + PyBytesWriter_Discard(w); + return NULL; + } + continue; } - } - - // checks indent of the last line. - indent_count = bstr + blen - lastline; - if (indent_count > 0) { - indent_char = lastline[0]; - - for (Py_ssize_t i = 1; i < indent_count; i++) { - if (lastline[i] != indent_char) { - RAISE_ERROR_KNOWN_LOCATION( - p, PyExc_TabError, last_item->end_lineno, i, last_item->end_lineno, i+1, - "inconsistent use of tabs and spaces in indentation" - ); + if (item->kind == Constant_kind) { + Py_ssize_t blen; + const char *bstr = PyUnicode_AsUTF8AndSize(item->v.Constant.value, &blen); + if (bstr == NULL || PyBytesWriter_WriteBytes(w, bstr, blen) < 0) { + PyBytesWriter_Discard(w); return NULL; } + continue; } } + // Add a terminator to include the last line before the ending quote + if (PyBytesWriter_WriteBytes(w, "X", 1) < 0) { + PyBytesWriter_Discard(w); + return NULL; + } + + // TODO: instead of creating temp_bytes, we could search + // common index from each part directly. But this need reimplementation + // of _Py_search_longest_common_leading_whitespace. + PyObject *temp_bytes = PyBytesWriter_Finish(w); + if (temp_bytes == NULL) { + return NULL; + } + _PyArena_AddPyObject(p->arena, temp_bytes); + const char *temp_str = PyBytes_AsString(temp_bytes); + const char *temp_end = temp_str + PyBytes_GET_SIZE(temp_bytes); + common_indent_len = _Py_search_longest_common_leading_whitespace( + temp_str, temp_end, &common_indent_start); } Py_ssize_t index = 0; @@ -1542,7 +1554,7 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b } if (item->kind == Constant_kind) { - item = _PyPegen_decode_fstring_part(p, i == 0, is_raw, indent_char, indent_count, item, b); + item = _PyPegen_decode_fstring_part(p, i == 0, is_raw, common_indent_start, common_indent_len, item, b); if (item == NULL) { return NULL; } diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index dc3d369c023119e..cfccdd55a6ae112 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -490,9 +490,6 @@ maybe_raise_syntax_error_for_string_prefixes(struct tok_state *tok, if (saw_b && saw_t) { RETURN_SYNTAX_ERROR("b", "t"); } - if (saw_b && saw_d) { - RETURN_SYNTAX_ERROR("b", "d"); - } if (saw_f && saw_t) { RETURN_SYNTAX_ERROR("f", "t"); diff --git a/Parser/string_parser.c b/Parser/string_parser.c index 99f73c099ff7a29..0fbc20ee4ce6fbb 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -247,66 +247,54 @@ _PyPegen_decode_string(Parser *p, int raw, const char *s, size_t len, Token *t) return decode_unicode_with_escapes(p, s, len, t); } +/* defined in unicodeobject.c */ +extern Py_ssize_t +_Py_search_longest_common_leading_whitespace( + const char *const src, + const char *const end, + const char **output + ); + +// Dedent d-string and return result as a bytes. static PyObject* -_PyPegen_dedent_string(Parser *p, int is_raw, const char *s, Py_ssize_t len, Token* token) +_PyPegen_dedent_string(Parser *p, const char *s, Py_ssize_t len, Token* token) { // this function is for d-string without t/f-string. // dt/df-string are processed in action_helper.c:_get_resized_exprs Py_ssize_t lineno = token->lineno; + const char *end = s + len; + // skips the first newline. if (len == 0 || s[0] != '\n') { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( token, - "d-string must start with a newline at line %d", - lineno + "d-string must start with a newline" ); return NULL; } - // find the last newline and check all chars after it are spaces or tabs. - const char *endline = s + len; - while (endline[-1] != '\n') { - assert(endline > s); // we know at least the first char is a newline. - endline--; - if (*endline != ' ' && *endline != '\t') { - RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, - // specify the location of just before closing triple quotes. - token->end_lineno, token->end_col_offset - 3, token->end_lineno, token->end_col_offset-2, - "d-string must end with an indent line"); - return NULL; - } - } - - // Now, prefix is both the dedent indentation and the end of the d-string body. - Py_ssize_t indent_len = s + len - endline; - int indent_char = endline[0]; // ' ', '\t', or '\0'. + // We find common indent from [s, end+1) because we want to include the last line + // for indent calculation. + assert(*end == '"' || *end == '\''); // end[0:3] is the trailing quotes + const char *indent; + Py_ssize_t indent_len = _Py_search_longest_common_leading_whitespace(s, end+1, &indent); - // checks the prefix is consistant. - for (Py_ssize_t i = 1; i < indent_len; i++) { - if (endline[i] != indent_char) { - RAISE_ERROR_KNOWN_LOCATION( - p, PyExc_TabError, token->end_lineno, token->end_col_offset - 3, token->end_lineno, token->end_col_offset -2, - "inconsistent use of tabs and spaces in indentation"); - return NULL; - } - } - - PyUnicodeWriter *w = PyUnicodeWriter_Create(endline - s); + PyBytesWriter *w = PyBytesWriter_Create(0); if (w == NULL) { return NULL; } - const char *line_start = s + 1; // skip the first newline + const char *line_start = s + 1; - while (line_start < endline) { + while (line_start < end) { lineno++; Py_ssize_t i; - for (i = 0; i < indent_len && line_start + i < endline; i++) { - if (line_start[i] != indent_char) { + for (i = 0; i < indent_len; i++) { + if (line_start[i] != indent[i]) { if (line_start[i] == '\n') { break; // empty line } - PyUnicodeWriter_Discard(w); + PyBytesWriter_Discard(w); RAISE_ERROR_KNOWN_LOCATION(p, PyExc_IndentationError, lineno, i, lineno, i+1, "d-string missing valid indentation"); return NULL; @@ -314,8 +302,8 @@ _PyPegen_dedent_string(Parser *p, int is_raw, const char *s, Py_ssize_t len, Tok } if (line_start[i] == '\n') { // found an empty line with newline. - if (PyUnicodeWriter_WriteChar(w, '\n') < 0) { - PyUnicodeWriter_Discard(w); + if (PyBytesWriter_WriteBytes(w, "\n", 1) < 0) { + PyBytesWriter_Discard(w); return NULL; } line_start += i+1; @@ -324,28 +312,21 @@ _PyPegen_dedent_string(Parser *p, int is_raw, const char *s, Py_ssize_t len, Tok // found a indented line. let's dedent it. line_start += i; - const char *line_end = memchr(line_start, '\n', endline - line_start); - assert(line_end != NULL); // we know there is at least one newline before endline. - line_end++; // include the newline in the line - - if (is_raw || memchr(line_start, '\\', line_end - line_start) == NULL) { - if (PyUnicodeWriter_WriteUTF8(w, line_start, line_end - line_start) < 0) { - PyUnicodeWriter_Discard(w); - return NULL; - } + const char *line_end = memchr(line_start, '\n', end - line_start); + if (line_end == NULL) { + line_end = end; // last line without newline } else { - PyObject *line = _PyPegen_decode_string(p, 1, line_start, line_end - line_start, token); - if (line == NULL || PyUnicodeWriter_WriteStr(w, line) < 0) { - Py_XDECREF(line); - return NULL; - } - Py_DECREF(line); + line_end++; // include the newline in the line } + if (PyBytesWriter_WriteBytes(w, line_start, line_end - line_start) < 0) { + PyBytesWriter_Discard(w); + return NULL; + } line_start = line_end; } - return PyUnicodeWriter_Finish(w); + return PyBytesWriter_Finish(w); } /* s must include the bracketing quote characters, and r, b &/or f prefixes @@ -427,11 +408,22 @@ _PyPegen_parse_string(Parser *p, Token *t) "d-string must be triple-quoted"); return NULL; } + PyObject *dedent_bytes = NULL; + if (dedentmode) { + dedent_bytes = _PyPegen_dedent_string(p, s, len, t); + if (dedent_bytes == NULL) { + return NULL; + } + if (PyBytes_AsStringAndSize(dedent_bytes, &s, (Py_ssize_t*)&len) < 0) { + Py_DECREF(dedent_bytes); + return NULL; + } + } /* Avoid invoking escape decoding routines if possible. */ rawmode = rawmode || strchr(s, '\\') == NULL; + PyObject *result; if (bytesmode) { - assert(!dedentmode); /* Disallow non-ASCII characters. */ const char *ch; for (ch = s; *ch; ch++) { @@ -440,17 +432,21 @@ _PyPegen_parse_string(Parser *p, Token *t) t, "bytes can only contain ASCII " "literal characters"); + Py_XDECREF(dedent_bytes); return NULL; } } if (rawmode) { - return PyBytes_FromStringAndSize(s, (Py_ssize_t)len); + result = PyBytes_FromStringAndSize(s, (Py_ssize_t)len); + } + else { + result = decode_bytes_with_escapes(p, s, (Py_ssize_t)len, t); } - return decode_bytes_with_escapes(p, s, (Py_ssize_t)len, t); } - if (dedentmode) { - return _PyPegen_dedent_string(p, rawmode, s, len, t); + else { + result = _PyPegen_decode_string(p, rawmode, s, len, t); } - return _PyPegen_decode_string(p, rawmode, s, len, t); + Py_XDECREF(dedent_bytes); + return result; } From c94788d8c71e49fb9f1c16d230f6122a1350bcd5 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 23 Jan 2026 23:03:26 +0900 Subject: [PATCH 03/10] fix invalid escape sequences position --- Parser/action_helpers.c | 60 ++++++++++++++++------- Parser/string_parser.c | 104 +++++++++++++++++++++++----------------- 2 files changed, 103 insertions(+), 61 deletions(-) diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index f85a8ddc1c4bca2..b31811984c21245 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1314,17 +1314,34 @@ unicodewriter_write_line(Parser *p, PyUnicodeWriter *w, const char *line_start, } static PyObject* -_PyPegen_dedent_string_part(Parser *p, const char *s, size_t len, const char *indent, Py_ssize_t indent_len, - int is_first, int is_raw, expr_ty constant, Token* token) +_PyPegen_dedent_string_part( + Parser *p, const char *s, size_t len, const char *indent, Py_ssize_t indent_len, + int is_first, int is_raw, expr_ty constant, Token* token) { Py_ssize_t lineno = constant->lineno; const char *line_start = s; - const char *s_end = s + len; + const char *end = s + len; + + int _prev_call_invalid = p->call_invalid_rules; + if (!_prev_call_invalid && !is_raw) { + // _PyPegen_decode_string() and decode_bytes_with_escapes() may call + // warn_invalid_escape_sequence(). It may emit issue or raise SyntaxError + // for invalid escape sequences. + // We need to call it before dedenting since SyntaxError needs exact lineno + // and col_offset of invalid escape sequences. + PyObject *t = _PyPegen_decode_string(p, 0, s, len, token); + if (t == NULL) { + return NULL; + } + Py_DECREF(t); + p->call_invalid_rules = 1; + } PyUnicodeWriter *w = PyUnicodeWriter_Create(len); if (w == NULL) { return NULL; } + if (is_first) { assert (line_start[0] == '\n'); line_start++; // skip the first newline @@ -1335,25 +1352,24 @@ _PyPegen_dedent_string_part(Parser *p, const char *s, size_t len, const char *in // next line // """" // We don't need to dedent the first line in the non-first parts. - const char *line_end = memchr(line_start, '\n', s_end - line_start); + const char *line_end = memchr(line_start, '\n', end - line_start); if (line_end) { line_end++; // include the newline } else { - line_end = s_end; + line_end = end; } if (unicodewriter_write_line(p, w, line_start, line_end, is_raw, token) < 0) { - PyUnicodeWriter_Discard(w); - return NULL; + goto error; } line_start = line_end; } - while (line_start < s + len) { + while (line_start < end) { lineno++; Py_ssize_t i = 0; - while (line_start + i < s_end && i < indent_len && line_start[i] == indent[i]) { + while (line_start + i < end && i < indent_len && line_start[i] == indent[i]) { i++; } @@ -1362,36 +1378,39 @@ _PyPegen_dedent_string_part(Parser *p, const char *s, size_t len, const char *in } if (line_start[i] == '\n') { // found an empty line with newline. if (PyUnicodeWriter_WriteChar(w, '\n') < 0) { - PyUnicodeWriter_Discard(w); - return NULL; + goto error; } line_start += i+1; continue; } if (i < indent_len) { // found an invalid indent. assert(line_start[i] != indent[i]); - PyUnicodeWriter_Discard(w); RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, lineno, i, lineno, i+1, "d-string line missing valid indentation"); - return NULL; + goto error; } // found a indented line. let's dedent it. line_start += i; - const char *line_end = memchr(line_start, '\n', s_end - line_start); + const char *line_end = memchr(line_start, '\n', end - line_start); if (line_end) { line_end++; // include the newline } else { - line_end = s_end; + line_end = end; } if (unicodewriter_write_line(p, w, line_start, line_end, is_raw, token) < 0) { - PyUnicodeWriter_Discard(w); - return NULL; + goto error; } line_start = line_end; } + p->call_invalid_rules = _prev_call_invalid; return PyUnicodeWriter_Finish(w); + +error: + p->call_invalid_rules = _prev_call_invalid; + PyUnicodeWriter_Discard(w); + return NULL; } static expr_ty @@ -1408,7 +1427,7 @@ _PyPegen_decode_fstring_part(Parser* p, int is_first, int is_raw, is_raw = is_raw || strchr(bstr, '\\') == NULL; PyObject *str = NULL; - if (indent_len > 0) { + if (indent != NULL) { str = _PyPegen_dedent_string_part(p, bstr, strlen(bstr), indent, indent_len, is_first, is_raw, constant, token); } @@ -1521,6 +1540,11 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b const char *temp_end = temp_str + PyBytes_GET_SIZE(temp_bytes); common_indent_len = _Py_search_longest_common_leading_whitespace( temp_str, temp_end, &common_indent_start); + // _py_serach_longest_common_leading_whitespace() may return NULL when + // indent_len is 0. + if (common_indent_len == 0) { + common_indent_start = ""; + } } Py_ssize_t index = 0; diff --git a/Parser/string_parser.c b/Parser/string_parser.c index 0fbc20ee4ce6fbb..3425b856796fe5a 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -257,41 +257,20 @@ _Py_search_longest_common_leading_whitespace( // Dedent d-string and return result as a bytes. static PyObject* -_PyPegen_dedent_string(Parser *p, const char *s, Py_ssize_t len, Token* token) +_PyPegen_dedent_string(Parser *p, const char *s, Py_ssize_t len, + const char *indent, Py_ssize_t indent_len, int lineno) { - // this function is for d-string without t/f-string. - // dt/df-string are processed in action_helper.c:_get_resized_exprs - Py_ssize_t lineno = token->lineno; - const char *end = s + len; - - // skips the first newline. - if (len == 0 || s[0] != '\n') { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION( - token, - "d-string must start with a newline" - ); - return NULL; - } - - // We find common indent from [s, end+1) because we want to include the last line - // for indent calculation. - assert(*end == '"' || *end == '\''); // end[0:3] is the trailing quotes - const char *indent; - Py_ssize_t indent_len = _Py_search_longest_common_leading_whitespace(s, end+1, &indent); - PyBytesWriter *w = PyBytesWriter_Create(0); if (w == NULL) { return NULL; } - const char *line_start = s + 1; - - while (line_start < end) { - lineno++; + const char *end = s + len; + for (; s < end; lineno++) { Py_ssize_t i; for (i = 0; i < indent_len; i++) { - if (line_start[i] != indent[i]) { - if (line_start[i] == '\n') { + if (s[i] != indent[i]) { + if (s[i] == '\n') { break; // empty line } PyBytesWriter_Discard(w); @@ -301,18 +280,18 @@ _PyPegen_dedent_string(Parser *p, const char *s, Py_ssize_t len, Token* token) } } - if (line_start[i] == '\n') { // found an empty line with newline. + if (s[i] == '\n') { // found an empty line with newline. if (PyBytesWriter_WriteBytes(w, "\n", 1) < 0) { PyBytesWriter_Discard(w); return NULL; } - line_start += i+1; + s += i+1; continue; } // found a indented line. let's dedent it. - line_start += i; - const char *line_end = memchr(line_start, '\n', end - line_start); + s += i; + const char *line_end = memchr(s, '\n', end - s); if (line_end == NULL) { line_end = end; // last line without newline } @@ -320,11 +299,11 @@ _PyPegen_dedent_string(Parser *p, const char *s, Py_ssize_t len, Token* token) line_end++; // include the newline in the line } - if (PyBytesWriter_WriteBytes(w, line_start, line_end - line_start) < 0) { + if (PyBytesWriter_WriteBytes(w, s, line_end - s) < 0) { PyBytesWriter_Discard(w); return NULL; } - line_start = line_end; + s = line_end; } return PyBytesWriter_Finish(w); } @@ -403,25 +382,62 @@ _PyPegen_parse_string(Parser *p, Token *t) } } else if (dedentmode) { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION( - t, - "d-string must be triple-quoted"); + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(t, "d-string must be triple-quoted"); return NULL; } + + /* Avoid invoking escape decoding routines if possible. */ + rawmode = rawmode || strchr(s, '\\') == NULL; + + int _prev_call_invald = p->call_invalid_rules; + PyObject *dedent_bytes = NULL; if (dedentmode) { - dedent_bytes = _PyPegen_dedent_string(p, s, len, t); - if (dedent_bytes == NULL) { + if (len == 0 || s[0] != '\n') { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(t, "d-string must start with a newline"); return NULL; } - if (PyBytes_AsStringAndSize(dedent_bytes, &s, (Py_ssize_t*)&len) < 0) { - Py_DECREF(dedent_bytes); - return NULL; + + // _PyPegen_decode_string() and decode_bytes_with_escapes() emit + // a warning for invalid escape sequences. + // We need to call it before dedenting since it shifts the positions. + if (!_prev_call_invald && !rawmode) { + PyObject *temp; + if (bytesmode) { + temp = decode_bytes_with_escapes(p, s, len, t); + } + else { + temp = _PyPegen_decode_string(p, 0, s, len, t); + } + if (temp == NULL) { + return NULL; + } + Py_DECREF(temp); + } + + // We find common indent from [s, end+1) because we want to include the last line + // for indent calculation. + const char *end = s + len; + assert(*end == '"' || *end == '\''); // end[0:3] is the trailing quotes + const char *indent; + Py_ssize_t indent_len = _Py_search_longest_common_leading_whitespace(s+1, end+1, &indent); + + s++; len--; // skip the first newline + if (indent_len > 0) { + // dedent the string + dedent_bytes = _PyPegen_dedent_string(p, s, len, indent, indent_len, t->lineno + 1); + if (dedent_bytes == NULL) { + return NULL; + } + if (PyBytes_AsStringAndSize(dedent_bytes, (char**)&s, (Py_ssize_t*)&len) < 0) { + Py_DECREF(dedent_bytes); + return NULL; + } } + + p->call_invalid_rules = 1; } - /* Avoid invoking escape decoding routines if possible. */ - rawmode = rawmode || strchr(s, '\\') == NULL; PyObject *result; if (bytesmode) { /* Disallow non-ASCII characters. */ @@ -433,6 +449,7 @@ _PyPegen_parse_string(Parser *p, Token *t) "bytes can only contain ASCII " "literal characters"); Py_XDECREF(dedent_bytes); + p->call_invalid_rules = _prev_call_invald; return NULL; } } @@ -447,6 +464,7 @@ _PyPegen_parse_string(Parser *p, Token *t) result = _PyPegen_decode_string(p, rawmode, s, len, t); } Py_XDECREF(dedent_bytes); + p->call_invalid_rules = _prev_call_invald; return result; } From 905758261aa40b2e7e3547dc3ff4b15b005c2d55 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 25 Jan 2026 17:27:22 +0900 Subject: [PATCH 04/10] improve tests --- Lib/test/test_dstring.py | 94 ++++++++++++++++++++++++++++++++++------ Parser/action_helpers.c | 7 +++ Parser/lexer/lexer.c | 2 +- 3 files changed, 88 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_dstring.py b/Lib/test/test_dstring.py index 7927877e8bb088a..5161c21b2a6c26b 100644 --- a/Lib/test/test_dstring.py +++ b/Lib/test/test_dstring.py @@ -1,37 +1,103 @@ import unittest +_dstring_prefixes = "d db df dt dr drb drf drt".split() +_dstring_prefixes += [p.upper() for p in _dstring_prefixes] + + +def d(s): + # Helper function to evaluate d-strings. + if '"""' in s: + return eval(f"d'''{s}'''") + else: + return eval(f'd"""{s}"""') + + class DStringTestCase(unittest.TestCase): def assertAllRaise(self, exception_type, regex, error_strings): for str in error_strings: with self.subTest(str=str): with self.assertRaisesRegex(exception_type, regex) as cm: eval(str) - # print("Testing expression:", repr(str)) - # print(repr(cm.exception)) - # print(repr(cm.exception.text)) def test_single_quote(self): exprs = [ - "d'hello'", - 'D"hello"', - "d'hello\\nworld'", + f"{p}'hello, world'" for p in _dstring_prefixes + ] + [ + f'{p}"hello, world"' for p in _dstring_prefixes ] self.assertAllRaise(SyntaxError, "d-string must be triple-quoted", exprs) def test_empty_dstring(self): exprs = [ - "d''''''", - 'D""""""', + f"{p}''''''" for p in _dstring_prefixes + ] + [ + f'{p}""""""' for p in _dstring_prefixes ] self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) - def test_simple_dstring(self): - self.assertEqual(eval('d"""\n hello world\n """'), "hello world\n") - self.assertEqual(eval('d"""\n hello world\n """'), " hello world\n") - self.assertEqual(eval('d"""\n hello world\n"""'), " hello world\n") - self.assertEqual(eval('d"""\n hello world\\\n """'), " hello world") - self.assertEqual(eval('dr"""\n hello world\\\n """'), " hello world\\\n") + for prefix in _dstring_prefixes: + expr = f"{prefix}'''\n'''" + expr2 = f'{prefix}"""\n"""' + with self.subTest(expr=expr): + v = eval(expr) + v2 = eval(expr2) + if 't' in prefix.lower(): + self.assertEqual(v.strings, ("",)) + self.assertEqual(v2.strings, ("",)) + elif 'b' in prefix.lower(): + self.assertEqual(v, b"") + self.assertEqual(v2, b"") + else: + self.assertEqual(v, "") + self.assertEqual(v2, "") + + def test_dedent(self): + # Basic dedent - remove common leading whitespace + result = d(""" + hello + world + """) + self.assertEqual(result, "hello\nworld\n") + + # Dedent with varying indentation + result = d(""" + line1 + line2 + line3 + """) + self.assertEqual(result, " line1\n line2\nline3\n ") + + # Dedent with tabs + result = d(""" +\thello +\tworld +\t""") + self.assertEqual(result, "hello\nworld\n") + + # Mixed spaces and tabs (using common leading whitespace) + result = d(""" +\t\t hello +\t\t world +\t\t """) + self.assertEqual(result, " hello\n world\n") + + # Empty lines do not affect the calculation of common leading whitespace + result = d(""" + hello + + world + """) + self.assertEqual(result, "hello\n\nworld\n") + + # Lines with only whitespace also have their indentation removed. + result = d(""" + hello + \n\ + \n\ + world + """) + self.assertEqual(result, "hello\n\n \nworld\n") if __name__ == '__main__': diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index b31811984c21245..bc61e882a580af3 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1484,6 +1484,13 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b Py_ssize_t common_indent_len = 0; if (is_dedent) { + if (total_items == 0) { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + a, + "d-string must start with a newline" + ); + return NULL; + } expr_ty first_item = asdl_seq_GET(raw_expressions, 0); if (first_item->kind != Constant_kind || PyUnicode_ReadChar(first_item->v.Constant.value, 0) != '\n') { diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index cfccdd55a6ae112..07c61f1cb4386b3 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -1124,7 +1124,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t case 'd': case 'D': if (quote_size != 3) { - return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "d-string must be a multiline string")); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "d-string must be triple-quoted")); } break; default: From 4f952738cde7ed21529341f0f1e02c2873275163 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 28 Jan 2026 19:30:21 +0900 Subject: [PATCH 05/10] t/f-string: calculate common indent without temp string --- Parser/action_helpers.c | 151 +++++++++++++++++++++++++++------------- 1 file changed, 103 insertions(+), 48 deletions(-) diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index bc61e882a580af3..d5315c07f13bb13 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1448,13 +1448,102 @@ _PyPegen_decode_fstring_part(Parser* p, int is_first, int is_raw, p->arena); } -/* defined in unicodeobject.c */ -extern Py_ssize_t -_Py_search_longest_common_leading_whitespace( +/* +This function is customized version of _Py_search_longest_common_leading_whitespace() +in unicodeobject.c +*/ +static void +search_longest_common_leading_whitespace( const char *const src, const char *const end, - const char **output - ); + const char **indent, + Py_ssize_t *indent_len) +{ + // [_start, _start + _len) + // describes the current longest common leading whitespace + const char *_start = *indent; + Py_ssize_t _len = *indent_len; + + // skip the first line. for example: + // s = df""" + // first part + // first part{x}second part + // second part + // """ + // we don't need newline after opening qute. + // we don't need first line in the second part too. + const char *iter = memchr(src, '\n', end - src); + if (iter == NULL) { + // single line string + return; + } + + for (iter++; iter <= end; iter++) { + const char *line_start = iter; + const char *leading_whitespace_end = NULL; + + // scan the whole line + while (iter < end && *iter != '\n') { + if (!leading_whitespace_end && *iter != ' ' && *iter != '\t') { + /* `iter` points to the first non-whitespace character + in this line */ + if (iter == line_start) { + // some line has no indent, fast exit! + *indent = iter; + *indent_len = 0; + return; + } + leading_whitespace_end = iter; + } + ++iter; + } + + if (!leading_whitespace_end) { + // if this line has all white space, skip it + if (iter < end) { + continue; + } + leading_whitespace_end = iter; // last line may not end with '\n' + } + + if (!_start) { + // update the first leading whitespace + _start = line_start; + _len = leading_whitespace_end - line_start; + } + else { + /* We then compare with the current longest leading whitespace. + + [line_start, leading_whitespace_end) is the leading + whitespace of this line, + + [_start, _start + _len) is the leading whitespace of the + current longest leading whitespace. */ + Py_ssize_t new_len = 0; + const char *_iter = _start, *line_iter = line_start; + + while (_iter < _start + _len && line_iter < leading_whitespace_end + && *_iter == *line_iter) + { + ++_iter; + ++line_iter; + ++new_len; + } + + _len = new_len; + if (_len == 0) { + // No common things now, fast exit! + *indent = _start; + *indent_len = 0; + return; + } + } + } + + *indent = _start; + *indent_len = _len; +} + static asdl_expr_seq * _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b, enum string_kind_t string_kind) @@ -1480,8 +1569,8 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b return NULL; } - const char *common_indent_start = NULL; - Py_ssize_t common_indent_len = 0; + const char *indent_start = NULL; + Py_ssize_t indent_len = 0; if (is_dedent) { if (total_items == 0) { @@ -1501,56 +1590,22 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b return NULL; } - // Instead of calculating common indent from all parts, - // build temporary string and calculate common indent from it. - PyBytesWriter *w = PyBytesWriter_Create(0); - if (w == NULL) { - return NULL; - } - for (Py_ssize_t i = 0; i < n_items; i++) { expr_ty item = asdl_seq_GET(raw_expressions, i); - - if (item->kind == JoinedStr_kind) { - // Write a placeholder. - if (PyBytesWriter_WriteBytes(w, "X", 1) < 0) { - PyBytesWriter_Discard(w); - return NULL; - } - continue; - } if (item->kind == Constant_kind) { Py_ssize_t blen; const char *bstr = PyUnicode_AsUTF8AndSize(item->v.Constant.value, &blen); - if (bstr == NULL || PyBytesWriter_WriteBytes(w, bstr, blen) < 0) { - PyBytesWriter_Discard(w); + if (bstr == NULL) { return NULL; } - continue; + search_longest_common_leading_whitespace(bstr, bstr + blen, &indent_start, &indent_len); } } - // Add a terminator to include the last line before the ending quote - if (PyBytesWriter_WriteBytes(w, "X", 1) < 0) { - PyBytesWriter_Discard(w); - return NULL; - } - // TODO: instead of creating temp_bytes, we could search - // common index from each part directly. But this need reimplementation - // of _Py_search_longest_common_leading_whitespace. - PyObject *temp_bytes = PyBytesWriter_Finish(w); - if (temp_bytes == NULL) { - return NULL; - } - _PyArena_AddPyObject(p->arena, temp_bytes); - const char *temp_str = PyBytes_AsString(temp_bytes); - const char *temp_end = temp_str + PyBytes_GET_SIZE(temp_bytes); - common_indent_len = _Py_search_longest_common_leading_whitespace( - temp_str, temp_end, &common_indent_start); - // _py_serach_longest_common_leading_whitespace() may return NULL when - // indent_len is 0. - if (common_indent_len == 0) { - common_indent_start = ""; + assert(indent_start != NULL); // TODO: is this assert true? + // _py_serach_longest_common_leading_whitespace() may not set indent_start when string is empty. + if (indent_len == 0) { + indent_start = ""; } } @@ -1585,7 +1640,7 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b } if (item->kind == Constant_kind) { - item = _PyPegen_decode_fstring_part(p, i == 0, is_raw, common_indent_start, common_indent_len, item, b); + item = _PyPegen_decode_fstring_part(p, i == 0, is_raw, indent_start, indent_len, item, b); if (item == NULL) { return NULL; } From 44c1aa49a71c75cd7430700e06732ca7e88113ef Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 31 Jan 2026 23:52:41 +0900 Subject: [PATCH 06/10] add tests --- Lib/test/test_dstring.py | 113 ++++++++++++++++++++++++++++----------- 1 file changed, 83 insertions(+), 30 deletions(-) diff --git a/Lib/test/test_dstring.py b/Lib/test/test_dstring.py index 5161c21b2a6c26b..7682d0c6cd5f9d1 100644 --- a/Lib/test/test_dstring.py +++ b/Lib/test/test_dstring.py @@ -6,12 +6,27 @@ def d(s): - # Helper function to evaluate d-strings. + """Helper function to evaluate d-strings.""" if '"""' in s: return eval(f"d'''{s}'''") else: return eval(f'd"""{s}"""') +def db(s): + """Helper function to evaluate db-strings.""" + if '"""' in s: + return eval(f"db'''{s}'''") + else: + return eval(f'db"""{s}"""') + +def fd(s, globals=None): + """Helper function to evaluate fd-strings.""" + if '"""' in s: + return eval(f"fd'''{s}'''", globals=globals) + else: + return eval(f'fd"""{s}"""', globals=globals) + + class DStringTestCase(unittest.TestCase): def assertAllRaise(self, exception_type, regex, error_strings): @@ -52,53 +67,91 @@ def test_empty_dstring(self): self.assertEqual(v, "") self.assertEqual(v2, "") - def test_dedent(self): + def check_dbstring(self, s, expected): + self.assertEqual(d(s), expected) + self.assertEqual(db(s), expected.encode()) + + def test_dbstring(self): # Basic dedent - remove common leading whitespace - result = d(""" - hello - world - """) - self.assertEqual(result, "hello\nworld\n") + source = """ + hello + world + """ + self.check_dbstring(source, "hello\nworld\n") + + # closing quote on same line as last content line + source = """ + hello + world""" + self.check_dbstring(source, "hello\nworld") # Dedent with varying indentation - result = d(""" - line1 - line2 - line3 - """) - self.assertEqual(result, " line1\n line2\nline3\n ") + source = """ + .line1 + ...line2 + line3 + ..""".replace('.', ' ') + self.check_dbstring(source, " line1\n line2\nline3\n ") # Dedent with tabs - result = d(""" + source = """ \thello \tworld -\t""") - self.assertEqual(result, "hello\nworld\n") +\t""" + self.check_dbstring(source, "hello\nworld\n") # Mixed spaces and tabs (using common leading whitespace) - result = d(""" + source = """ \t\t hello \t\t world -\t\t """) - self.assertEqual(result, " hello\n world\n") +\t\t """ + self.check_dbstring(source, " hello\n world\n") # Empty lines do not affect the calculation of common leading whitespace - result = d(""" + source = """ hello world - """) - self.assertEqual(result, "hello\n\nworld\n") + """ + self.check_dbstring(source, "hello\n\nworld\n") # Lines with only whitespace also have their indentation removed. - result = d(""" - hello - \n\ - \n\ - world - """) - self.assertEqual(result, "hello\n\n \nworld\n") - + source = """ +....hello +.. +...... +....world +....""".replace('.', ' ') + self.check_dbstring(source, "hello\n\n \nworld\n") + + # Line continuation with backslash works as usual. + # But you cannot put a backslash right after the opening quotes. + source = r""" + Hello \ + World!\ + """ + self.check_dbstring(source, "Hello World!") + + def test_fdstring(self): + g = {"world": 42} + + source = r""" + Hello + {world} + """ + self.assertEqual(fd(source, globals=g), "Hello\n 42\n") + + source = r""" + Hello + {world} + """ + self.assertEqual(fd(source, globals=g), " Hello\n42\n ") + + source = r""" + Hello {world} Lorum + ipsum dolor sit amet, + """ + self.assertEqual(fd(source, globals=g), "Hello 42 Lorum\nipsum dolor sit amet,\n") if __name__ == '__main__': unittest.main() From 98a26603adc93d352bfa21cf5070a20612ef8757 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 18 Mar 2026 22:35:23 +0900 Subject: [PATCH 07/10] add test cases --- Lib/test/test_dstring.py | 107 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/Lib/test/test_dstring.py b/Lib/test/test_dstring.py index 7682d0c6cd5f9d1..d05829d4fbaada8 100644 --- a/Lib/test/test_dstring.py +++ b/Lib/test/test_dstring.py @@ -1,5 +1,7 @@ import unittest +from test.test_string._support import TStringBaseCase + _dstring_prefixes = "d db df dt dr drb drf drt".split() _dstring_prefixes += [p.upper() for p in _dstring_prefixes] @@ -67,6 +69,15 @@ def test_empty_dstring(self): self.assertEqual(v, "") self.assertEqual(v2, "") + def test_missing_newline_in_plain_and_raw_prefixes(self): + exprs = [ + 'd"""x"""', + 'dr"""x"""', + 'db"""x"""', + 'drb"""x"""', + ] + self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + def check_dbstring(self, s, expected): self.assertEqual(d(s), expected) self.assertEqual(db(s), expected.encode()) @@ -132,6 +143,39 @@ def test_dbstring(self): """ self.check_dbstring(source, "Hello World!") + def test_raw_dstring(self): + source = r""" + path\\to\\file + keep\\n + """ + self.assertEqual(d(source), "path\\to\\file\nkeep\\n\n") + + def test_raw_dbstring(self): + source = r""" + path\\to\\file + keep\\n + """ + self.assertEqual(db(source), b"path\\to\\file\nkeep\\n\n") + + def test_dbstring_non_ascii_error(self): + with self.assertRaisesRegex(SyntaxError, "bytes can only contain ASCII literal characters"): + eval('db"""\n \u00e9\n """') + + def test_concat_bytes_and_nonbytes_error(self): + exprs = [ + 'd"""\n x\n """ db"""\n y\n """', + 'db"""\n x\n """ d"""\n y\n """', + ] + self.assertAllRaise(SyntaxError, "cannot mix bytes and nonbytes literals", exprs) + + +class DStringFStringInteractionTestCase(unittest.TestCase): + def assertAllRaise(self, exception_type, regex, error_strings): + for str in error_strings: + with self.subTest(str=str): + with self.assertRaisesRegex(exception_type, regex): + eval(str) + def test_fdstring(self): g = {"world": 42} @@ -153,5 +197,68 @@ def test_fdstring(self): """ self.assertEqual(fd(source, globals=g), "Hello 42 Lorum\nipsum dolor sit amet,\n") + def test_missing_newline_in_f_variants(self): + exprs = [ + 'df"""x"""', + 'drf"""x"""', + ] + self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + + def test_fdstring_conversion_and_format(self): + g = {"x": 3.14159, "name": "Alice"} + + source = r""" + {x:.2f} {name!r} + """ + self.assertEqual(fd(source, globals=g), "3.14 'Alice'\n") + + source = r""" + {x=} + """ + self.assertEqual(fd(source, globals=g), "x=3.14159\n") + + def test_concat_with_fstring(self): + expr = 'd"""\n hello\n """ f"world"' + self.assertEqual(eval(expr), "hello\nworld") + + +class DStringTStringInteractionTestCase(unittest.TestCase, TStringBaseCase): + def assertAllRaise(self, exception_type, regex, error_strings): + for str in error_strings: + with self.subTest(str=str): + with self.assertRaisesRegex(exception_type, regex): + eval(str) + + def test_missing_newline_in_t_variants(self): + exprs = [ + 'dt"""x"""', + 'drt"""x"""', + ] + self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + + def test_dtstring_basic(self): + name = "Python" + t = eval('dt"""\n Hello, {name}\n """', {"name": name}) + self.assertTStringEqual(t, ("Hello, ", "\n"), [(name, "name")]) + + def test_drtstring_raw_content(self): + t = eval('drt"""\n keep\\n\n """') + self.assertTStringEqual(t, ("keep\\n\n",), ()) + + def test_concat_with_tstring_is_rejected(self): + exprs = [ + 'd"""\n x\n """ t"hello"', + 't"hello" d"""\n x\n """', + 'db"""\n x\n """ t"hello"', + 't"hello" db"""\n x\n """', + ] + self.assertAllRaise( + SyntaxError, + "cannot mix t-string literals with string or bytes literals", + exprs, + ) + + + if __name__ == '__main__': unittest.main() From c2b0a4655d9151d52383bd75da43608f071becff Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 Jul 2026 17:37:47 +0900 Subject: [PATCH 08/10] d-string v2 --- Include/internal/pycore_unicodeobject.h | 10 + Lib/test/test_dstring.py | 109 +++++++-- Parser/action_helpers.c | 282 +++++++++++++----------- Parser/lexer/lexer.c | 95 ++++++++ Parser/lexer/state.c | 5 + Parser/lexer/state.h | 9 + Parser/string_parser.c | 55 +++-- 7 files changed, 395 insertions(+), 170 deletions(-) diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 75d5068f815b918..b5f9067360139bc 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -326,6 +326,16 @@ extern PyObject* _PyUnicode_XStrip( ); +/* Search the longest common leading whitespace of all lines in [src, end). + Returns the length of the common leading whitespace and sets `*output` to + point to the beginning of the common leading whitespace if length > 0. + Export for 'test_peg_generator.test_c_parser'. + */ +PyAPI_FUNC(Py_ssize_t) _Py_search_longest_common_leading_whitespace( + const char *src, + const char *end, + const char **output); + /* Dedent a string. Intended to dedent Python source. Unlike `textwrap.dedent`, this only supports spaces and tabs and doesn't normalize empty lines. diff --git a/Lib/test/test_dstring.py b/Lib/test/test_dstring.py index d05829d4fbaada8..a7cfffc8d34498b 100644 --- a/Lib/test/test_dstring.py +++ b/Lib/test/test_dstring.py @@ -21,12 +21,12 @@ def db(s): else: return eval(f'db"""{s}"""') -def fd(s, globals=None): - """Helper function to evaluate fd-strings.""" +def df(s, globals=None): + """Helper function to evaluate df-strings.""" if '"""' in s: - return eval(f"fd'''{s}'''", globals=globals) + return eval(f"df'''{s}'''", globals=globals) else: - return eval(f'fd"""{s}"""', globals=globals) + return eval(f'df"""{s}"""', globals=globals) @@ -126,14 +126,33 @@ def test_dbstring(self): """ self.check_dbstring(source, "hello\n\nworld\n") - # Lines with only whitespace also have their indentation removed. + # Blank lines are normalized to single newlines, even when they + # are longer than the common indentation. source = """ ....hello .. ...... ....world ....""".replace('.', ' ') - self.check_dbstring(source, "hello\n\n \nworld\n") + self.check_dbstring(source, "hello\n\n\nworld\n") + + # A blank line that does not match the common indentation is not + # an error; it is just normalized to an empty line. + source = """ +....hello +\t +....world +....""".replace('.', ' ') + self.check_dbstring(source, "hello\n\nworld\n") + + # Blank lines are normalized even when there is no common + # indentation. + source = """ +hello +.. +world +""".replace('.', ' ') + self.check_dbstring(source, "hello\n\nworld\n") # Line continuation with backslash works as usual. # But you cannot put a backslash right after the opening quotes. @@ -159,7 +178,7 @@ def test_raw_dbstring(self): def test_dbstring_non_ascii_error(self): with self.assertRaisesRegex(SyntaxError, "bytes can only contain ASCII literal characters"): - eval('db"""\n \u00e9\n """') + db('\n \u00e9\n ') def test_concat_bytes_and_nonbytes_error(self): exprs = [ @@ -176,26 +195,26 @@ def assertAllRaise(self, exception_type, regex, error_strings): with self.assertRaisesRegex(exception_type, regex): eval(str) - def test_fdstring(self): + def test_dfstring(self): g = {"world": 42} source = r""" Hello {world} """ - self.assertEqual(fd(source, globals=g), "Hello\n 42\n") + self.assertEqual(df(source, globals=g), "Hello\n 42\n") source = r""" Hello {world} """ - self.assertEqual(fd(source, globals=g), " Hello\n42\n ") + self.assertEqual(df(source, globals=g), " Hello\n42\n ") source = r""" Hello {world} Lorum ipsum dolor sit amet, """ - self.assertEqual(fd(source, globals=g), "Hello 42 Lorum\nipsum dolor sit amet,\n") + self.assertEqual(df(source, globals=g), "Hello 42 Lorum\nipsum dolor sit amet,\n") def test_missing_newline_in_f_variants(self): exprs = [ @@ -204,23 +223,72 @@ def test_missing_newline_in_f_variants(self): ] self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) - def test_fdstring_conversion_and_format(self): + def test_dfstring_conversion_and_format(self): g = {"x": 3.14159, "name": "Alice"} source = r""" {x:.2f} {name!r} """ - self.assertEqual(fd(source, globals=g), "3.14 'Alice'\n") + self.assertEqual(df(source, globals=g), "3.14 'Alice'\n") source = r""" {x=} """ - self.assertEqual(fd(source, globals=g), "x=3.14159\n") + self.assertEqual(df(source, globals=g), "x=3.14159\n") def test_concat_with_fstring(self): - expr = 'd"""\n hello\n """ f"world"' + expr = r'''d""" + hello + """ f"world"''' self.assertEqual(eval(expr), "hello\nworld") + def test_blank_line_normalization(self): + # Blank lines are normalized to single newlines, even when their + # whitespace doesn't match the common indentation. + self.assertEqual(df('\n foo\n\t\n bar {1}\n '), "foo\n\nbar 1\n") + + def test_multiline_expression_affects_indent(self): + # A line starting inside a replacement field also takes part in + # the common indentation calculation. + self.assertEqual(df(r''' + Hello {1 + + 2} + '''), " Hello 3\n ") + + def test_multiline_format_spec(self): + class Spec: + def __format__(self, spec): + return spec + + # Lines inside a multi-line format spec are dedented too. + self.assertEqual(df(r''' + {s:>6 + } + ''', globals={"s": Spec()}), ">6\n\n") + + # Nested replacement fields in the format spec keep working. + self.assertEqual(df(r''' + {s:{"a"}b + c} + ''', globals={"s": Spec()}), "ab\nc\n") + + def test_multiline_debug_text(self): + # The static text of a debug expression spanning multiple lines + # is dedented too. + self.assertEqual(df(r''' + {1 + + 1=} + '''), "1 +\n1=2\n") + + def test_nested_string_lines_affect_indent(self): + # Physical lines inside nested string literals in replacement + # fields are not excluded from the common indentation calculation. + self.assertEqual(df(r""" + {x or ''' +foo'''} + bar + """, globals={"x": ""}), " \nfoo\n bar\n ") + class DStringTStringInteractionTestCase(unittest.TestCase, TStringBaseCase): def assertAllRaise(self, exception_type, regex, error_strings): @@ -245,6 +313,17 @@ def test_drtstring_raw_content(self): t = eval('drt"""\n keep\\n\n """') self.assertTStringEqual(t, ("keep\\n\n",), ()) + def test_multiline_expression_text(self): + # The captured expression text of a multi-line interpolation is + # dedented too. + t = eval('dt"""\n {1 +\n 1}\n """') + self.assertTStringEqual(t, ("", "\n"), [(2, "1 +\n1")]) + + def test_multiline_format_spec(self): + # Lines inside a multi-line format spec are dedented too. + t = eval('dt"""\n {1:>6\n }\n """') + self.assertEqual(t.interpolations[0].format_spec, ">6\n") + def test_concat_with_tstring_is_rejected(self): exprs = [ 'd"""\n x\n """ t"hello"', diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index d5315c07f13bb13..63f97bfcd89d0ff 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1303,7 +1303,7 @@ unicodewriter_write_line(Parser *p, PyUnicodeWriter *w, const char *line_start, return PyUnicodeWriter_WriteUTF8(w, line_start, line_end - line_start); } else { - PyObject *line = _PyPegen_decode_string(p, 1, line_start, line_end - line_start, token); + PyObject *line = _PyPegen_decode_string(p, 0, line_start, line_end - line_start, token); if (line == NULL || PyUnicodeWriter_WriteStr(w, line) < 0) { Py_XDECREF(line); return -1; @@ -1316,14 +1316,13 @@ unicodewriter_write_line(Parser *p, PyUnicodeWriter *w, const char *line_start, static PyObject* _PyPegen_dedent_string_part( Parser *p, const char *s, size_t len, const char *indent, Py_ssize_t indent_len, - int is_first, int is_raw, expr_ty constant, Token* token) + int is_first, int is_raw, Token* token) { - Py_ssize_t lineno = constant->lineno; const char *line_start = s; const char *end = s + len; - int _prev_call_invalid = p->call_invalid_rules; - if (!_prev_call_invalid && !is_raw) { + int prev_call_invalid = p->call_invalid_rules; + if (!prev_call_invalid && !is_raw) { // _PyPegen_decode_string() and decode_bytes_with_escapes() may call // warn_invalid_escape_sequence(). It may emit issue or raise SyntaxError // for invalid escape sequences. @@ -1339,6 +1338,7 @@ _PyPegen_dedent_string_part( PyUnicodeWriter *w = PyUnicodeWriter_Create(len); if (w == NULL) { + p->call_invalid_rules = prev_call_invalid; return NULL; } @@ -1351,7 +1351,8 @@ _PyPegen_dedent_string_part( // first part {param} second part // next line // """" - // We don't need to dedent the first line in the non-first parts. + // The first line in a non-first part doesn't start at the + // beginning of a physical line; copy it verbatim. const char *line_end = memchr(line_start, '\n', end - line_start); if (line_end) { line_end++; // include the newline @@ -1366,32 +1367,28 @@ _PyPegen_dedent_string_part( } while (line_start < end) { - lineno++; - - Py_ssize_t i = 0; - while (line_start + i < end && i < indent_len && line_start[i] == indent[i]) { - i++; - } - - if (line_start[i] == '\0') { // found an empty line without newline. - break; + // A blank line (whitespace-only line with a newline) is normalized + // to a single newline. A whitespace-only tail without a newline is + // not blank: the physical line continues with a replacement field + // or the closing quotes. + const char *q = line_start; + while (q < end && (*q == ' ' || *q == '\t')) { + q++; } - if (line_start[i] == '\n') { // found an empty line with newline. + if (q < end && *q == '\n') { if (PyUnicodeWriter_WriteChar(w, '\n') < 0) { goto error; } - line_start += i+1; + line_start = q + 1; continue; } - if (i < indent_len) { // found an invalid indent. - assert(line_start[i] != indent[i]); - RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, lineno, i, lineno, i+1, - "d-string line missing valid indentation"); - goto error; - } - // found a indented line. let's dedent it. - line_start += i; + // A non-blank line. The common indent was computed from all + // physical lines of the literal, so it is always a prefix of + // the leading whitespace of this line. + assert(q - line_start >= indent_len); + assert(memcmp(line_start, indent, (size_t)indent_len) == 0); + line_start += indent_len; const char *line_end = memchr(line_start, '\n', end - line_start); if (line_end) { line_end++; // include the newline @@ -1408,7 +1405,7 @@ _PyPegen_dedent_string_part( return PyUnicodeWriter_Finish(w); error: - p->call_invalid_rules = _prev_call_invalid; + p->call_invalid_rules = prev_call_invalid; PyUnicodeWriter_Discard(w); return NULL; } @@ -1429,7 +1426,7 @@ _PyPegen_decode_fstring_part(Parser* p, int is_first, int is_raw, PyObject *str = NULL; if (indent != NULL) { str = _PyPegen_dedent_string_part(p, bstr, strlen(bstr), indent, indent_len, - is_first, is_raw, constant, token); + is_first, is_raw, token); } else { str = _PyPegen_decode_string(p, is_raw, bstr, strlen(bstr), token); @@ -1448,100 +1445,110 @@ _PyPegen_decode_fstring_part(Parser* p, int is_first, int is_raw, p->arena); } -/* -This function is customized version of _Py_search_longest_common_leading_whitespace() -in unicodeobject.c -*/ -static void -search_longest_common_leading_whitespace( - const char *const src, - const char *const end, - const char **indent, - Py_ssize_t *indent_len) -{ - // [_start, _start + _len) - // describes the current longest common leading whitespace - const char *_start = *indent; - Py_ssize_t _len = *indent_len; - - // skip the first line. for example: - // s = df""" - // first part - // first part{x}second part - // second part - // """ - // we don't need newline after opening qute. - // we don't need first line in the second part too. - const char *iter = memchr(src, '\n', end - src); - if (iter == NULL) { - // single line string - return; - } - - for (iter++; iter <= end; iter++) { - const char *line_start = iter; - const char *leading_whitespace_end = NULL; - - // scan the whole line - while (iter < end && *iter != '\n') { - if (!leading_whitespace_end && *iter != ' ' && *iter != '\t') { - /* `iter` points to the first non-whitespace character - in this line */ - if (iter == line_start) { - // some line has no indent, fast exit! - *indent = iter; - *indent_len = 0; - return; - } - leading_whitespace_end = iter; - } - ++iter; - } +// Dedent raw source text (debug text of `{expr=}` and the expression text +// of Interpolation nodes) that may span multiple physical lines. The first +// line starts in the middle of a physical line, so it is kept verbatim. +// Returns a new (or the same, if nothing to do) object registered in the +// arena, or NULL on error. +static PyObject * +dedent_raw_text(Parser *p, PyObject *str, const char *indent, Py_ssize_t indent_len, + Token *token) +{ + assert(PyUnicode_CheckExact(str)); + Py_ssize_t len; + const char *bstr = PyUnicode_AsUTF8AndSize(str, &len); + if (bstr == NULL) { + return NULL; + } + if (memchr(bstr, '\n', len) == NULL) { + return str; // single line, nothing to dedent + } + PyObject *res = _PyPegen_dedent_string_part(p, bstr, len, indent, indent_len, + /*is_first=*/0, /*is_raw=*/1, token); + if (res == NULL) { + return NULL; + } + if (_PyArena_AddPyObject(p->arena, res) < 0) { + Py_DECREF(res); + return NULL; + } + return res; +} - if (!leading_whitespace_end) { - // if this line has all white space, skip it - if (iter < end) { - continue; - } - leading_whitespace_end = iter; // last line may not end with '\n' - } +static int +dedent_replacement_field(Parser *p, expr_ty item, const char *indent, + Py_ssize_t indent_len, int is_raw, Token *token); - if (!_start) { - // update the first leading whitespace - _start = line_start; - _len = leading_whitespace_end - line_start; +// Dedent and decode the constant parts of a format spec inside a d-string. +// In d-strings, _PyPegen_decoded_constant_from_token() keeps the format +// spec parts as raw (undecoded) text because the common indent is unknown +// until FSTRING_END; here we dedent them and apply escape decoding. +static int +dedent_format_spec(Parser *p, expr_ty spec, const char *indent, + Py_ssize_t indent_len, int is_raw, Token *token) +{ + if (spec->kind == Constant_kind) { + if (p->call_invalid_rules) { + // In the second (error reporting) parser pass, the tokenizer has + // already finished, so _PyPegen_decoded_constant_from_token() + // cannot detect the d-string mode and decodes the format spec + // parts immediately. The AST built in this pass is only used + // for error reporting; skip dedenting to avoid double decoding. + return 0; } - else { - /* We then compare with the current longest leading whitespace. - - [line_start, leading_whitespace_end) is the leading - whitespace of this line, - - [_start, _start + _len) is the leading whitespace of the - current longest leading whitespace. */ - Py_ssize_t new_len = 0; - const char *_iter = _start, *line_iter = line_start; - - while (_iter < _start + _len && line_iter < leading_whitespace_end - && *_iter == *line_iter) - { - ++_iter; - ++line_iter; - ++new_len; + expr_ty decoded = _PyPegen_decode_fstring_part(p, /*is_first=*/0, is_raw, + indent, indent_len, spec, token); + if (decoded == NULL) { + return -1; + } + spec->v.Constant.value = decoded->v.Constant.value; + return 0; + } + if (spec->kind == JoinedStr_kind) { + asdl_expr_seq *values = spec->v.JoinedStr.values; + for (Py_ssize_t i = 0; i < asdl_seq_LEN(values); i++) { + expr_ty value = asdl_seq_GET(values, i); + if (value->kind == Constant_kind) { + if (dedent_format_spec(p, value, indent, indent_len, is_raw, token) < 0) { + return -1; + } } - - _len = new_len; - if (_len == 0) { - // No common things now, fast exit! - *indent = _start; - *indent_len = 0; - return; + else if (dedent_replacement_field(p, value, indent, indent_len, + is_raw, token) < 0) { + return -1; } } + return 0; } + return dedent_replacement_field(p, spec, indent, indent_len, is_raw, token); +} - *indent = _start; - *indent_len = _len; +// Apply d-string dedent to the parts of a FormattedValue/Interpolation +// node that carry source text spanning multiple physical lines: the format +// spec and (for t-strings) the expression text. +static int +dedent_replacement_field(Parser *p, expr_ty item, const char *indent, + Py_ssize_t indent_len, int is_raw, Token *token) +{ + expr_ty format_spec = NULL; + if (item->kind == FormattedValue_kind) { + format_spec = item->v.FormattedValue.format_spec; + } + else if (item->kind == Interpolation_kind) { + format_spec = item->v.Interpolation.format_spec; + PyObject *exprstr = item->v.Interpolation.str; + if (exprstr != NULL && PyUnicode_CheckExact(exprstr)) { + PyObject *dedented = dedent_raw_text(p, exprstr, indent, indent_len, token); + if (dedented == NULL) { + return -1; + } + item->v.Interpolation.str = dedented; + } + } + if (format_spec != NULL) { + return dedent_format_spec(p, format_spec, indent, indent_len, is_raw, token); + } + return 0; } @@ -1590,22 +1597,20 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b return NULL; } - for (Py_ssize_t i = 0; i < n_items; i++) { - expr_ty item = asdl_seq_GET(raw_expressions, i); - if (item->kind == Constant_kind) { - Py_ssize_t blen; - const char *bstr = PyUnicode_AsUTF8AndSize(item->v.Constant.value, &blen); - if (bstr == NULL) { - return NULL; - } - search_longest_common_leading_whitespace(bstr, bstr + blen, &indent_start, &indent_len); + // The longest common leading whitespace is computed by the + // tokenizer over all physical lines of the literal (including + // lines starting inside replacement fields) and attached to the + // FSTRING_END/TSTRING_END token as metadata. + assert(b->metadata != NULL && PyUnicode_CheckExact(b->metadata)); + if (b->metadata != NULL && PyUnicode_CheckExact(b->metadata)) { + indent_start = PyUnicode_AsUTF8AndSize(b->metadata, &indent_len); + if (indent_start == NULL) { + return NULL; } } - - assert(indent_start != NULL); // TODO: is this assert true? - // _py_serach_longest_common_leading_whitespace() may not set indent_start when string is empty. - if (indent_len == 0) { + else { indent_start = ""; + indent_len = 0; } } @@ -1630,10 +1635,24 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b expr_ty first = asdl_seq_GET(values, 0); assert(first->kind == Constant_kind); + if (is_dedent) { + // The debug text of `{expr=}` is raw source text that may + // span multiple physical lines; dedent it too. + PyObject *dedented = dedent_raw_text(p, first->v.Constant.value, + indent_start, indent_len, b); + if (dedented == NULL) { + return NULL; + } + first->v.Constant.value = dedented; + } asdl_seq_SET(seq, index++, first); expr_ty second = asdl_seq_GET(values, 1); assert((string_kind == TSTRING && second->kind == Interpolation_kind) || second->kind == FormattedValue_kind); + if (is_dedent && dedent_replacement_field(p, second, indent_start, + indent_len, is_raw, b) < 0) { + return NULL; + } asdl_seq_SET(seq, index++, second); continue; @@ -1653,6 +1672,10 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b continue; } } + else if (is_dedent && dedent_replacement_field(p, item, indent_start, + indent_len, is_raw, b) < 0) { + return NULL; + } asdl_seq_SET(seq, index++, item); } @@ -1707,6 +1730,13 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) { int is_raw = 0; if (INSIDE_FSTRING(p->tok)) { tokenizer_mode *mode = TOK_GET_MODE(p->tok); + if (mode->dedent) { + // PEP 822: dedent must happen before escape decoding, but the + // common indent isn't known until FSTRING_END/TSTRING_END. + // Keep the raw text; _get_resized_exprs() dedents and decodes + // format spec parts of d-strings later. + return _PyPegen_constant_from_token(p, tok); + } is_raw = mode->raw; } diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 07c61f1cb4386b3..a8ce069436724fe 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -55,6 +55,72 @@ contains_null_bytes(const char* str, size_t size) return memchr(str, 0, size) != NULL; } +/* PEP 822: fold the leading whitespace of the freshly read physical line + [tok->cur, tok->inp) into the longest common leading whitespace of every + active d-string. All physical lines between the opening and the closing + quotes are considered, including lines starting inside replacement + fields and nested string literals. Blank lines (whitespace-only) are + ignored. The line that contains the closing quotes is naturally + non-blank because the quote characters are on it. + Return 1 on success, 0 on memory error (tok->done is set). */ +static int +update_dstring_indents(struct tok_state *tok) +{ + int any_dedent = 0; + for (int i = 1; i <= tok->tok_mode_stack_index; i++) { + if (tok->tok_mode_stack[i].dedent) { + any_dedent = 1; + break; + } + } + if (!any_dedent) { + return 1; + } + + const char *line = tok->cur; + const char *end = tok->inp; + const char *ws_end = line; + while (ws_end < end && (*ws_end == ' ' || *ws_end == '\t')) { + ws_end++; + } + if (ws_end == end || *ws_end == '\n' || *ws_end == '\r') { + // A blank line. It doesn't affect the common leading whitespace. + return 1; + } + Py_ssize_t len = ws_end - line; + + for (int i = 1; i <= tok->tok_mode_stack_index; i++) { + tokenizer_mode *mode = &(tok->tok_mode_stack[i]); + if (!mode->dedent) { + continue; + } + if (!mode->dedent_seen_line) { + mode->dedent_seen_line = 1; + assert(mode->dedent_indent == NULL); + mode->dedent_indent_len = 0; + if (len > 0) { + mode->dedent_indent = PyMem_Malloc(len); + if (mode->dedent_indent == NULL) { + tok->done = E_NOMEM; + tok->cur = tok->inp; + return 0; + } + memcpy(mode->dedent_indent, line, len); + mode->dedent_indent_len = len; + } + } + else { + Py_ssize_t common = 0; + while (common < mode->dedent_indent_len && common < len + && mode->dedent_indent[common] == line[common]) { + common++; + } + mode->dedent_indent_len = common; + } + } + return 1; +} + /* Get next char, updating state; error code goes into tok->done */ static int tok_nextc(struct tok_state *tok) @@ -91,6 +157,10 @@ tok_nextc(struct tok_state *tok) tok->cur = tok->inp; return EOF; } + + if (tok->tok_mode_stack_index > 0 && !update_dstring_indents(tok)) { + return EOF; + } } Py_UNREACHABLE(); } @@ -1106,6 +1176,10 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t the_current_tok->last_expr_end = -1; the_current_tok->in_format_spec = 0; the_current_tok->in_debug = 0; + the_current_tok->dedent = 0; + the_current_tok->dedent_seen_line = 0; + the_current_tok->dedent_indent = NULL; + the_current_tok->dedent_indent_len = 0; enum string_kind_t string_kind = FSTRING; for (const char *p = tok->start; *p != c; p++) { @@ -1126,6 +1200,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t if (quote_size != 3) { return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "d-string must be triple-quoted")); } + the_current_tok->dedent = 1; break; default: Py_UNREACHABLE(); @@ -1448,6 +1523,26 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct current_tok->last_expr_end = -1; } + if (current_tok->dedent) { + // Attach the longest common leading whitespace of the whole + // d-string to the FSTRING_END/TSTRING_END token so that + // _PyPegen_joined_str()/_PyPegen_template_str() can dedent + // the string parts. + PyObject *indent = PyUnicode_FromStringAndSize( + current_tok->dedent_indent == NULL ? "" : current_tok->dedent_indent, + current_tok->dedent_indent_len); + if (current_tok->dedent_indent != NULL) { + PyMem_Free(current_tok->dedent_indent); + current_tok->dedent_indent = NULL; + current_tok->dedent_indent_len = 0; + } + if (indent == NULL) { + return MAKE_TOKEN(ERRORTOKEN); + } + assert(token->metadata == NULL); + token->metadata = indent; + } + p_start = tok->start; p_end = tok->cur; tok->tok_mode_stack_index--; diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 5cf9b4d768c3ebb..429ae64405a1546 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -81,6 +81,11 @@ free_fstring_expressions(struct tok_state *tok) mode->last_expr_end = -1; mode->in_format_spec = 0; } + if (mode->dedent_indent != NULL) { + PyMem_Free(mode->dedent_indent); + mode->dedent_indent = NULL; + mode->dedent_indent_len = 0; + } } } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 9cd196a114c7cb1..de285f02f50573d 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -67,6 +67,15 @@ typedef struct _tokenizer_mode { int in_debug; int in_format_spec; + /* PEP 822 d-string support. While tokenizing a d-string, the longest + common leading whitespace of all physical lines seen so far is + maintained in [dedent_indent, dedent_indent + dedent_indent_len). + It is attached to the FSTRING_END/TSTRING_END token as metadata. */ + int dedent; + int dedent_seen_line; + char *dedent_indent; + Py_ssize_t dedent_indent_len; + enum string_kind_t string_kind; } tokenizer_mode; diff --git a/Parser/string_parser.c b/Parser/string_parser.c index 3425b856796fe5a..6866997d5f397cf 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -258,7 +258,7 @@ _Py_search_longest_common_leading_whitespace( // Dedent d-string and return result as a bytes. static PyObject* _PyPegen_dedent_string(Parser *p, const char *s, Py_ssize_t len, - const char *indent, Py_ssize_t indent_len, int lineno) + const char *indent, Py_ssize_t indent_len) { PyBytesWriter *w = PyBytesWriter_Create(0); if (w == NULL) { @@ -266,31 +266,29 @@ _PyPegen_dedent_string(Parser *p, const char *s, Py_ssize_t len, } const char *end = s + len; - for (; s < end; lineno++) { - Py_ssize_t i; - for (i = 0; i < indent_len; i++) { - if (s[i] != indent[i]) { - if (s[i] == '\n') { - break; // empty line - } - PyBytesWriter_Discard(w); - RAISE_ERROR_KNOWN_LOCATION(p, PyExc_IndentationError, lineno, i, lineno, i+1, - "d-string missing valid indentation"); - return NULL; - } + while (s < end) { + // A blank line (whitespace-only line with a newline) is normalized + // to a single newline. The last line (before the closing quotes) + // has no newline and is never considered blank. + const char *q = s; + while (q < end && (*q == ' ' || *q == '\t')) { + q++; } - - if (s[i] == '\n') { // found an empty line with newline. + if (q < end && *q == '\n') { if (PyBytesWriter_WriteBytes(w, "\n", 1) < 0) { PyBytesWriter_Discard(w); return NULL; } - s += i+1; + s = q + 1; continue; } - // found a indented line. let's dedent it. - s += i; + // A non-blank line. The common indent was computed from all lines + // including the closing quotes line, so it is always a prefix of + // the leading whitespace of this line. + assert(q - s >= indent_len); + assert(memcmp(s, indent, (size_t)indent_len) == 0); + s += indent_len; const char *line_end = memchr(s, '\n', end - s); if (line_end == NULL) { line_end = end; // last line without newline @@ -419,20 +417,19 @@ _PyPegen_parse_string(Parser *p, Token *t) // for indent calculation. const char *end = s + len; assert(*end == '"' || *end == '\''); // end[0:3] is the trailing quotes - const char *indent; + const char *indent = ""; Py_ssize_t indent_len = _Py_search_longest_common_leading_whitespace(s+1, end+1, &indent); s++; len--; // skip the first newline - if (indent_len > 0) { - // dedent the string - dedent_bytes = _PyPegen_dedent_string(p, s, len, indent, indent_len, t->lineno + 1); - if (dedent_bytes == NULL) { - return NULL; - } - if (PyBytes_AsStringAndSize(dedent_bytes, (char**)&s, (Py_ssize_t*)&len) < 0) { - Py_DECREF(dedent_bytes); - return NULL; - } + // Dedent even when indent_len == 0: blank lines must still be + // normalized to single newlines. + dedent_bytes = _PyPegen_dedent_string(p, s, len, indent, indent_len); + if (dedent_bytes == NULL) { + return NULL; + } + if (PyBytes_AsStringAndSize(dedent_bytes, (char**)&s, (Py_ssize_t*)&len) < 0) { + Py_DECREF(dedent_bytes); + return NULL; } p->call_invalid_rules = 1; From 2357f93de72f43e9b3e3d5856ac52a4053b6a092 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 3 Jul 2026 18:59:22 +0900 Subject: [PATCH 09/10] improve test --- Lib/test/test_dstring.py | 118 +++++++++++++++++++++++++++++++++++---- Parser/lexer/state.h | 11 ++++ Parser/string_parser.c | 8 --- 3 files changed, 118 insertions(+), 19 deletions(-) diff --git a/Lib/test/test_dstring.py b/Lib/test/test_dstring.py index a7cfffc8d34498b..1ca02260d909afe 100644 --- a/Lib/test/test_dstring.py +++ b/Lib/test/test_dstring.py @@ -1,10 +1,22 @@ +import itertools import unittest from test.test_string._support import TStringBaseCase -_dstring_prefixes = "d db df dt dr drb drf drt".split() -_dstring_prefixes += [p.upper() for p in _dstring_prefixes] +def _prefix_variants(prefix): + variants = set() + for permutation in itertools.permutations(prefix): + for letters in itertools.product( + *((c.lower(), c.upper()) for c in permutation) + ): + variants.add("".join(letters)) + return sorted(variants) + + +_dstring_prefixes = [] +for _prefix in "d db df dt dr drb drf drt".split(): + _dstring_prefixes.extend(_prefix_variants(_prefix)) def d(s): @@ -28,6 +40,13 @@ def df(s, globals=None): else: return eval(f'df"""{s}"""', globals=globals) +def dt(s, globals=None): + """Helper function to evaluate dt-strings.""" + if '"""' in s: + return eval(f"dt'''{s}'''", globals=globals) + else: + return eval(f'dt"""{s}"""', globals=globals) + class DStringTestCase(unittest.TestCase): @@ -78,6 +97,18 @@ def test_missing_newline_in_plain_and_raw_prefixes(self): ] self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + def test_backslash_after_opening_quotes(self): + exprs = [ + f'{p}"""\\\nhello\n"""' for p in _dstring_prefixes + ] + self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + + def test_u_prefix_is_rejected(self): + exprs = [ + f'{p}"""\nhello\n"""' for p in _prefix_variants("du") + ] + self.assertAllRaise(SyntaxError, "'u' and 'd' prefixes are incompatible", exprs) + def check_dbstring(self, s, expected): self.assertEqual(d(s), expected) self.assertEqual(db(s), expected.encode()) @@ -100,9 +131,10 @@ def test_dbstring(self): source = """ .line1 ...line2 + line3 ..""".replace('.', ' ') - self.check_dbstring(source, " line1\n line2\nline3\n ") + self.check_dbstring(source, " line1\n line2\n\nline3\n ") # Dedent with tabs source = """ @@ -167,14 +199,7 @@ def test_raw_dstring(self): path\\to\\file keep\\n """ - self.assertEqual(d(source), "path\\to\\file\nkeep\\n\n") - - def test_raw_dbstring(self): - source = r""" - path\\to\\file - keep\\n - """ - self.assertEqual(db(source), b"path\\to\\file\nkeep\\n\n") + self.check_dbstring(source, "path\\to\\file\nkeep\\n\n") def test_dbstring_non_ascii_error(self): with self.assertRaisesRegex(SyntaxError, "bytes can only contain ASCII literal characters"): @@ -242,6 +267,14 @@ def test_concat_with_fstring(self): """ f"world"''' self.assertEqual(eval(expr), "hello\nworld") + def test_closing_quote_on_content_line(self): + g = {"value": "Python"} + + source = r""" + hello {value} + world""" + self.assertEqual(df(source, globals=g), "hello Python\n world") + def test_blank_line_normalization(self): # Blank lines are normalized to single newlines, even when their # whitespace doesn't match the common indentation. @@ -280,6 +313,11 @@ def test_multiline_debug_text(self): 1=} '''), "1 +\n1=2\n") + self.assertEqual(df(r''' + {24 * + 3=} + '''), " 24 *\n 3=72\n") + def test_nested_string_lines_affect_indent(self): # Physical lines inside nested string literals in replacement # fields are not excluded from the common indentation calculation. @@ -289,6 +327,51 @@ def test_nested_string_lines_affect_indent(self): bar """, globals={"x": ""}), " \nfoo\n bar\n ") + def test_nested_dstring_inside_dfstring(self): + # A nested d-string literal in a replacement field is dedented + # independently when the expression is evaluated. + expr = r'''df""" + outer line + {d""" + nested + line + """} + """''' + self.assertEqual(eval(expr), "outer line\n nested\nline\n\n") + + # Unlike a regular triple-quoted string, the nested d-string + # content is dedented rather than preserving indentation from the + # surrounding source. + expr = r'''df""" + {x or d""" + foo + bar + """} + outer + """''' + self.assertEqual(eval(expr, {"x": ""}), "foo\nbar\n\nouter\n") + + # Nested df-strings also work and interpolate normally. + expr = r'''df""" + prefix {df""" + value {42} + """} suffix + """''' + self.assertEqual(eval(expr), "prefix value 42\n suffix\n") + + # Nested raw d-strings keep backslashes. + expr = r'''df""" + {dr""" + path\\to\\file + """} + """''' + self.assertEqual(eval(expr), r"path\\to\\file" + "\n\n") + + # Nested d-strings must still start with a newline. + expr = 'df"""\n {d"""x"""}\n """' + with self.assertRaisesRegex(SyntaxError, "d-string must start with a newline"): + eval(expr) + class DStringTStringInteractionTestCase(unittest.TestCase, TStringBaseCase): def assertAllRaise(self, exception_type, regex, error_strings): @@ -309,6 +392,13 @@ def test_dtstring_basic(self): t = eval('dt"""\n Hello, {name}\n """', {"name": name}) self.assertTStringEqual(t, ("Hello, ", "\n"), [(name, "name")]) + def test_closing_quote_on_content_line(self): + value = "Python" + t = dt(r""" + Hello, {value} + goodbye""", globals={"value": value}) + self.assertTStringEqual(t, ("Hello, ", "\n goodbye"), [(value, "value")]) + def test_drtstring_raw_content(self): t = eval('drt"""\n keep\\n\n """') self.assertTStringEqual(t, ("keep\\n\n",), ()) @@ -319,6 +409,12 @@ def test_multiline_expression_text(self): t = eval('dt"""\n {1 +\n 1}\n """') self.assertTStringEqual(t, ("", "\n"), [(2, "1 +\n1")]) + def test_multiline_expression_affects_indent(self): + # A line starting inside a replacement field also takes part in + # the common indentation calculation. + t = eval('dt"""\n Hello {1 +\n 2}\n """') + self.assertTStringEqual(t, (" Hello ", "\n "), [(3, "1 +\n2")]) + def test_multiline_format_spec(self): # Lines inside a multi-line format spec are dedented too. t = eval('dt"""\n {1:>6\n }\n """') diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index de285f02f50573d..a3d421c9c27d5c9 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -30,6 +30,17 @@ struct token { int level; int lineno, col_offset, end_lineno, end_col_offset; const char *start, *end; + /* Extra data (str or NULL) attached to some tokens, used by the parser + actions. The lexer sets it in two (mutually exclusive) cases: + - On the '}', ':' or '!' token terminating a replacement field + expression: the source text of the expression, for f-string debug + expressions (`f"{expr=}"`) and t-string Interpolation.str + (see set_ftstring_expr()). + - On the FSTRING_END/TSTRING_END token of a d-string: the longest + common leading whitespace of all physical lines of the literal, + used to dedent the string parts (PEP 822). + The owner is this struct (freed by _PyToken_Free()); pegen's + initialize_token() moves it into the arena-managed Token. */ PyObject *metadata; }; diff --git a/Parser/string_parser.c b/Parser/string_parser.c index 6866997d5f397cf..651b4d4161117cf 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -247,14 +247,6 @@ _PyPegen_decode_string(Parser *p, int raw, const char *s, size_t len, Token *t) return decode_unicode_with_escapes(p, s, len, t); } -/* defined in unicodeobject.c */ -extern Py_ssize_t -_Py_search_longest_common_leading_whitespace( - const char *const src, - const char *const end, - const char **output - ); - // Dedent d-string and return result as a bytes. static PyObject* _PyPegen_dedent_string(Parser *p, const char *s, Py_ssize_t len, From 907c747bd6244470ff8eec88c4f52b90cf6cf59a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 7 Jul 2026 11:01:16 +0000 Subject: [PATCH 10/10] fixup --- Parser/action_helpers.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 63f97bfcd89d0ff..f478bf68ceaed5f 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1401,7 +1401,7 @@ _PyPegen_dedent_string_part( } line_start = line_end; } - p->call_invalid_rules = _prev_call_invalid; + p->call_invalid_rules = prev_call_invalid; return PyUnicodeWriter_Finish(w); error: