diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 75d5068f815b918..b5f9067360139bc 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -326,6 +326,16 @@ extern PyObject* _PyUnicode_XStrip( ); +/* Search the longest common leading whitespace of all lines in [src, end). + Returns the length of the common leading whitespace and sets `*output` to + point to the beginning of the common leading whitespace if length > 0. + Export for 'test_peg_generator.test_c_parser'. + */ +PyAPI_FUNC(Py_ssize_t) _Py_search_longest_common_leading_whitespace( + const char *src, + const char *end, + const char **output); + /* Dedent a string. Intended to dedent Python source. Unlike `textwrap.dedent`, this only supports spaces and tabs and doesn't normalize empty lines. diff --git a/Lib/test/test_dstring.py b/Lib/test/test_dstring.py new file mode 100644 index 000000000000000..1ca02260d909afe --- /dev/null +++ b/Lib/test/test_dstring.py @@ -0,0 +1,439 @@ +import itertools +import unittest + +from test.test_string._support import TStringBaseCase + + +def _prefix_variants(prefix): + variants = set() + for permutation in itertools.permutations(prefix): + for letters in itertools.product( + *((c.lower(), c.upper()) for c in permutation) + ): + variants.add("".join(letters)) + return sorted(variants) + + +_dstring_prefixes = [] +for _prefix in "d db df dt dr drb drf drt".split(): + _dstring_prefixes.extend(_prefix_variants(_prefix)) + + +def d(s): + """Helper function to evaluate d-strings.""" + if '"""' in s: + return eval(f"d'''{s}'''") + else: + return eval(f'd"""{s}"""') + +def db(s): + """Helper function to evaluate db-strings.""" + if '"""' in s: + return eval(f"db'''{s}'''") + else: + return eval(f'db"""{s}"""') + +def df(s, globals=None): + """Helper function to evaluate df-strings.""" + if '"""' in s: + return eval(f"df'''{s}'''", globals=globals) + else: + return eval(f'df"""{s}"""', globals=globals) + +def dt(s, globals=None): + """Helper function to evaluate dt-strings.""" + if '"""' in s: + return eval(f"dt'''{s}'''", globals=globals) + else: + return eval(f'dt"""{s}"""', globals=globals) + + + +class DStringTestCase(unittest.TestCase): + def assertAllRaise(self, exception_type, regex, error_strings): + for str in error_strings: + with self.subTest(str=str): + with self.assertRaisesRegex(exception_type, regex) as cm: + eval(str) + + def test_single_quote(self): + exprs = [ + f"{p}'hello, world'" for p in _dstring_prefixes + ] + [ + f'{p}"hello, world"' for p in _dstring_prefixes + ] + self.assertAllRaise(SyntaxError, "d-string must be triple-quoted", exprs) + + def test_empty_dstring(self): + exprs = [ + f"{p}''''''" for p in _dstring_prefixes + ] + [ + f'{p}""""""' for p in _dstring_prefixes + ] + self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + + for prefix in _dstring_prefixes: + expr = f"{prefix}'''\n'''" + expr2 = f'{prefix}"""\n"""' + with self.subTest(expr=expr): + v = eval(expr) + v2 = eval(expr2) + if 't' in prefix.lower(): + self.assertEqual(v.strings, ("",)) + self.assertEqual(v2.strings, ("",)) + elif 'b' in prefix.lower(): + self.assertEqual(v, b"") + self.assertEqual(v2, b"") + else: + self.assertEqual(v, "") + self.assertEqual(v2, "") + + def test_missing_newline_in_plain_and_raw_prefixes(self): + exprs = [ + 'd"""x"""', + 'dr"""x"""', + 'db"""x"""', + 'drb"""x"""', + ] + self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + + def test_backslash_after_opening_quotes(self): + exprs = [ + f'{p}"""\\\nhello\n"""' for p in _dstring_prefixes + ] + self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + + def test_u_prefix_is_rejected(self): + exprs = [ + f'{p}"""\nhello\n"""' for p in _prefix_variants("du") + ] + self.assertAllRaise(SyntaxError, "'u' and 'd' prefixes are incompatible", exprs) + + def check_dbstring(self, s, expected): + self.assertEqual(d(s), expected) + self.assertEqual(db(s), expected.encode()) + + def test_dbstring(self): + # Basic dedent - remove common leading whitespace + source = """ + hello + world + """ + self.check_dbstring(source, "hello\nworld\n") + + # closing quote on same line as last content line + source = """ + hello + world""" + self.check_dbstring(source, "hello\nworld") + + # Dedent with varying indentation + source = """ + .line1 + ...line2 + + line3 + ..""".replace('.', ' ') + self.check_dbstring(source, " line1\n line2\n\nline3\n ") + + # Dedent with tabs + source = """ +\thello +\tworld +\t""" + self.check_dbstring(source, "hello\nworld\n") + + # Mixed spaces and tabs (using common leading whitespace) + source = """ +\t\t hello +\t\t world +\t\t """ + self.check_dbstring(source, " hello\n world\n") + + # Empty lines do not affect the calculation of common leading whitespace + source = """ + hello + + world + """ + self.check_dbstring(source, "hello\n\nworld\n") + + # Blank lines are normalized to single newlines, even when they + # are longer than the common indentation. + source = """ +....hello +.. +...... +....world +....""".replace('.', ' ') + self.check_dbstring(source, "hello\n\n\nworld\n") + + # A blank line that does not match the common indentation is not + # an error; it is just normalized to an empty line. + source = """ +....hello +\t +....world +....""".replace('.', ' ') + self.check_dbstring(source, "hello\n\nworld\n") + + # Blank lines are normalized even when there is no common + # indentation. + source = """ +hello +.. +world +""".replace('.', ' ') + self.check_dbstring(source, "hello\n\nworld\n") + + # Line continuation with backslash works as usual. + # But you cannot put a backslash right after the opening quotes. + source = r""" + Hello \ + World!\ + """ + self.check_dbstring(source, "Hello World!") + + def test_raw_dstring(self): + source = r""" + path\\to\\file + keep\\n + """ + self.check_dbstring(source, "path\\to\\file\nkeep\\n\n") + + def test_dbstring_non_ascii_error(self): + with self.assertRaisesRegex(SyntaxError, "bytes can only contain ASCII literal characters"): + db('\n \u00e9\n ') + + def test_concat_bytes_and_nonbytes_error(self): + exprs = [ + 'd"""\n x\n """ db"""\n y\n """', + 'db"""\n x\n """ d"""\n y\n """', + ] + self.assertAllRaise(SyntaxError, "cannot mix bytes and nonbytes literals", exprs) + + +class DStringFStringInteractionTestCase(unittest.TestCase): + def assertAllRaise(self, exception_type, regex, error_strings): + for str in error_strings: + with self.subTest(str=str): + with self.assertRaisesRegex(exception_type, regex): + eval(str) + + def test_dfstring(self): + g = {"world": 42} + + source = r""" + Hello + {world} + """ + self.assertEqual(df(source, globals=g), "Hello\n 42\n") + + source = r""" + Hello + {world} + """ + self.assertEqual(df(source, globals=g), " Hello\n42\n ") + + source = r""" + Hello {world} Lorum + ipsum dolor sit amet, + """ + self.assertEqual(df(source, globals=g), "Hello 42 Lorum\nipsum dolor sit amet,\n") + + def test_missing_newline_in_f_variants(self): + exprs = [ + 'df"""x"""', + 'drf"""x"""', + ] + self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + + def test_dfstring_conversion_and_format(self): + g = {"x": 3.14159, "name": "Alice"} + + source = r""" + {x:.2f} {name!r} + """ + self.assertEqual(df(source, globals=g), "3.14 'Alice'\n") + + source = r""" + {x=} + """ + self.assertEqual(df(source, globals=g), "x=3.14159\n") + + def test_concat_with_fstring(self): + expr = r'''d""" + hello + """ f"world"''' + self.assertEqual(eval(expr), "hello\nworld") + + def test_closing_quote_on_content_line(self): + g = {"value": "Python"} + + source = r""" + hello {value} + world""" + self.assertEqual(df(source, globals=g), "hello Python\n world") + + def test_blank_line_normalization(self): + # Blank lines are normalized to single newlines, even when their + # whitespace doesn't match the common indentation. + self.assertEqual(df('\n foo\n\t\n bar {1}\n '), "foo\n\nbar 1\n") + + def test_multiline_expression_affects_indent(self): + # A line starting inside a replacement field also takes part in + # the common indentation calculation. + self.assertEqual(df(r''' + Hello {1 + + 2} + '''), " Hello 3\n ") + + def test_multiline_format_spec(self): + class Spec: + def __format__(self, spec): + return spec + + # Lines inside a multi-line format spec are dedented too. + self.assertEqual(df(r''' + {s:>6 + } + ''', globals={"s": Spec()}), ">6\n\n") + + # Nested replacement fields in the format spec keep working. + self.assertEqual(df(r''' + {s:{"a"}b + c} + ''', globals={"s": Spec()}), "ab\nc\n") + + def test_multiline_debug_text(self): + # The static text of a debug expression spanning multiple lines + # is dedented too. + self.assertEqual(df(r''' + {1 + + 1=} + '''), "1 +\n1=2\n") + + self.assertEqual(df(r''' + {24 * + 3=} + '''), " 24 *\n 3=72\n") + + def test_nested_string_lines_affect_indent(self): + # Physical lines inside nested string literals in replacement + # fields are not excluded from the common indentation calculation. + self.assertEqual(df(r""" + {x or ''' +foo'''} + bar + """, globals={"x": ""}), " \nfoo\n bar\n ") + + def test_nested_dstring_inside_dfstring(self): + # A nested d-string literal in a replacement field is dedented + # independently when the expression is evaluated. + expr = r'''df""" + outer line + {d""" + nested + line + """} + """''' + self.assertEqual(eval(expr), "outer line\n nested\nline\n\n") + + # Unlike a regular triple-quoted string, the nested d-string + # content is dedented rather than preserving indentation from the + # surrounding source. + expr = r'''df""" + {x or d""" + foo + bar + """} + outer + """''' + self.assertEqual(eval(expr, {"x": ""}), "foo\nbar\n\nouter\n") + + # Nested df-strings also work and interpolate normally. + expr = r'''df""" + prefix {df""" + value {42} + """} suffix + """''' + self.assertEqual(eval(expr), "prefix value 42\n suffix\n") + + # Nested raw d-strings keep backslashes. + expr = r'''df""" + {dr""" + path\\to\\file + """} + """''' + self.assertEqual(eval(expr), r"path\\to\\file" + "\n\n") + + # Nested d-strings must still start with a newline. + expr = 'df"""\n {d"""x"""}\n """' + with self.assertRaisesRegex(SyntaxError, "d-string must start with a newline"): + eval(expr) + + +class DStringTStringInteractionTestCase(unittest.TestCase, TStringBaseCase): + def assertAllRaise(self, exception_type, regex, error_strings): + for str in error_strings: + with self.subTest(str=str): + with self.assertRaisesRegex(exception_type, regex): + eval(str) + + def test_missing_newline_in_t_variants(self): + exprs = [ + 'dt"""x"""', + 'drt"""x"""', + ] + self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + + def test_dtstring_basic(self): + name = "Python" + t = eval('dt"""\n Hello, {name}\n """', {"name": name}) + self.assertTStringEqual(t, ("Hello, ", "\n"), [(name, "name")]) + + def test_closing_quote_on_content_line(self): + value = "Python" + t = dt(r""" + Hello, {value} + goodbye""", globals={"value": value}) + self.assertTStringEqual(t, ("Hello, ", "\n goodbye"), [(value, "value")]) + + def test_drtstring_raw_content(self): + t = eval('drt"""\n keep\\n\n """') + self.assertTStringEqual(t, ("keep\\n\n",), ()) + + def test_multiline_expression_text(self): + # The captured expression text of a multi-line interpolation is + # dedented too. + t = eval('dt"""\n {1 +\n 1}\n """') + self.assertTStringEqual(t, ("", "\n"), [(2, "1 +\n1")]) + + def test_multiline_expression_affects_indent(self): + # A line starting inside a replacement field also takes part in + # the common indentation calculation. + t = eval('dt"""\n Hello {1 +\n 2}\n """') + self.assertTStringEqual(t, (" Hello ", "\n "), [(3, "1 +\n2")]) + + def test_multiline_format_spec(self): + # Lines inside a multi-line format spec are dedented too. + t = eval('dt"""\n {1:>6\n }\n """') + self.assertEqual(t.interpolations[0].format_spec, ">6\n") + + def test_concat_with_tstring_is_rejected(self): + exprs = [ + 'd"""\n x\n """ t"hello"', + 't"hello" d"""\n x\n """', + 'db"""\n x\n """ t"hello"', + 't"hello" db"""\n x\n """', + ] + self.assertAllRaise( + SyntaxError, + "cannot mix t-string literals with string or bytes literals", + exprs, + ) + + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index ab53a20cff55392..9a878164489681a 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -3421,7 +3421,7 @@ def determine_valid_prefixes(): # some uppercase-only prefix is added. for letter in itertools.chain(string.ascii_lowercase, string.ascii_uppercase): try: - eval(f'{letter}""') + eval(f'{letter}"""\n"""') # d-string needs multiline single_char_valid_prefixes.add(letter.lower()) except SyntaxError: pass @@ -3445,7 +3445,7 @@ def determine_valid_prefixes(): # because it's a valid expression: not "" continue try: - eval(f'{p}""') + eval(f'{p}"""\n"""') # d-string needs multiline # No syntax error, so p is a valid string # prefix. diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 3545d92c4f5d7ff..3ea6c5b986ea78a 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -87,7 +87,8 @@ def _all_string_prefixes(): # The valid string prefixes. Only contain the lower case versions, # and don't contain any permutations (include 'fr', but not # 'rf'). The various permutations will be generated. - _valid_string_prefixes = ['b', 'r', 'u', 'f', 't', 'br', 'fr', 'tr'] + _valid_string_prefixes = ['b', 'r', 'u', 'f', 't', 'd', 'br', 'fr', 'tr', + 'bd', 'rd', 'fd', 'td', 'brd', 'frd', 'trd'] # if we add binary f-strings, add: ['fb', 'fbr'] result = {''} for prefix in _valid_string_prefixes: diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 3c689761de9b199..f26f9062793d2f4 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -13507,8 +13507,8 @@ of all lines in the [src, end). It returns the length of the common leading whitespace and sets `output` to point to the beginning of the common leading whitespace if length > 0. */ -static Py_ssize_t -search_longest_common_leading_whitespace( +Py_ssize_t +_Py_search_longest_common_leading_whitespace( const char *const src, const char *const end, const char **output) @@ -13604,7 +13604,7 @@ _PyUnicode_Dedent(PyObject *unicode) // [whitespace_start, whitespace_start + whitespace_len) // describes the current longest common leading whitespace const char *whitespace_start = NULL; - Py_ssize_t whitespace_len = search_longest_common_leading_whitespace( + Py_ssize_t whitespace_len = _Py_search_longest_common_leading_whitespace( src, end, &whitespace_start); if (whitespace_len == 0) { diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index c74eb34deb7c35c..f478bf68ceaed5f 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1295,24 +1295,143 @@ _PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq // Fstring stuff +static int +unicodewriter_write_line(Parser *p, PyUnicodeWriter *w, const char *line_start, const char *line_end, + int is_raw, Token* token) +{ + if (is_raw || memchr(line_start, '\\', line_end - line_start) == NULL) { + return PyUnicodeWriter_WriteUTF8(w, line_start, line_end - line_start); + } + else { + PyObject *line = _PyPegen_decode_string(p, 0, line_start, line_end - line_start, token); + if (line == NULL || PyUnicodeWriter_WriteStr(w, line) < 0) { + Py_XDECREF(line); + return -1; + } + Py_DECREF(line); + } + return 0; +} + +static PyObject* +_PyPegen_dedent_string_part( + Parser *p, const char *s, size_t len, const char *indent, Py_ssize_t indent_len, + int is_first, int is_raw, Token* token) +{ + const char *line_start = s; + const char *end = s + len; + + int prev_call_invalid = p->call_invalid_rules; + if (!prev_call_invalid && !is_raw) { + // _PyPegen_decode_string() and decode_bytes_with_escapes() may call + // warn_invalid_escape_sequence(). It may emit issue or raise SyntaxError + // for invalid escape sequences. + // We need to call it before dedenting since SyntaxError needs exact lineno + // and col_offset of invalid escape sequences. + PyObject *t = _PyPegen_decode_string(p, 0, s, len, token); + if (t == NULL) { + return NULL; + } + Py_DECREF(t); + p->call_invalid_rules = 1; + } + + PyUnicodeWriter *w = PyUnicodeWriter_Create(len); + if (w == NULL) { + p->call_invalid_rules = prev_call_invalid; + return NULL; + } + + if (is_first) { + assert (line_start[0] == '\n'); + line_start++; // skip the first newline + } + else { + // Example: df""" + // first part {param} second part + // next line + // """" + // The first line in a non-first part doesn't start at the + // beginning of a physical line; copy it verbatim. + const char *line_end = memchr(line_start, '\n', end - line_start); + if (line_end) { + line_end++; // include the newline + } + else { + line_end = end; + } + if (unicodewriter_write_line(p, w, line_start, line_end, is_raw, token) < 0) { + goto error; + } + line_start = line_end; + } + + while (line_start < end) { + // A blank line (whitespace-only line with a newline) is normalized + // to a single newline. A whitespace-only tail without a newline is + // not blank: the physical line continues with a replacement field + // or the closing quotes. + const char *q = line_start; + while (q < end && (*q == ' ' || *q == '\t')) { + q++; + } + if (q < end && *q == '\n') { + if (PyUnicodeWriter_WriteChar(w, '\n') < 0) { + goto error; + } + line_start = q + 1; + continue; + } + + // A non-blank line. The common indent was computed from all + // physical lines of the literal, so it is always a prefix of + // the leading whitespace of this line. + assert(q - line_start >= indent_len); + assert(memcmp(line_start, indent, (size_t)indent_len) == 0); + line_start += indent_len; + const char *line_end = memchr(line_start, '\n', end - line_start); + if (line_end) { + line_end++; // include the newline + } + else { + line_end = end; + } + if (unicodewriter_write_line(p, w, line_start, line_end, is_raw, token) < 0) { + goto error; + } + line_start = line_end; + } + p->call_invalid_rules = prev_call_invalid; + return PyUnicodeWriter_Finish(w); + +error: + p->call_invalid_rules = prev_call_invalid; + PyUnicodeWriter_Discard(w); + return NULL; +} + static expr_ty -_PyPegen_decode_fstring_part(Parser* p, int is_raw, expr_ty constant, Token* token) { +_PyPegen_decode_fstring_part(Parser* p, int is_first, int is_raw, + const char *indent, Py_ssize_t indent_len, + expr_ty constant, Token* token) +{ assert(PyUnicode_CheckExact(constant->v.Constant.value)); const char* bstr = PyUnicode_AsUTF8(constant->v.Constant.value); if (bstr == NULL) { return NULL; } + is_raw = is_raw || strchr(bstr, '\\') == NULL; - size_t len; - if (strcmp(bstr, "{{") == 0 || strcmp(bstr, "}}") == 0) { - len = 1; - } else { - len = strlen(bstr); + PyObject *str = NULL; + if (indent != NULL) { + str = _PyPegen_dedent_string_part(p, bstr, strlen(bstr), indent, indent_len, + is_first, is_raw, token); + } + else { + str = _PyPegen_decode_string(p, is_raw, bstr, strlen(bstr), token); } - is_raw = is_raw || strchr(bstr, '\\') == NULL; - PyObject *str = _PyPegen_decode_string(p, is_raw, bstr, len, token); if (str == NULL) { _Pypegen_raise_decode_error(p); return NULL; @@ -1326,6 +1445,113 @@ _PyPegen_decode_fstring_part(Parser* p, int is_raw, expr_ty constant, Token* tok p->arena); } +// Dedent raw source text (debug text of `{expr=}` and the expression text +// of Interpolation nodes) that may span multiple physical lines. The first +// line starts in the middle of a physical line, so it is kept verbatim. +// Returns a new (or the same, if nothing to do) object registered in the +// arena, or NULL on error. +static PyObject * +dedent_raw_text(Parser *p, PyObject *str, const char *indent, Py_ssize_t indent_len, + Token *token) +{ + assert(PyUnicode_CheckExact(str)); + Py_ssize_t len; + const char *bstr = PyUnicode_AsUTF8AndSize(str, &len); + if (bstr == NULL) { + return NULL; + } + if (memchr(bstr, '\n', len) == NULL) { + return str; // single line, nothing to dedent + } + PyObject *res = _PyPegen_dedent_string_part(p, bstr, len, indent, indent_len, + /*is_first=*/0, /*is_raw=*/1, token); + if (res == NULL) { + return NULL; + } + if (_PyArena_AddPyObject(p->arena, res) < 0) { + Py_DECREF(res); + return NULL; + } + return res; +} + +static int +dedent_replacement_field(Parser *p, expr_ty item, const char *indent, + Py_ssize_t indent_len, int is_raw, Token *token); + +// Dedent and decode the constant parts of a format spec inside a d-string. +// In d-strings, _PyPegen_decoded_constant_from_token() keeps the format +// spec parts as raw (undecoded) text because the common indent is unknown +// until FSTRING_END; here we dedent them and apply escape decoding. +static int +dedent_format_spec(Parser *p, expr_ty spec, const char *indent, + Py_ssize_t indent_len, int is_raw, Token *token) +{ + if (spec->kind == Constant_kind) { + if (p->call_invalid_rules) { + // In the second (error reporting) parser pass, the tokenizer has + // already finished, so _PyPegen_decoded_constant_from_token() + // cannot detect the d-string mode and decodes the format spec + // parts immediately. The AST built in this pass is only used + // for error reporting; skip dedenting to avoid double decoding. + return 0; + } + expr_ty decoded = _PyPegen_decode_fstring_part(p, /*is_first=*/0, is_raw, + indent, indent_len, spec, token); + if (decoded == NULL) { + return -1; + } + spec->v.Constant.value = decoded->v.Constant.value; + return 0; + } + if (spec->kind == JoinedStr_kind) { + asdl_expr_seq *values = spec->v.JoinedStr.values; + for (Py_ssize_t i = 0; i < asdl_seq_LEN(values); i++) { + expr_ty value = asdl_seq_GET(values, i); + if (value->kind == Constant_kind) { + if (dedent_format_spec(p, value, indent, indent_len, is_raw, token) < 0) { + return -1; + } + } + else if (dedent_replacement_field(p, value, indent, indent_len, + is_raw, token) < 0) { + return -1; + } + } + return 0; + } + return dedent_replacement_field(p, spec, indent, indent_len, is_raw, token); +} + +// Apply d-string dedent to the parts of a FormattedValue/Interpolation +// node that carry source text spanning multiple physical lines: the format +// spec and (for t-strings) the expression text. +static int +dedent_replacement_field(Parser *p, expr_ty item, const char *indent, + Py_ssize_t indent_len, int is_raw, Token *token) +{ + expr_ty format_spec = NULL; + if (item->kind == FormattedValue_kind) { + format_spec = item->v.FormattedValue.format_spec; + } + else if (item->kind == Interpolation_kind) { + format_spec = item->v.Interpolation.format_spec; + PyObject *exprstr = item->v.Interpolation.str; + if (exprstr != NULL && PyUnicode_CheckExact(exprstr)) { + PyObject *dedented = dedent_raw_text(p, exprstr, indent, indent_len, token); + if (dedented == NULL) { + return -1; + } + item->v.Interpolation.str = dedented; + } + } + if (format_spec != NULL) { + return dedent_format_spec(p, format_spec, indent, indent_len, is_raw, token); + } + return 0; +} + + static asdl_expr_seq * _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b, enum string_kind_t string_kind) { @@ -1343,12 +1569,51 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b return NULL; } int is_raw = strpbrk(quote_str, "rR") != NULL; + int is_dedent = strpbrk(quote_str, "dD") != NULL; asdl_expr_seq *seq = _Py_asdl_expr_seq_new(total_items, p->arena); if (seq == NULL) { return NULL; } + const char *indent_start = NULL; + Py_ssize_t indent_len = 0; + + if (is_dedent) { + if (total_items == 0) { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + a, + "d-string must start with a newline" + ); + return NULL; + } + expr_ty first_item = asdl_seq_GET(raw_expressions, 0); + if (first_item->kind != Constant_kind + || PyUnicode_ReadChar(first_item->v.Constant.value, 0) != '\n') { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + first_item, + "d-string must start with a newline" + ); + return NULL; + } + + // The longest common leading whitespace is computed by the + // tokenizer over all physical lines of the literal (including + // lines starting inside replacement fields) and attached to the + // FSTRING_END/TSTRING_END token as metadata. + assert(b->metadata != NULL && PyUnicode_CheckExact(b->metadata)); + if (b->metadata != NULL && PyUnicode_CheckExact(b->metadata)) { + indent_start = PyUnicode_AsUTF8AndSize(b->metadata, &indent_len); + if (indent_start == NULL) { + return NULL; + } + } + else { + indent_start = ""; + indent_len = 0; + } + } + Py_ssize_t index = 0; for (Py_ssize_t i = 0; i < n_items; i++) { expr_ty item = asdl_seq_GET(raw_expressions, i); @@ -1370,17 +1635,31 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b expr_ty first = asdl_seq_GET(values, 0); assert(first->kind == Constant_kind); + if (is_dedent) { + // The debug text of `{expr=}` is raw source text that may + // span multiple physical lines; dedent it too. + PyObject *dedented = dedent_raw_text(p, first->v.Constant.value, + indent_start, indent_len, b); + if (dedented == NULL) { + return NULL; + } + first->v.Constant.value = dedented; + } asdl_seq_SET(seq, index++, first); expr_ty second = asdl_seq_GET(values, 1); assert((string_kind == TSTRING && second->kind == Interpolation_kind) || second->kind == FormattedValue_kind); + if (is_dedent && dedent_replacement_field(p, second, indent_start, + indent_len, is_raw, b) < 0) { + return NULL; + } asdl_seq_SET(seq, index++, second); continue; } if (item->kind == Constant_kind) { - item = _PyPegen_decode_fstring_part(p, is_raw, item, b); + item = _PyPegen_decode_fstring_part(p, i == 0, is_raw, indent_start, indent_len, item, b); if (item == NULL) { return NULL; } @@ -1393,6 +1672,10 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b continue; } } + else if (is_dedent && dedent_replacement_field(p, item, indent_start, + indent_len, is_raw, b) < 0) { + return NULL; + } asdl_seq_SET(seq, index++, item); } @@ -1447,6 +1730,13 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) { int is_raw = 0; if (INSIDE_FSTRING(p->tok)) { tokenizer_mode *mode = TOK_GET_MODE(p->tok); + if (mode->dedent) { + // PEP 822: dedent must happen before escape decoding, but the + // common indent isn't known until FSTRING_END/TSTRING_END. + // Keep the raw text; _get_resized_exprs() dedents and decodes + // format spec parts of d-strings later. + return _PyPegen_constant_from_token(p, tok); + } is_raw = mode->raw; } diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 7f25afec302c225..a8ce069436724fe 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -55,6 +55,72 @@ contains_null_bytes(const char* str, size_t size) return memchr(str, 0, size) != NULL; } +/* PEP 822: fold the leading whitespace of the freshly read physical line + [tok->cur, tok->inp) into the longest common leading whitespace of every + active d-string. All physical lines between the opening and the closing + quotes are considered, including lines starting inside replacement + fields and nested string literals. Blank lines (whitespace-only) are + ignored. The line that contains the closing quotes is naturally + non-blank because the quote characters are on it. + Return 1 on success, 0 on memory error (tok->done is set). */ +static int +update_dstring_indents(struct tok_state *tok) +{ + int any_dedent = 0; + for (int i = 1; i <= tok->tok_mode_stack_index; i++) { + if (tok->tok_mode_stack[i].dedent) { + any_dedent = 1; + break; + } + } + if (!any_dedent) { + return 1; + } + + const char *line = tok->cur; + const char *end = tok->inp; + const char *ws_end = line; + while (ws_end < end && (*ws_end == ' ' || *ws_end == '\t')) { + ws_end++; + } + if (ws_end == end || *ws_end == '\n' || *ws_end == '\r') { + // A blank line. It doesn't affect the common leading whitespace. + return 1; + } + Py_ssize_t len = ws_end - line; + + for (int i = 1; i <= tok->tok_mode_stack_index; i++) { + tokenizer_mode *mode = &(tok->tok_mode_stack[i]); + if (!mode->dedent) { + continue; + } + if (!mode->dedent_seen_line) { + mode->dedent_seen_line = 1; + assert(mode->dedent_indent == NULL); + mode->dedent_indent_len = 0; + if (len > 0) { + mode->dedent_indent = PyMem_Malloc(len); + if (mode->dedent_indent == NULL) { + tok->done = E_NOMEM; + tok->cur = tok->inp; + return 0; + } + memcpy(mode->dedent_indent, line, len); + mode->dedent_indent_len = len; + } + } + else { + Py_ssize_t common = 0; + while (common < mode->dedent_indent_len && common < len + && mode->dedent_indent[common] == line[common]) { + common++; + } + mode->dedent_indent_len = common; + } + } + return 1; +} + /* Get next char, updating state; error code goes into tok->done */ static int tok_nextc(struct tok_state *tok) @@ -91,6 +157,10 @@ tok_nextc(struct tok_state *tok) tok->cur = tok->inp; return EOF; } + + if (tok->tok_mode_stack_index > 0 && !update_dstring_indents(tok)) { + return EOF; + } } Py_UNREACHABLE(); } @@ -455,7 +525,7 @@ tok_continuation_line(struct tok_state *tok) { static int maybe_raise_syntax_error_for_string_prefixes(struct tok_state *tok, int saw_b, int saw_r, int saw_u, - int saw_f, int saw_t) { + int saw_f, int saw_t, int saw_d) { // Supported: rb, rf, rt (in any order) // Unsupported: ub, ur, uf, ut, bf, bt, ft (in any order) @@ -480,6 +550,9 @@ maybe_raise_syntax_error_for_string_prefixes(struct tok_state *tok, if (saw_u && saw_t) { RETURN_SYNTAX_ERROR("u", "t"); } + if (saw_u && saw_d) { + RETURN_SYNTAX_ERROR("u", "d"); + } if (saw_b && saw_f) { RETURN_SYNTAX_ERROR("b", "f"); @@ -741,8 +814,8 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t /* Identifier (most frequent token!) */ nonascii = 0; if (is_potential_identifier_start(c)) { - /* Process the various legal combinations of b"", r"", u"", and f"". */ - int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0, saw_t = 0; + /* Process the various legal combinations of b"", r"", u"", f"", and d"". */ + int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0, saw_t = 0, saw_d = 0; while (1) { if (!saw_b && (c == 'b' || c == 'B')) { saw_b = 1; @@ -762,6 +835,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t else if (!saw_t && (c == 't' || c == 'T')) { saw_t = 1; } + else if (!saw_d && (c == 'd' || c == 'D')) { + saw_d = 1; + } else { break; } @@ -769,7 +845,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t if (c == '"' || c == '\'') { // Raise error on incompatible string prefixes: int status = maybe_raise_syntax_error_for_string_prefixes( - tok, saw_b, saw_r, saw_u, saw_f, saw_t); + tok, saw_b, saw_r, saw_u, saw_f, saw_t, saw_d); if (status < 0) { return MAKE_TOKEN(ERRORTOKEN); } @@ -1049,7 +1125,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t } f_string_quote: - if (((Py_TOLOWER(*tok->start) == 'f' || Py_TOLOWER(*tok->start) == 'r' || Py_TOLOWER(*tok->start) == 't') + if (((Py_TOLOWER(*tok->start) == 'f' || Py_TOLOWER(*tok->start) == 'r' || Py_TOLOWER(*tok->start) == 't' || Py_TOLOWER(*tok->start) == 'd') && (c == '\'' || c == '"'))) { int quote = c; @@ -1089,6 +1165,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t the_current_tok->kind = TOK_FSTRING_MODE; the_current_tok->quote = quote; the_current_tok->quote_size = quote_size; + the_current_tok->raw = 0; the_current_tok->start = tok->start; the_current_tok->multi_line_start = tok->line_start; the_current_tok->first_line = tok->lineno; @@ -1099,27 +1176,35 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t the_current_tok->last_expr_end = -1; the_current_tok->in_format_spec = 0; the_current_tok->in_debug = 0; + the_current_tok->dedent = 0; + the_current_tok->dedent_seen_line = 0; + the_current_tok->dedent_indent = NULL; + the_current_tok->dedent_indent_len = 0; enum string_kind_t string_kind = FSTRING; - switch (*tok->start) { - case 'T': - case 't': - the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r'; - string_kind = TSTRING; - break; - case 'F': - case 'f': - the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r'; - break; - case 'R': - case 'r': - the_current_tok->raw = 1; - if (Py_TOLOWER(*(tok->start + 1)) == 't') { + for (const char *p = tok->start; *p != c; p++) { + switch (*p) { + case 'f': + case 'F': + break; + case 't': + case 'T': string_kind = TSTRING; - } - break; - default: - Py_UNREACHABLE(); + break; + case 'r': + case 'R': + the_current_tok->raw = 1; + break; + case 'd': + case 'D': + if (quote_size != 3) { + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "d-string must be triple-quoted")); + } + the_current_tok->dedent = 1; + break; + default: + Py_UNREACHABLE(); + } } the_current_tok->string_kind = string_kind; @@ -1438,6 +1523,26 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct current_tok->last_expr_end = -1; } + if (current_tok->dedent) { + // Attach the longest common leading whitespace of the whole + // d-string to the FSTRING_END/TSTRING_END token so that + // _PyPegen_joined_str()/_PyPegen_template_str() can dedent + // the string parts. + PyObject *indent = PyUnicode_FromStringAndSize( + current_tok->dedent_indent == NULL ? "" : current_tok->dedent_indent, + current_tok->dedent_indent_len); + if (current_tok->dedent_indent != NULL) { + PyMem_Free(current_tok->dedent_indent); + current_tok->dedent_indent = NULL; + current_tok->dedent_indent_len = 0; + } + if (indent == NULL) { + return MAKE_TOKEN(ERRORTOKEN); + } + assert(token->metadata == NULL); + token->metadata = indent; + } + p_start = tok->start; p_end = tok->cur; tok->tok_mode_stack_index--; diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 5cf9b4d768c3ebb..429ae64405a1546 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -81,6 +81,11 @@ free_fstring_expressions(struct tok_state *tok) mode->last_expr_end = -1; mode->in_format_spec = 0; } + if (mode->dedent_indent != NULL) { + PyMem_Free(mode->dedent_indent); + mode->dedent_indent = NULL; + mode->dedent_indent_len = 0; + } } } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 9cd196a114c7cb1..a3d421c9c27d5c9 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -30,6 +30,17 @@ struct token { int level; int lineno, col_offset, end_lineno, end_col_offset; const char *start, *end; + /* Extra data (str or NULL) attached to some tokens, used by the parser + actions. The lexer sets it in two (mutually exclusive) cases: + - On the '}', ':' or '!' token terminating a replacement field + expression: the source text of the expression, for f-string debug + expressions (`f"{expr=}"`) and t-string Interpolation.str + (see set_ftstring_expr()). + - On the FSTRING_END/TSTRING_END token of a d-string: the longest + common leading whitespace of all physical lines of the literal, + used to dedent the string parts (PEP 822). + The owner is this struct (freed by _PyToken_Free()); pegen's + initialize_token() moves it into the arena-managed Token. */ PyObject *metadata; }; @@ -67,6 +78,15 @@ typedef struct _tokenizer_mode { int in_debug; int in_format_spec; + /* PEP 822 d-string support. While tokenizing a d-string, the longest + common leading whitespace of all physical lines seen so far is + maintained in [dedent_indent, dedent_indent + dedent_indent_len). + It is attached to the FSTRING_END/TSTRING_END token as metadata. */ + int dedent; + int dedent_seen_line; + char *dedent_indent; + Py_ssize_t dedent_indent_len; + enum string_kind_t string_kind; } tokenizer_mode; diff --git a/Parser/string_parser.c b/Parser/string_parser.c index b164dfbc81a9339..651b4d4161117cf 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -247,6 +247,57 @@ _PyPegen_decode_string(Parser *p, int raw, const char *s, size_t len, Token *t) return decode_unicode_with_escapes(p, s, len, t); } +// Dedent d-string and return result as a bytes. +static PyObject* +_PyPegen_dedent_string(Parser *p, const char *s, Py_ssize_t len, + const char *indent, Py_ssize_t indent_len) +{ + PyBytesWriter *w = PyBytesWriter_Create(0); + if (w == NULL) { + return NULL; + } + + const char *end = s + len; + while (s < end) { + // A blank line (whitespace-only line with a newline) is normalized + // to a single newline. The last line (before the closing quotes) + // has no newline and is never considered blank. + const char *q = s; + while (q < end && (*q == ' ' || *q == '\t')) { + q++; + } + if (q < end && *q == '\n') { + if (PyBytesWriter_WriteBytes(w, "\n", 1) < 0) { + PyBytesWriter_Discard(w); + return NULL; + } + s = q + 1; + continue; + } + + // A non-blank line. The common indent was computed from all lines + // including the closing quotes line, so it is always a prefix of + // the leading whitespace of this line. + assert(q - s >= indent_len); + assert(memcmp(s, indent, (size_t)indent_len) == 0); + s += indent_len; + const char *line_end = memchr(s, '\n', end - s); + if (line_end == NULL) { + line_end = end; // last line without newline + } + else { + line_end++; // include the newline in the line + } + + if (PyBytesWriter_WriteBytes(w, s, line_end - s) < 0) { + PyBytesWriter_Discard(w); + return NULL; + } + s = line_end; + } + return PyBytesWriter_Finish(w); +} + /* s must include the bracketing quote characters, and r, b &/or f prefixes (if any), and embedded escape sequences (if any). (f-strings are handled by the parser) _PyPegen_parse_string parses it, and returns the decoded Python string object. */ @@ -262,9 +313,10 @@ _PyPegen_parse_string(Parser *p, Token *t) int quote = Py_CHARMASK(*s); int bytesmode = 0; int rawmode = 0; + int dedentmode = 0; if (Py_ISALPHA(quote)) { - while (!bytesmode || !rawmode) { + while (!bytesmode || !rawmode || !dedentmode) { if (quote == 'b' || quote == 'B') { quote =(unsigned char)*++s; bytesmode = 1; @@ -276,6 +328,10 @@ _PyPegen_parse_string(Parser *p, Token *t) quote = (unsigned char)*++s; rawmode = 1; } + else if (quote == 'd' || quote == 'D') { + quote =(unsigned char)*++s; + dedentmode = 1; + } else { break; } @@ -315,9 +371,63 @@ _PyPegen_parse_string(Parser *p, Token *t) return NULL; } } + else if (dedentmode) { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(t, "d-string must be triple-quoted"); + return NULL; + } /* Avoid invoking escape decoding routines if possible. */ rawmode = rawmode || strchr(s, '\\') == NULL; + + int _prev_call_invald = p->call_invalid_rules; + + PyObject *dedent_bytes = NULL; + if (dedentmode) { + if (len == 0 || s[0] != '\n') { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(t, "d-string must start with a newline"); + return NULL; + } + + // _PyPegen_decode_string() and decode_bytes_with_escapes() emit + // a warning for invalid escape sequences. + // We need to call it before dedenting since it shifts the positions. + if (!_prev_call_invald && !rawmode) { + PyObject *temp; + if (bytesmode) { + temp = decode_bytes_with_escapes(p, s, len, t); + } + else { + temp = _PyPegen_decode_string(p, 0, s, len, t); + } + if (temp == NULL) { + return NULL; + } + Py_DECREF(temp); + } + + // We find common indent from [s, end+1) because we want to include the last line + // for indent calculation. + const char *end = s + len; + assert(*end == '"' || *end == '\''); // end[0:3] is the trailing quotes + const char *indent = ""; + Py_ssize_t indent_len = _Py_search_longest_common_leading_whitespace(s+1, end+1, &indent); + + s++; len--; // skip the first newline + // Dedent even when indent_len == 0: blank lines must still be + // normalized to single newlines. + dedent_bytes = _PyPegen_dedent_string(p, s, len, indent, indent_len); + if (dedent_bytes == NULL) { + return NULL; + } + if (PyBytes_AsStringAndSize(dedent_bytes, (char**)&s, (Py_ssize_t*)&len) < 0) { + Py_DECREF(dedent_bytes); + return NULL; + } + + p->call_invalid_rules = 1; + } + + PyObject *result; if (bytesmode) { /* Disallow non-ASCII characters. */ const char *ch; @@ -327,13 +437,23 @@ _PyPegen_parse_string(Parser *p, Token *t) t, "bytes can only contain ASCII " "literal characters"); + Py_XDECREF(dedent_bytes); + p->call_invalid_rules = _prev_call_invald; return NULL; } } if (rawmode) { - return PyBytes_FromStringAndSize(s, (Py_ssize_t)len); + result = PyBytes_FromStringAndSize(s, (Py_ssize_t)len); } - return decode_bytes_with_escapes(p, s, (Py_ssize_t)len, t); + else { + result = decode_bytes_with_escapes(p, s, (Py_ssize_t)len, t); + } + } + else { + result = _PyPegen_decode_string(p, rawmode, s, len, t); } - return _PyPegen_decode_string(p, rawmode, s, len, t); + Py_XDECREF(dedent_bytes); + p->call_invalid_rules = _prev_call_invald; + return result; } +