From 0c4185ff3b6ec64d93b9ec526ec738f438e37996 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 11 Oct 2025 14:40:57 +0100 Subject: [PATCH 001/192] [WIP] Add initial nativeparse test case --- mypy/nativeparse.py | 30 ++++++++++++++++++++++++++++++ mypy/nodes.py | 5 +++++ mypy/test/test_nativeparse.py | 19 +++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 mypy/nativeparse.py create mode 100644 mypy/test/test_nativeparse.py diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py new file mode 100644 index 0000000000000..1bcee9516721b --- /dev/null +++ b/mypy/nativeparse.py @@ -0,0 +1,30 @@ +"""Python parser that directly constructs a native AST (when compiled). + +Use a Rust extension to generate a serialized AST, and deserialize the AST directly +to a mypy AST. + +NOTE: This is heavily work in progress. + +Key planned features compared the mypy.fastparse: + * No intermediate non-mypyc AST created, to improve performance + * Support all Python syntax even if running mypy on older Python versions + * Parsing doesn't need GIL => multithreading to produce serialized AST in parallel + * Produce import dependencies without having to build an AST (helps parallel type checking) +""" + +from __future__ import annotations + +import os +import subprocess + +from mypy.nodes import MypyFile + + +def native_parse(filename: str) -> MypyFile: + assert False + + +def parse_to_binary_ast(filename: str) -> bytes: + binpath = os.path.expanduser("~/src/ruff/target/release/mypy_parser") + result = subprocess.run([binpath, "serialize-ast", filename], capture_output=True, check=True) + return result.stdout diff --git a/mypy/nodes.py b/mypy/nodes.py index fe22d5d306353..123b64952cb29 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5017,6 +5017,11 @@ def local_definitions( CLASS_DEF: Final[Tag] = 60 SYMBOL_TABLE_NODE: Final[Tag] = 61 +EXPR_STMT: Final[Tag] = 11 +CALL_EXPR: Final[Tag] = 12 +NAME_EXPR: Final[Tag] = 13 +STR_EXPR: Final[Tag] = 14 + def read_symbol(data: ReadBuffer) -> SymbolNode: tag = read_tag(data) diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py new file mode 100644 index 0000000000000..22ee4cb02547e --- /dev/null +++ b/mypy/test/test_nativeparse.py @@ -0,0 +1,19 @@ +import os +import tempfile +import unittest + +from mypy import nodes +from mypy.nativeparse import parse_to_binary_ast + + +class TestNativeParse(unittest.TestCase): + def test_hello_world_binary(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = os.path.join(temp_dir, "t.py") + with open(temp_path, "w") as f: + f.write("print('hello')") + + b = parse_to_binary_ast(temp_path) + assert list(b) == [nodes.EXPR_STMT, nodes.CALL_EXPR, nodes.NAME_EXPR, 5] + list( + b"print" + ) + [1, nodes.STR_EXPR, 5] + list(b"hello") From 8c0f90648228eba7a234d33c73db8bfeab9201f2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 11 Oct 2025 16:14:42 +0100 Subject: [PATCH 002/192] [WIP] First steps towards parsing something --- mypy/nativeparse.py | 43 +++++++++++++++++++++++++++++++++-- mypy/test/test_nativeparse.py | 33 +++++++++++++++++++-------- 2 files changed, 64 insertions(+), 12 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 1bcee9516721b..21d001ddab9e5 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -17,14 +17,53 @@ import os import subprocess -from mypy.nodes import MypyFile +from mypy import nodes +from mypy.cache import Buffer, read_int, read_str, read_tag +from mypy.nodes import ( + ARG_POS, + CallExpr, + Expression, + ExpressionStmt, + MypyFile, + NameExpr, + Statement, + StrExpr, +) def native_parse(filename: str) -> MypyFile: - assert False + b = parse_to_binary_ast(filename) + print(repr(b)) + data = Buffer(b) + stmt = read_statement(data) + return MypyFile([stmt], []) def parse_to_binary_ast(filename: str) -> bytes: binpath = os.path.expanduser("~/src/ruff/target/release/mypy_parser") result = subprocess.run([binpath, "serialize-ast", filename], capture_output=True, check=True) return result.stdout + + +def read_statement(data: Buffer) -> Statement: + tag = read_tag(data) + if tag == nodes.EXPR_STMT: + return ExpressionStmt(read_expression(data)) + else: + assert False + + +def read_expression(data: Buffer) -> Expression: + tag = read_tag(data) + if tag == nodes.CALL_EXPR: + callee = read_expression(data) + n = read_int(data) + args = [read_expression(data) for i in range(n)] + return CallExpr(callee, args, [ARG_POS] * n, [None] * n) + elif tag == nodes.NAME_EXPR: + n = read_str(data) + return NameExpr(n) + elif tag == nodes.STR_EXPR: + return StrExpr(read_str(data)) + else: + assert False diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index 22ee4cb02547e..0c714c5ef9353 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -1,19 +1,32 @@ +import contextlib import os import tempfile import unittest +from collections.abc import Iterator from mypy import nodes -from mypy.nativeparse import parse_to_binary_ast +from mypy.nativeparse import native_parse, parse_to_binary_ast +from mypy.nodes import MypyFile class TestNativeParse(unittest.TestCase): - def test_hello_world_binary(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - temp_path = os.path.join(temp_dir, "t.py") - with open(temp_path, "w") as f: - f.write("print('hello')") - - b = parse_to_binary_ast(temp_path) - assert list(b) == [nodes.EXPR_STMT, nodes.CALL_EXPR, nodes.NAME_EXPR, 5] + list( + def test_trivial_binary_data(self) -> None: + with temp_source("print('hello')") as fnam: + b = parse_to_binary_ast(fnam) + assert list(b) == [nodes.EXPR_STMT, nodes.CALL_EXPR, nodes.NAME_EXPR, 10] + list( b"print" - ) + [1, nodes.STR_EXPR, 5] + list(b"hello") + ) + [22, nodes.STR_EXPR, 10] + list(b"hello") + + def test_deserialize_hello(self) -> None: + with temp_source("print('hello')") as fnam: + node = native_parse(fnam) + assert isinstance(node, MypyFile) + + +@contextlib.contextmanager +def temp_source(text: str) -> Iterator[str]: + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = os.path.join(temp_dir, "t.py") + with open(temp_path, "w") as f: + f.write(text) + yield temp_path From d5291bec9f929d9a219facb213cf5609d579eccd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 11 Oct 2025 16:29:03 +0100 Subject: [PATCH 003/192] Support multiple defs --- mypy/nativeparse.py | 7 +++++-- mypy/test/test_nativeparse.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 21d001ddab9e5..a10a3887d0df8 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -35,8 +35,11 @@ def native_parse(filename: str) -> MypyFile: b = parse_to_binary_ast(filename) print(repr(b)) data = Buffer(b) - stmt = read_statement(data) - return MypyFile([stmt], []) + n = read_int(data) + defs = [] + for i in range(n): + defs.append(read_statement(data)) + return MypyFile(defs, []) def parse_to_binary_ast(filename: str) -> bytes: diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index 0c714c5ef9353..4f4133fca17b3 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -13,7 +13,7 @@ class TestNativeParse(unittest.TestCase): def test_trivial_binary_data(self) -> None: with temp_source("print('hello')") as fnam: b = parse_to_binary_ast(fnam) - assert list(b) == [nodes.EXPR_STMT, nodes.CALL_EXPR, nodes.NAME_EXPR, 10] + list( + assert list(b) == [22, nodes.EXPR_STMT, nodes.CALL_EXPR, nodes.NAME_EXPR, 10] + list( b"print" ) + [22, nodes.STR_EXPR, 10] + list(b"hello") From 4c722411f1ed25a3940775e718a956616b7660bf Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 12 Oct 2025 11:26:08 +0100 Subject: [PATCH 004/192] Read line/column information --- mypy/nativeparse.py | 28 ++++++++++++++++++++++++---- mypy/test/test_nativeparse.py | 26 +++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index a10a3887d0df8..fc53f7991046a 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -26,6 +26,7 @@ ExpressionStmt, MypyFile, NameExpr, + Node, Statement, StrExpr, ) @@ -51,7 +52,12 @@ def parse_to_binary_ast(filename: str) -> bytes: def read_statement(data: Buffer) -> Statement: tag = read_tag(data) if tag == nodes.EXPR_STMT: - return ExpressionStmt(read_expression(data)) + es = ExpressionStmt(read_expression(data)) + es.line = es.expr.line + es.column = es.expr.column + es.end_line = es.expr.end_line + es.end_column = es.expr.end_column + return es else: assert False @@ -62,11 +68,25 @@ def read_expression(data: Buffer) -> Expression: callee = read_expression(data) n = read_int(data) args = [read_expression(data) for i in range(n)] - return CallExpr(callee, args, [ARG_POS] * n, [None] * n) + ce = CallExpr(callee, args, [ARG_POS] * n, [None] * n) + read_loc(data, ce) + return ce elif tag == nodes.NAME_EXPR: n = read_str(data) - return NameExpr(n) + ne = NameExpr(n) + read_loc(data, ne) + return ne elif tag == nodes.STR_EXPR: - return StrExpr(read_str(data)) + se = StrExpr(read_str(data)) + read_loc(data, se) + return se else: assert False + + +def read_loc(data: Buffer, node: Node) -> None: + line = read_int(data) + node.line = line + node.column = read_int(data) + node.end_line = line + read_int(data) + node.end_column = read_int(data) diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index 4f4133fca17b3..d483745b3d436 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -11,11 +11,31 @@ class TestNativeParse(unittest.TestCase): def test_trivial_binary_data(self) -> None: + def int_enc(n: int) -> int: + return (n + 10) << 1 + + def locs(start_line: int, start_column: int, end_line: int, end_column) -> list[int]: + return [ + int_enc(start_line), + int_enc(start_column), + int_enc(end_line - start_line), + int_enc(end_column), + ] + with temp_source("print('hello')") as fnam: b = parse_to_binary_ast(fnam) - assert list(b) == [22, nodes.EXPR_STMT, nodes.CALL_EXPR, nodes.NAME_EXPR, 10] + list( - b"print" - ) + [22, nodes.STR_EXPR, 10] + list(b"hello") + assert list(b) == ( + [22, nodes.EXPR_STMT, nodes.CALL_EXPR] + + [nodes.NAME_EXPR] + + [10] + + list(b"print") + + locs(1, 1, 1, 6) + + [22, nodes.STR_EXPR] + + [10] + + list(b"hello") + + locs(1, 7, 1, 14) + + locs(1, 1, 1, 15) + ) def test_deserialize_hello(self) -> None: with temp_source("print('hello')") as fnam: From 3798d06d3c461ef598a02d60e396a3625570cf5f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 12 Oct 2025 11:52:02 +0100 Subject: [PATCH 005/192] Remove debug print --- mypy/nativeparse.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index fc53f7991046a..88979b30d2b2f 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -34,7 +34,6 @@ def native_parse(filename: str) -> MypyFile: b = parse_to_binary_ast(filename) - print(repr(b)) data = Buffer(b) n = read_int(data) defs = [] From 32d855e9e6df88d6fb3bcaeaa5bf99edc1b70e43 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 12 Oct 2025 12:56:41 +0100 Subject: [PATCH 006/192] Fix self check, update docstring --- mypy/nativeparse.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 88979b30d2b2f..ae23f14065188 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -5,11 +5,14 @@ NOTE: This is heavily work in progress. -Key planned features compared the mypy.fastparse: +Expected benefits over mypy.fastparse: * No intermediate non-mypyc AST created, to improve performance - * Support all Python syntax even if running mypy on older Python versions - * Parsing doesn't need GIL => multithreading to produce serialized AST in parallel - * Produce import dependencies without having to build an AST (helps parallel type checking) + * Parsing doesn't need GIL => use multithreading to construct serialized ASTs in parallel + * Produce import dependencies without having to build an AST => helps parallel type checking + * Support all Python syntax even if running mypy on an older Python version + * Generate an AST even if there are syntax errors + * Potential to support incremental parsing (quickly process modified sections in a file) + * Stripping function bodies in third-party code can happen earlier, for extra performance """ from __future__ import annotations @@ -71,8 +74,8 @@ def read_expression(data: Buffer) -> Expression: read_loc(data, ce) return ce elif tag == nodes.NAME_EXPR: - n = read_str(data) - ne = NameExpr(n) + s = read_str(data) + ne = NameExpr(s) read_loc(data, ne) return ne elif tag == nodes.STR_EXPR: From cf9aefd2393c2627e7aecf297dedba6e7dc73cb5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 22 Nov 2025 14:26:51 +0000 Subject: [PATCH 007/192] WIP add parse/deserialize benchmark --- mypy/test/test_nativeparse.py | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index d483745b3d436..20ccf9fe153fb 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -8,6 +8,10 @@ from mypy.nativeparse import native_parse, parse_to_binary_ast from mypy.nodes import MypyFile +import gc + +gc.set_threshold(200 * 1000, 30, 30) + class TestNativeParse(unittest.TestCase): def test_trivial_binary_data(self) -> None: @@ -42,6 +46,40 @@ def test_deserialize_hello(self) -> None: node = native_parse(fnam) assert isinstance(node, MypyFile) + def test_deserialize_bench(self) -> None: + with temp_source("print('hello')\n" * 4000) as fnam: + import time + for i in range(10): + native_parse(fnam) + t0 = time.time() + for i in range(25): + node = native_parse(fnam) + assert isinstance(node, MypyFile) + print(len(node.defs)) + print((time.time() - t0) * 1000) + assert False, 1 / ((time.time() - t0) / 100000) + + def test_parse_bench(self) -> None: + with temp_source("print('hello')\n" * 4000) as fnam: + import time + from mypy.parse import parse + from mypy.errors import Errors + from mypy.options import Options + o = Options() + + for i in range(10): + with open(fnam, "rb") as f: + data = f.read() + node = parse(data, fnam, "__main__", Errors(o), o) + + t0 = time.time() + for i in range(25): + with open(fnam, "rb") as f: + data = f.read() + node = parse(data, fnam, "__main__", Errors(o), o) + assert False, 1 / ((time.time() - t0) / 100000) + assert isinstance(node, MypyFile) + @contextlib.contextmanager def temp_source(text: str) -> Iterator[str]: From 9da3988a84f91a5aaa03f5547ce9092af14b02f6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 23 Nov 2025 11:04:52 +0000 Subject: [PATCH 008/192] Update for new cache format --- mypy/cache.py | 2 ++ mypy/nativeparse.py | 54 +++++++++++++++++++++++++++-------- mypy/nodes.py | 8 +++--- mypy/test/test_nativeparse.py | 24 ++++++++++------ 4 files changed, 63 insertions(+), 25 deletions(-) diff --git a/mypy/cache.py b/mypy/cache.py index 832cfb4732ea9..579664dca6276 100644 --- a/mypy/cache.py +++ b/mypy/cache.py @@ -239,6 +239,8 @@ def read(cls, data: ReadBuffer, data_file: str) -> CacheMeta | None: # Misc classes. EXTRA_ATTRS: Final[Tag] = 150 DT_SPEC: Final[Tag] = 151 +# Four integers representing source file (line, column) range. +LOCATION: Final[Tag] = 152 END_TAG: Final[Tag] = 255 diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index ae23f14065188..f9d3e24cd42fa 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -21,7 +21,17 @@ import subprocess from mypy import nodes -from mypy.cache import Buffer, read_int, read_str, read_tag +from mypy.cache import ( + END_TAG, + LIST_GEN, + LOCATION, + ReadBuffer, + Tag, + read_int, + read_int_bare, + read_str, + read_tag, +) from mypy.nodes import ( ARG_POS, CallExpr, @@ -35,9 +45,17 @@ ) +def expect_end_tag(data: ReadBuffer) -> None: + assert read_tag(data) == END_TAG + + +def expect_tag(data: ReadBuffer, tag: Tag) -> None: + assert read_tag(data) == tag + + def native_parse(filename: str) -> MypyFile: b = parse_to_binary_ast(filename) - data = Buffer(b) + data = ReadBuffer(b) n = read_int(data) defs = [] for i in range(n): @@ -51,7 +69,7 @@ def parse_to_binary_ast(filename: str) -> bytes: return result.stdout -def read_statement(data: Buffer) -> Statement: +def read_statement(data: ReadBuffer) -> Statement: tag = read_tag(data) if tag == nodes.EXPR_STMT: es = ExpressionStmt(read_expression(data)) @@ -59,36 +77,48 @@ def read_statement(data: Buffer) -> Statement: es.column = es.expr.column es.end_line = es.expr.end_line es.end_column = es.expr.end_column + expect_end_tag(data) return es else: - assert False + assert False, tag -def read_expression(data: Buffer) -> Expression: +def read_expression(data: ReadBuffer) -> Expression: tag = read_tag(data) if tag == nodes.CALL_EXPR: callee = read_expression(data) - n = read_int(data) - args = [read_expression(data) for i in range(n)] + args = read_expression_list(data) + n = len(args) ce = CallExpr(callee, args, [ARG_POS] * n, [None] * n) read_loc(data, ce) + expect_end_tag(data) return ce elif tag == nodes.NAME_EXPR: s = read_str(data) ne = NameExpr(s) read_loc(data, ne) + expect_end_tag(data) return ne elif tag == nodes.STR_EXPR: se = StrExpr(read_str(data)) read_loc(data, se) + expect_end_tag(data) return se else: assert False -def read_loc(data: Buffer, node: Node) -> None: - line = read_int(data) +def read_expression_list(data: ReadBuffer) -> list[Expression]: + expect_tag(data, LIST_GEN) + n = read_int_bare(data) + return [read_expression(data) for i in range(n)] + + +def read_loc(data: ReadBuffer, node: Node) -> None: + expect_tag(data, LOCATION) + line = read_int_bare(data) node.line = line - node.column = read_int(data) - node.end_line = line + read_int(data) - node.end_column = read_int(data) + column = read_int_bare(data) + node.column = column + node.end_line = line + read_int_bare(data) + node.end_column = column + read_int_bare(data) diff --git a/mypy/nodes.py b/mypy/nodes.py index 123b64952cb29..242abd98d5dae 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5017,10 +5017,10 @@ def local_definitions( CLASS_DEF: Final[Tag] = 60 SYMBOL_TABLE_NODE: Final[Tag] = 61 -EXPR_STMT: Final[Tag] = 11 -CALL_EXPR: Final[Tag] = 12 -NAME_EXPR: Final[Tag] = 13 -STR_EXPR: Final[Tag] = 14 +EXPR_STMT: Final[Tag] = 160 +CALL_EXPR: Final[Tag] = 161 +NAME_EXPR: Final[Tag] = 162 +STR_EXPR: Final[Tag] = 163 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index 20ccf9fe153fb..36f69d7b058e8 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -1,15 +1,15 @@ import contextlib +import gc import os import tempfile import unittest from collections.abc import Iterator from mypy import nodes +from mypy.cache import END_TAG, LIST_GEN, LITERAL_INT, LITERAL_STR, LOCATION from mypy.nativeparse import native_parse, parse_to_binary_ast from mypy.nodes import MypyFile -import gc - gc.set_threshold(200 * 1000, 30, 30) @@ -20,25 +20,28 @@ def int_enc(n: int) -> int: def locs(start_line: int, start_column: int, end_line: int, end_column) -> list[int]: return [ + LOCATION, int_enc(start_line), int_enc(start_column), int_enc(end_line - start_line), - int_enc(end_column), + int_enc(end_column - start_column), ] with temp_source("print('hello')") as fnam: b = parse_to_binary_ast(fnam) assert list(b) == ( - [22, nodes.EXPR_STMT, nodes.CALL_EXPR] - + [nodes.NAME_EXPR] - + [10] + [LITERAL_INT, 22, nodes.EXPR_STMT, nodes.CALL_EXPR] + + [nodes.NAME_EXPR, LITERAL_STR] + + [int_enc(5)] + list(b"print") + locs(1, 1, 1, 6) - + [22, nodes.STR_EXPR] - + [10] + + [END_TAG, LIST_GEN, 22, nodes.STR_EXPR] + + [LITERAL_STR, int_enc(5)] + list(b"hello") + locs(1, 7, 1, 14) + + [END_TAG] + locs(1, 1, 1, 15) + + [END_TAG, END_TAG] ) def test_deserialize_hello(self) -> None: @@ -49,6 +52,7 @@ def test_deserialize_hello(self) -> None: def test_deserialize_bench(self) -> None: with temp_source("print('hello')\n" * 4000) as fnam: import time + for i in range(10): native_parse(fnam) t0 = time.time() @@ -62,9 +66,11 @@ def test_deserialize_bench(self) -> None: def test_parse_bench(self) -> None: with temp_source("print('hello')\n" * 4000) as fnam: import time - from mypy.parse import parse + from mypy.errors import Errors from mypy.options import Options + from mypy.parse import parse + o = Options() for i in range(10): From c374e0a4d3bd6273be2c203a5f36e815bafcb50e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 23 Nov 2025 14:05:11 +0000 Subject: [PATCH 009/192] Deserialize member expr --- mypy/nativeparse.py | 8 ++++++++ mypy/nodes.py | 2 ++ mypy/test/test_nativeparse.py | 9 ++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index f9d3e24cd42fa..2029616a0b451 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -37,6 +37,7 @@ CallExpr, Expression, ExpressionStmt, + MemberExpr, MypyFile, NameExpr, Node, @@ -99,6 +100,13 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, ne) expect_end_tag(data) return ne + elif tag == nodes.MEMBER_EXPR: + e = read_expression(data) + attr = read_str(data) + m = MemberExpr(e, attr) + read_loc(data, m) + expect_end_tag(data) + return m elif tag == nodes.STR_EXPR: se = StrExpr(read_str(data)) read_loc(data, se) diff --git a/mypy/nodes.py b/mypy/nodes.py index 242abd98d5dae..d7266fcd28c51 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5021,6 +5021,8 @@ def local_definitions( CALL_EXPR: Final[Tag] = 161 NAME_EXPR: Final[Tag] = 162 STR_EXPR: Final[Tag] = 163 +IMPORT: Final[Tag] = 164 +MEMBER_EXPR: Final[Tag] = 165 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index 36f69d7b058e8..ebfc958b1686a 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -8,7 +8,7 @@ from mypy import nodes from mypy.cache import END_TAG, LIST_GEN, LITERAL_INT, LITERAL_STR, LOCATION from mypy.nativeparse import native_parse, parse_to_binary_ast -from mypy.nodes import MypyFile +from mypy.nodes import ExpressionStmt, MemberExpr, MypyFile gc.set_threshold(200 * 1000, 30, 30) @@ -49,6 +49,13 @@ def test_deserialize_hello(self) -> None: node = native_parse(fnam) assert isinstance(node, MypyFile) + def test_deserialize_member_expr(self) -> None: + with temp_source("foo_bar.xyz2") as fnam: + node = native_parse(fnam) + assert isinstance(node, MypyFile) + assert isinstance(node.defs[0], ExpressionStmt) + assert isinstance(node.defs[0].expr, MemberExpr) + def test_deserialize_bench(self) -> None: with temp_source("print('hello')\n" * 4000) as fnam: import time From 46fe324ac0c04c319a1ecfd97bb90a0ded2b7cb3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 23 Nov 2025 14:16:02 +0000 Subject: [PATCH 010/192] Add data-driven tests --- mypy/test/test_nativeparse.py | 58 ++++++++++++++++++++++++++++++- test-data/unit/native-parser.test | 20 +++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 test-data/unit/native-parser.test diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index ebfc958b1686a..a7930f6c9893a 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -1,3 +1,7 @@ +"""Tests for the native mypy parser.""" + +from __future__ import annotations + import contextlib import gc import os @@ -5,14 +9,66 @@ import unittest from collections.abc import Iterator -from mypy import nodes +from mypy import defaults, nodes from mypy.cache import END_TAG, LIST_GEN, LITERAL_INT, LITERAL_STR, LOCATION +from mypy.config_parser import parse_mypy_comments +from mypy.errors import CompileError from mypy.nativeparse import native_parse, parse_to_binary_ast from mypy.nodes import ExpressionStmt, MemberExpr, MypyFile +from mypy.options import Options +from mypy.test.data import DataDrivenTestCase, DataSuite +from mypy.test.helpers import assert_string_arrays_equal +from mypy.util import get_mypy_comments gc.set_threshold(200 * 1000, 30, 30) +class NativeParserSuite(DataSuite): + required_out_section = True + base_path = "." + files = ["native-parser.test"] + + def run_case(self, testcase: DataDrivenTestCase) -> None: + test_parser(testcase) + + +def test_parser(testcase: DataDrivenTestCase) -> None: + """Perform a single native parser test case. + + The argument contains the description of the test case. + """ + options = Options() + options.hide_error_codes = True + + if testcase.file.endswith("python310.test"): + options.python_version = (3, 10) + elif testcase.file.endswith("python312.test"): + options.python_version = (3, 12) + elif testcase.file.endswith("python313.test"): + options.python_version = (3, 13) + elif testcase.file.endswith("python314.test"): + options.python_version = (3, 14) + else: + options.python_version = defaults.PYTHON3_VERSION + + source = "\n".join(testcase.input) + + # Apply mypy: comments to options. + comments = get_mypy_comments(source) + changes, _ = parse_mypy_comments(comments, options) + options = options.apply_changes(changes) + + try: + with temp_source(source) as fnam: + node = native_parse(fnam) + a = node.str_with_options(options).split("\n") + except CompileError as e: + a = e.messages + assert_string_arrays_equal( + testcase.output, a, f"Invalid parser output ({testcase.file}, line {testcase.line})" + ) + + class TestNativeParse(unittest.TestCase): def test_trivial_binary_data(self) -> None: def int_enc(n: int) -> int: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test new file mode 100644 index 0000000000000..a32a683d03c32 --- /dev/null +++ b/test-data/unit/native-parser.test @@ -0,0 +1,20 @@ +[case testHello] +print("hello") +[out] +MypyFile:1( + + ExpressionStmt:1( + CallExpr:1( + NameExpr(print) + Args( + StrExpr(hello))))) + +[case testMemberExpr] +foo_bar.xyz2 +[out] +MypyFile:1( + + ExpressionStmt:1( + MemberExpr:1( + NameExpr(foo_bar) + xyz2))) From e8ce8f857caa5d6f64657dbf93ec8457840ae1a6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 23 Nov 2025 14:42:17 +0000 Subject: [PATCH 011/192] Fix empty line in test output --- mypy/nativeparse.py | 4 +++- mypy/test/test_nativeparse.py | 1 + test-data/unit/native-parser.test | 2 -- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 2029616a0b451..fe9ef7b485bbe 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -61,7 +61,9 @@ def native_parse(filename: str) -> MypyFile: defs = [] for i in range(n): defs.append(read_statement(data)) - return MypyFile(defs, []) + node = MypyFile(defs, []) + node.path = filename + return node def parse_to_binary_ast(filename: str) -> bytes: diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index a7930f6c9893a..d94880fa952ce 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -61,6 +61,7 @@ def test_parser(testcase: DataDrivenTestCase) -> None: try: with temp_source(source) as fnam: node = native_parse(fnam) + node.path = "main" a = node.str_with_options(options).split("\n") except CompileError as e: a = e.messages diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index a32a683d03c32..12ca46641859e 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -2,7 +2,6 @@ print("hello") [out] MypyFile:1( - ExpressionStmt:1( CallExpr:1( NameExpr(print) @@ -13,7 +12,6 @@ MypyFile:1( foo_bar.xyz2 [out] MypyFile:1( - ExpressionStmt:1( MemberExpr:1( NameExpr(foo_bar) From bab774bae01d4fb303a3a42ca33c71d5fdaa74e8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 29 Dec 2025 16:51:54 +0000 Subject: [PATCH 012/192] Deserialize tuple expressions --- mypy/nativeparse.py | 7 +++++++ mypy/nodes.py | 6 ++++++ test-data/unit/native-parser.test | 9 +++++++++ 3 files changed, 22 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index fe9ef7b485bbe..8ae99a4641efc 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -43,6 +43,7 @@ Node, Statement, StrExpr, + TupleExpr, ) @@ -114,6 +115,12 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, se) expect_end_tag(data) return se + elif tag == nodes.TUPLE_EXPR: + items = read_expression_list(data) + t = TupleExpr(items) + read_loc(data, t) + expect_end_tag(data) + return t else: assert False diff --git a/mypy/nodes.py b/mypy/nodes.py index d7266fcd28c51..cdb615e061518 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5023,6 +5023,12 @@ def local_definitions( STR_EXPR: Final[Tag] = 163 IMPORT: Final[Tag] = 164 MEMBER_EXPR: Final[Tag] = 165 +OP_EXPR: Final[Tag] = 166 +INT_EXPR: Final[Tag] = 167 +IF_STMT: Final[Tag] = 168 +ASSIGNMENT_STMT: Final[Tag] = 169 +TUPLE_EXPR: Final[Tag] = 170 +BLOCK: Final[Tag] = 171 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 12ca46641859e..6c906437372c4 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -16,3 +16,12 @@ MypyFile:1( MemberExpr:1( NameExpr(foo_bar) xyz2))) + +[case testTupleExpr] +("a", "b") +[out] +MypyFile:1( + ExpressionStmt:1( + TupleExpr:1( + StrExpr(a) + StrExpr(b)))) From ffd4b8df9ca20a88c6513c7fa98aaeda5c4e710a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 29 Dec 2025 17:16:30 +0000 Subject: [PATCH 013/192] Deserialize binary operations --- mypy/nativeparse.py | 20 +++++++++++++++++++- test-data/unit/native-parser.test | 13 +++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 8ae99a4641efc..335eb8b948bd5 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -19,6 +19,7 @@ import os import subprocess +from typing import Final from mypy import nodes from mypy.cache import ( @@ -41,6 +42,7 @@ MypyFile, NameExpr, Node, + OpExpr, Statement, StrExpr, TupleExpr, @@ -87,6 +89,10 @@ def read_statement(data: ReadBuffer) -> Statement: assert False, tag +bin_ops: Final = ["+", "-", "*", "@", "/", "%", "**", "<<", ">>", "|", "^", "&", "//"] + + + def read_expression(data: ReadBuffer) -> Expression: tag = read_tag(data) if tag == nodes.CALL_EXPR: @@ -121,8 +127,20 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, t) expect_end_tag(data) return t + elif tag == nodes.OP_EXPR: + op = bin_ops[read_int(data)] + left = read_expression(data) + right = read_expression(data) + o = OpExpr(op, left, right) + # TODO: Store these explicitly? + o.line = left.line + o.column = left.column + o.end_line = right.end_line + o.end_column = right.end_column + expect_end_tag(data) + return o else: - assert False + assert False, tag def read_expression_list(data: ReadBuffer) -> list[Expression]: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 6c906437372c4..7864d9eb883ac 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -25,3 +25,16 @@ MypyFile:1( TupleExpr:1( StrExpr(a) StrExpr(b)))) + +[case testOpExpr] +("x" + "y") * "z" +[out] +MypyFile:1( + ExpressionStmt:1( + OpExpr:1( + * + OpExpr:1( + + + StrExpr(x) + StrExpr(y)) + StrExpr(z)))) From 310a44551a39a729ea0b78168bc7e4f063a524d5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 29 Dec 2025 17:21:06 +0000 Subject: [PATCH 014/192] Deserialize int expressions --- mypy/nativeparse.py | 6 ++++++ test-data/unit/native-parser.test | 14 ++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 335eb8b948bd5..480e4e682f4c5 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -38,6 +38,7 @@ CallExpr, Expression, ExpressionStmt, + IntExpr, MemberExpr, MypyFile, NameExpr, @@ -121,6 +122,11 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, se) expect_end_tag(data) return se + elif tag == nodes.INT_EXPR: + ie = IntExpr(read_int(data)) + read_loc(data, ie) + expect_end_tag(data) + return ie elif tag == nodes.TUPLE_EXPR: items = read_expression_list(data) t = TupleExpr(items) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 7864d9eb883ac..500206759f8e7 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -38,3 +38,17 @@ MypyFile:1( StrExpr(x) StrExpr(y)) StrExpr(z)))) + +[case testIntExpr] +(0, 1, 2, 203, 5345, 50123, 1234567890) +[out] +MypyFile:1( + ExpressionStmt:1( + TupleExpr:1( + IntExpr(0) + IntExpr(1) + IntExpr(2) + IntExpr(203) + IntExpr(5345) + IntExpr(50123) + IntExpr(1234567890)))) From 2f924469165efde07d5cc8983dede0a322e76eca Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 29 Dec 2025 17:41:26 +0000 Subject: [PATCH 015/192] Deserialize assignment --- mypy/nativeparse.py | 8 ++++++++ test-data/unit/native-parser.test | 16 ++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 480e4e682f4c5..6eebffe61bbab 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -35,6 +35,7 @@ ) from mypy.nodes import ( ARG_POS, + AssignmentStmt, CallExpr, Expression, ExpressionStmt, @@ -86,6 +87,13 @@ def read_statement(data: ReadBuffer) -> Statement: es.end_column = es.expr.end_column expect_end_tag(data) return es + elif tag == nodes.ASSIGNMENT_STMT: + lvalues = read_expression_list(data) + rvalue = read_expression(data) + a = AssignmentStmt(lvalues, rvalue) + read_loc(data, a) + expect_end_tag(data) + return a else: assert False, tag diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 500206759f8e7..7af79bf5e29c1 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -52,3 +52,19 @@ MypyFile:1( IntExpr(5345) IntExpr(50123) IntExpr(1234567890)))) + +[case testAssignment] +x = 1 +a, b = c = 6 +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + IntExpr(1)) + AssignmentStmt:2( + Lvalues( + TupleExpr:2( + NameExpr(a) + NameExpr(b)) + NameExpr(c)) + IntExpr(6))) From 845ae1f7b18a780be8f2d89303ae033d25efd56f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 29 Dec 2025 18:11:10 +0000 Subject: [PATCH 016/192] Deserialize if statement --- mypy/nativeparse.py | 34 ++++++++++++++++++++- test-data/unit/native-parser.test | 51 ++++++++++++++++++++++++++++++- 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 6eebffe61bbab..5c774a5776234 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -32,13 +32,16 @@ read_int_bare, read_str, read_tag, + read_bool, ) from mypy.nodes import ( ARG_POS, AssignmentStmt, + Block, CallExpr, Expression, ExpressionStmt, + IfStmt, IntExpr, MemberExpr, MypyFile, @@ -94,12 +97,41 @@ def read_statement(data: ReadBuffer) -> Statement: read_loc(data, a) expect_end_tag(data) return a + elif tag == nodes.IF_STMT: + expr = [read_expression(data)] + body = [read_block(data)] + num_elif = read_int(data) + for i in range(num_elif): + expr.append(read_expression(data)) + body.append(read_block(data)) + has_else = read_bool(data) + if has_else: + else_body = read_block(data) + else: + else_body = None + if_stmt = IfStmt(expr, body, else_body) + read_loc(data, if_stmt) + expect_end_tag(data) + return if_stmt else: assert False, tag -bin_ops: Final = ["+", "-", "*", "@", "/", "%", "**", "<<", ">>", "|", "^", "&", "//"] +def read_block(data: ReadBuffer) -> Block: + expect_tag(data, nodes.BLOCK) + expect_tag(data, LIST_GEN) + n = read_int_bare(data) + a = [read_statement(data) for i in range(n)] + expect_end_tag(data) + b = Block(a) + b.line = a[0].line + b.column = a[0].column + b.end_line = a[-1].end_line + b.end_column = a[-1].end_column + return b + +bin_ops: Final = ["+", "-", "*", "@", "/", "%", "**", "<<", ">>", "|", "^", "&", "//"] def read_expression(data: ReadBuffer) -> Expression: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 7af79bf5e29c1..873882812b87a 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -53,7 +53,7 @@ MypyFile:1( IntExpr(50123) IntExpr(1234567890)))) -[case testAssignment] +[case testAssignmentStmt] x = 1 a, b = c = 6 [out] @@ -68,3 +68,52 @@ MypyFile:1( NameExpr(b)) NameExpr(c)) IntExpr(6))) + +[case testIfStmt] +if x: + y +if a: + b +else: + c +[out] +MypyFile:1( + IfStmt:1( + If( + NameExpr(x)) + Then( + ExpressionStmt:2( + NameExpr(y)))) + IfStmt:3( + If( + NameExpr(a)) + Then( + ExpressionStmt:4( + NameExpr(b))) + Else( + ExpressionStmt:6( + NameExpr(c))))) + +[case testIfStmtElif] +if a: + b +elif c: + d +else: + e +[out] +MypyFile:1( + IfStmt:1( + If( + NameExpr(a)) + Then( + ExpressionStmt:2( + NameExpr(b))) + If( + NameExpr(c)) + Then( + ExpressionStmt:4( + NameExpr(d))) + Else( + ExpressionStmt:6( + NameExpr(e))))) From 52ed3ac193d05eb6ea0cb77a07a3902821256709 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 30 Dec 2025 10:41:36 +0000 Subject: [PATCH 017/192] Show informatino about panics in tests --- mypy/test/test_nativeparse.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index d94880fa952ce..501d2fc533b43 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -5,6 +5,7 @@ import contextlib import gc import os +import subprocess import tempfile import unittest from collections.abc import Iterator @@ -60,7 +61,11 @@ def test_parser(testcase: DataDrivenTestCase) -> None: try: with temp_source(source) as fnam: - node = native_parse(fnam) + try: + node = native_parse(fnam) + except subprocess.CalledProcessError as e: + print(f"Parse failed\nstdout: {e.stdout.decode()}\nstderr: {e.stderr.decode()}") + assert False node.path = "main" a = node.str_with_options(options).split("\n") except CompileError as e: From 068182970cadf56feecec513cc982e6a19b12790 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 30 Dec 2025 10:46:58 +0000 Subject: [PATCH 018/192] Deserialize additional node types --- mypy/nativeparse.py | 53 +++++++++++++++++++++++++++++++ mypy/nodes.py | 5 +++ test-data/unit/native-parser.test | 51 +++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 5c774a5776234..1c0c30d2f0436 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -42,15 +42,19 @@ Expression, ExpressionStmt, IfStmt, + IndexExpr, IntExpr, + ListExpr, MemberExpr, MypyFile, NameExpr, Node, OpExpr, + SetExpr, Statement, StrExpr, TupleExpr, + WhileStmt, ) @@ -82,6 +86,7 @@ def parse_to_binary_ast(filename: str) -> bytes: def read_statement(data: ReadBuffer) -> Statement: tag = read_tag(data) + stmt: Statement if tag == nodes.EXPR_STMT: es = ExpressionStmt(read_expression(data)) es.line = es.expr.line @@ -113,6 +118,16 @@ def read_statement(data: ReadBuffer) -> Statement: read_loc(data, if_stmt) expect_end_tag(data) return if_stmt + elif tag == nodes.RETURN_STMT: + assert NotImplementedError + elif tag == nodes.WHILE_STMT: + expr = read_expression(data) + body = read_block(data) + else_body = read_optional_block(data) + stmt = WhileStmt(expr, body, else_body) + read_loc(data, stmt) + expect_end_tag(data) + return stmt else: assert False, tag @@ -121,6 +136,7 @@ def read_block(data: ReadBuffer) -> Block: expect_tag(data, nodes.BLOCK) expect_tag(data, LIST_GEN) n = read_int_bare(data) + assert n > 0 a = [read_statement(data) for i in range(n)] expect_end_tag(data) b = Block(a) @@ -131,11 +147,29 @@ def read_block(data: ReadBuffer) -> Block: return b +def read_optional_block(data: ReadBuffer) -> Block | None: + expect_tag(data, nodes.BLOCK) + expect_tag(data, LIST_GEN) + n = read_int_bare(data) + if n == 0: + b = None + else: + a = [read_statement(data) for i in range(n)] + b = Block(a) + b.line = a[0].line + b.column = a[0].column + b.end_line = a[-1].end_line + b.end_column = a[-1].end_column + expect_end_tag(data) + return b + + bin_ops: Final = ["+", "-", "*", "@", "/", "%", "**", "<<", ">>", "|", "^", "&", "//"] def read_expression(data: ReadBuffer) -> Expression: tag = read_tag(data) + expr: Expression if tag == nodes.CALL_EXPR: callee = read_expression(data) args = read_expression_list(data) @@ -167,12 +201,24 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, ie) expect_end_tag(data) return ie + elif tag == nodes.LIST_EXPR: + items = read_expression_list(data) + expr = ListExpr(items) + read_loc(data, expr) + expect_end_tag(data) + return expr elif tag == nodes.TUPLE_EXPR: items = read_expression_list(data) t = TupleExpr(items) read_loc(data, t) expect_end_tag(data) return t + elif tag == nodes.SET_EXPR: + items = read_expression_list(data) + expr = SetExpr(items) + read_loc(data, expr) + expect_end_tag(data) + return expr elif tag == nodes.OP_EXPR: op = bin_ops[read_int(data)] left = read_expression(data) @@ -185,6 +231,13 @@ def read_expression(data: ReadBuffer) -> Expression: o.end_column = right.end_column expect_end_tag(data) return o + elif tag == nodes.INDEX_EXPR: + base = read_expression(data) + index = read_expression(data) + expr = IndexExpr(base, index) + read_loc(data, expr) + expect_end_tag(data) + return expr else: assert False, tag diff --git a/mypy/nodes.py b/mypy/nodes.py index cdb615e061518..1e3b0eebf022d 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5029,6 +5029,11 @@ def local_definitions( ASSIGNMENT_STMT: Final[Tag] = 169 TUPLE_EXPR: Final[Tag] = 170 BLOCK: Final[Tag] = 171 +INDEX_EXPR: Final[Tag] = 172 +LIST_EXPR: Final[Tag] = 173 +SET_EXPR: Final[Tag] = 174 +RETURN_STMT: Final[Tag] = 175 +WHILE_STMT: Final[Tag] = 176 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 873882812b87a..fd736468e83d1 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -117,3 +117,54 @@ MypyFile:1( Else( ExpressionStmt:6( NameExpr(e))))) + +[case testWhileStmt] +while a: + b +while c: + d +else: + e +[out] +MypyFile:1( + WhileStmt:1( + NameExpr(a) + Block:2( + ExpressionStmt:2( + NameExpr(b)))) + WhileStmt:3( + NameExpr(c) + Block:4( + ExpressionStmt:4( + NameExpr(d))) + Else( + ExpressionStmt:6( + NameExpr(e))))) + +[case testIndexExpr] +a[b] +[out] +MypyFile:1( + ExpressionStmt:1( + IndexExpr:1( + NameExpr(a) + NameExpr(b)))) + +[case testListAndSetExpr] +a = [] +b = [1, 2] +c = {3} +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(a) + ListExpr:1()) + AssignmentStmt:2( + NameExpr(b) + ListExpr:2( + IntExpr(1) + IntExpr(2))) + AssignmentStmt:3( + NameExpr(c) + SetExpr:3( + IntExpr(3)))) From 8046eaf672f3412fedf50dfdbea62188f4713d0b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 30 Dec 2025 11:22:22 +0000 Subject: [PATCH 019/192] Deserialize comparison and bool ops --- mypy/nativeparse.py | 33 ++++++++++ mypy/nodes.py | 2 + test-data/unit/native-parser.test | 102 ++++++++++++++++++++++++++++++ 3 files changed, 137 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 1c0c30d2f0436..345b8ae480c84 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -25,6 +25,7 @@ from mypy.cache import ( END_TAG, LIST_GEN, + LIST_INT, LOCATION, ReadBuffer, Tag, @@ -39,6 +40,7 @@ AssignmentStmt, Block, CallExpr, + ComparisonExpr, Expression, ExpressionStmt, IfStmt, @@ -165,6 +167,8 @@ def read_optional_block(data: ReadBuffer) -> Block | None: bin_ops: Final = ["+", "-", "*", "@", "/", "%", "**", "<<", ">>", "|", "^", "&", "//"] +bool_ops: Final = ["and", "or"] +cmp_ops: Final = ["==", "!=", "<", "<=", ">", ">=", "is", "is not", "in", "not in"] def read_expression(data: ReadBuffer) -> Expression: @@ -238,6 +242,35 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, expr) expect_end_tag(data) return expr + elif tag == nodes.BOOL_OP_EXPR: + op = bool_ops[read_int(data)] + values = read_expression_list(data) + # Convert list of values to nested OpExpr nodes + # E.g., [a, b, c] with "and" becomes OpExpr("and", OpExpr("and", a, b), c) + assert len(values) >= 2 + result = values[0] + for val in values[1:]: + result = OpExpr(op, result, val) + result.line = values[0].line + result.column = values[0].column + result.end_line = val.end_line + result.end_column = val.end_column + read_loc(data, result) + expect_end_tag(data) + return result + elif tag == nodes.COMPARISON_EXPR: + left = read_expression(data) + # Read operators list + expect_tag(data, LIST_INT) + n_ops = read_int_bare(data) + ops = [cmp_ops[read_int_bare(data)] for _ in range(n_ops)] + # Read comparators list + comparators = read_expression_list(data) + assert len(ops) == len(comparators) + expr = ComparisonExpr(ops, [left] + comparators) + read_loc(data, expr) + expect_end_tag(data) + return expr else: assert False, tag diff --git a/mypy/nodes.py b/mypy/nodes.py index 1e3b0eebf022d..904bea36ae0e1 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5034,6 +5034,8 @@ def local_definitions( SET_EXPR: Final[Tag] = 174 RETURN_STMT: Final[Tag] = 175 WHILE_STMT: Final[Tag] = 176 +COMPARISON_EXPR: Final[Tag] = 177 +BOOL_OP_EXPR: Final[Tag] = 178 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index fd736468e83d1..bba4652e6a48b 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -168,3 +168,105 @@ MypyFile:1( NameExpr(c) SetExpr:3( IntExpr(3)))) + +[case testBoolOpAnd] +a and b +a and b and c +[out] +MypyFile:1( + ExpressionStmt:1( + OpExpr:1( + and + NameExpr(a) + NameExpr(b))) + ExpressionStmt:2( + OpExpr:2( + and + OpExpr:2( + and + NameExpr(a) + NameExpr(b)) + NameExpr(c)))) + +[case testBoolOpOr] +x or y +1 or 2 or 3 +[out] +MypyFile:1( + ExpressionStmt:1( + OpExpr:1( + or + NameExpr(x) + NameExpr(y))) + ExpressionStmt:2( + OpExpr:2( + or + OpExpr:2( + or + IntExpr(1) + IntExpr(2)) + IntExpr(3)))) + +[case testComparisonSimple] +a < b +x == y +[out] +MypyFile:1( + ExpressionStmt:1( + ComparisonExpr:1( + < + NameExpr(a) + NameExpr(b))) + ExpressionStmt:2( + ComparisonExpr:2( + == + NameExpr(x) + NameExpr(y)))) + +[case testComparisonChained] +a < b < c +x == y != z +[out] +MypyFile:1( + ExpressionStmt:1( + ComparisonExpr:1( + < + < + NameExpr(a) + NameExpr(b) + NameExpr(c))) + ExpressionStmt:2( + ComparisonExpr:2( + == + != + NameExpr(x) + NameExpr(y) + NameExpr(z)))) + +[case testComparisonIsNotIn] +a is b +x is not y +z in w +q not in r +[out] +MypyFile:1( + ExpressionStmt:1( + ComparisonExpr:1( + is + NameExpr(a) + NameExpr(b))) + ExpressionStmt:2( + ComparisonExpr:2( + is not + NameExpr(x) + NameExpr(y))) + ExpressionStmt:3( + ComparisonExpr:3( + in + NameExpr(z) + NameExpr(w))) + ExpressionStmt:4( + ComparisonExpr:4( + not in + NameExpr(q) + NameExpr(r)))) From ddba816c54fcbf5e2721c8333f14c0f04353f2ca Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 30 Dec 2025 11:34:45 +0000 Subject: [PATCH 020/192] Add deserialization test for None, True and False --- test-data/unit/native-parser.test | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index bba4652e6a48b..dbb205a0c6d13 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -270,3 +270,13 @@ MypyFile:1( not in NameExpr(q) NameExpr(r)))) + +[case testLiterals] +(None, True, False) +[out] +MypyFile:1( + ExpressionStmt:1( + TupleExpr:1( + NameExpr(None) + NameExpr(True) + NameExpr(False)))) From ce7e013b3e5c11feb5c54cc1b2f6870d6024037c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 30 Dec 2025 12:04:31 +0000 Subject: [PATCH 021/192] Deserialize func defs (partial) and return statements --- mypy/nativeparse.py | 72 ++++++++++++++++++++++++++++++- mypy/nodes.py | 1 + test-data/unit/native-parser.test | 29 +++++++++++++ 3 files changed, 100 insertions(+), 2 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 345b8ae480c84..9b2c08ce9c8c7 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -37,12 +37,20 @@ ) from mypy.nodes import ( ARG_POS, + ARG_OPT, + ARG_STAR, + ARG_NAMED, + ARG_STAR2, + ARG_NAMED_OPT, + ARG_KINDS, + Argument, AssignmentStmt, Block, CallExpr, ComparisonExpr, Expression, ExpressionStmt, + FuncDef, IfStmt, IndexExpr, IntExpr, @@ -52,10 +60,12 @@ NameExpr, Node, OpExpr, + ReturnStmt, SetExpr, Statement, StrExpr, TupleExpr, + Var, WhileStmt, ) @@ -89,7 +99,57 @@ def parse_to_binary_ast(filename: str) -> bytes: def read_statement(data: ReadBuffer) -> Statement: tag = read_tag(data) stmt: Statement - if tag == nodes.EXPR_STMT: + if tag == nodes.FUNC_DEF_STMT: + # Function name + name = read_str(data) + + # Arguments + expect_tag(data, LIST_GEN) + n_args = read_int_bare(data) + arguments = [] + for _ in range(n_args): + arg_name = read_str(data) + arg_kind_int = read_int(data) + # Convert integer to ArgKind enum using ARG_KINDS tuple + arg_kind = ARG_KINDS[arg_kind_int] + # TODO: Read type annotation when implemented + has_type = read_bool(data) + assert not has_type, "Type annotations not yet supported" + # TODO: Read default value when implemented + has_default = read_bool(data) + assert not has_default, "Default values not yet supported" + pos_only = read_bool(data) + + var = Var(arg_name) + arg = Argument(var, None, None, arg_kind, pos_only) + arguments.append(arg) + + # Body + body = read_block(data) + + # Decorators + expect_tag(data, LIST_GEN) + n_decorators = read_int_bare(data) + assert n_decorators == 0, "Decorators not yet supported" + + # is_async + is_async = read_bool(data) + + # TODO: type_params + has_type_params = read_bool(data) + assert not has_type_params, "Type params not yet supported" + + # TODO: Return type annotation + has_return_type = read_bool(data) + assert not has_return_type, "Return type annotations not yet supported" + + func_def = FuncDef(name, arguments, body) + if is_async: + func_def.is_coroutine = True + read_loc(data, func_def) + expect_end_tag(data) + return func_def + elif tag == nodes.EXPR_STMT: es = ExpressionStmt(read_expression(data)) es.line = es.expr.line es.column = es.expr.column @@ -121,7 +181,15 @@ def read_statement(data: ReadBuffer) -> Statement: expect_end_tag(data) return if_stmt elif tag == nodes.RETURN_STMT: - assert NotImplementedError + has_value = read_bool(data) + if has_value: + value = read_expression(data) + else: + value = None + stmt = ReturnStmt(value) + read_loc(data, stmt) + expect_end_tag(data) + return stmt elif tag == nodes.WHILE_STMT: expr = read_expression(data) body = read_block(data) diff --git a/mypy/nodes.py b/mypy/nodes.py index 904bea36ae0e1..f01fde73337f6 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5036,6 +5036,7 @@ def local_definitions( WHILE_STMT: Final[Tag] = 176 COMPARISON_EXPR: Final[Tag] = 177 BOOL_OP_EXPR: Final[Tag] = 178 +FUNC_DEF_STMT: Final[Tag] = 179 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index dbb205a0c6d13..fad4eed2128b8 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -280,3 +280,32 @@ MypyFile:1( NameExpr(None) NameExpr(True) NameExpr(False)))) + +[case testSimpleFunction] +def foo(): + x = 1 +[out] +MypyFile:1( + FuncDef:1( + foo + Block:2( + AssignmentStmt:2( + NameExpr(x) + IntExpr(1))))) + +[case testFunctionWithArgs] +def add(x, y): + return x + y +[out] +MypyFile:1( + FuncDef:1( + add + Args( + Var:nil(x) + Var:nil(y)) + Block:2( + ReturnStmt:2( + OpExpr:2( + + + NameExpr(x) + NameExpr(y)))))) From a92202c0f69de245e7222dd797bf07e392c2fd00 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 30 Dec 2025 12:11:02 +0000 Subject: [PATCH 022/192] Deserialize 'pass' and test func defs more --- mypy/nativeparse.py | 6 +++ mypy/nodes.py | 1 + mypy/strconv.py | 2 + test-data/unit/native-parser.test | 68 +++++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 9b2c08ce9c8c7..aa18ca2059bc2 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -60,6 +60,7 @@ NameExpr, Node, OpExpr, + PassStmt, ReturnStmt, SetExpr, Statement, @@ -198,6 +199,11 @@ def read_statement(data: ReadBuffer) -> Statement: read_loc(data, stmt) expect_end_tag(data) return stmt + elif tag == nodes.PASS_STMT: + stmt = PassStmt() + read_loc(data, stmt) + expect_end_tag(data) + return stmt else: assert False, tag diff --git a/mypy/nodes.py b/mypy/nodes.py index f01fde73337f6..17746ed119148 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5037,6 +5037,7 @@ def local_definitions( COMPARISON_EXPR: Final[Tag] = 177 BOOL_OP_EXPR: Final[Tag] = 178 FUNC_DEF_STMT: Final[Tag] = 179 +PASS_STMT: Final[Tag] = 180 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/mypy/strconv.py b/mypy/strconv.py index 168a8bcffdc7e..db2b59d49850e 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -147,6 +147,8 @@ def visit_func_def(self, o: mypy.nodes.FuncDef) -> str: arg_kinds = {arg.kind for arg in o.arguments} if len(arg_kinds & {mypy.nodes.ARG_NAMED, mypy.nodes.ARG_NAMED_OPT}) > 0: a.insert(1, f"MaxPos({o.max_pos})") + if o.is_coroutine: + a.insert(1, "Async") if o.abstract_status in (mypy.nodes.IS_ABSTRACT, mypy.nodes.IMPLICITLY_ABSTRACT): a.insert(-1, "Abstract") if o.is_static: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index fad4eed2128b8..4e8a13988b68a 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -309,3 +309,71 @@ MypyFile:1( + NameExpr(x) NameExpr(y)))))) + +[case testFunctionWithVarArgs] +def foo(*args): + pass +[out] +MypyFile:1( + FuncDef:1( + foo + VarArg( + Var:nil(args)) + Block:2( + PassStmt:2()))) + +[case testFunctionWithKwargs] +def bar(**kwargs): + pass +[out] +MypyFile:1( + FuncDef:1( + bar + DictVarArg( + Var:nil(kwargs)) + Block:2( + PassStmt:2()))) + +[case testFunctionWithKwOnly] +def baz(x, *, y): + pass +[out] +MypyFile:1( + FuncDef:1( + baz + MaxPos(1) + Args( + Var:nil(x) + Var:nil(y)) + Block:2( + PassStmt:2()))) + +[case testFunctionWithAllArgKinds] +def complex_fn(a, b, *args, c, **kwargs): + pass +[out] +MypyFile:1( + FuncDef:1( + complex_fn + MaxPos(2) + Args( + Var:nil(a) + Var:nil(b) + Var:nil(c)) + VarArg( + Var:nil(args)) + DictVarArg( + Var:nil(kwargs)) + Block:2( + PassStmt:2()))) + +[case testAsyncFunction] +async def async_foo(): + pass +[out] +MypyFile:1( + FuncDef:1( + async_foo + Async + Block:2( + PassStmt:2()))) From 4db2423daf51509a019e58d78d07abc190a14e53 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 30 Dec 2025 12:29:26 +0000 Subject: [PATCH 023/192] Deserialize parameter defaults --- mypy/nativeparse.py | 9 +++-- test-data/unit/native-parser.test | 65 +++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index aa18ca2059bc2..49d3e98e5badf 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -116,13 +116,16 @@ def read_statement(data: ReadBuffer) -> Statement: # TODO: Read type annotation when implemented has_type = read_bool(data) assert not has_type, "Type annotations not yet supported" - # TODO: Read default value when implemented + # Read default value has_default = read_bool(data) - assert not has_default, "Default values not yet supported" + if has_default: + default = read_expression(data) + else: + default = None pos_only = read_bool(data) var = Var(arg_name) - arg = Argument(var, None, None, arg_kind, pos_only) + arg = Argument(var, None, default, arg_kind, pos_only) arguments.append(arg) # Body diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 4e8a13988b68a..6675906b9ebb2 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -377,3 +377,68 @@ MypyFile:1( Async Block:2( PassStmt:2()))) + +[case testFunctionWithDefaultArg] +def greet(name="World"): + pass +[out] +MypyFile:1( + FuncDef:1( + greet + Args( + default( + Var:nil(name) + StrExpr(World))) + Block:2( + PassStmt:2()))) + +[case testFunctionWithMultipleDefaults] +def add(x=1, y=2): + return x +[out] +MypyFile:1( + FuncDef:1( + add + Args( + default( + Var:nil(x) + IntExpr(1)) + default( + Var:nil(y) + IntExpr(2))) + Block:2( + ReturnStmt:2( + NameExpr(x))))) + +[case testFunctionMixedDefaultsAndRegular] +def func(a, b=5, c=10): + pass +[out] +MypyFile:1( + FuncDef:1( + func + Args( + Var:nil(a) + default( + Var:nil(b) + IntExpr(5)) + default( + Var:nil(c) + IntExpr(10))) + Block:2( + PassStmt:2()))) + +[case testFunctionWithKwOnlyDefault] +def func(*, x=42): + pass +[out] +MypyFile:1( + FuncDef:1( + func + MaxPos(0) + Args( + default( + Var:nil(x) + IntExpr(42))) + Block:2( + PassStmt:2()))) From 2430f83e36e1bf728dbf84803bdb43a3975d68e6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 30 Dec 2025 12:42:45 +0000 Subject: [PATCH 024/192] Deserialize keyword args in calls --- mypy/nativeparse.py | 22 +++++++++++++++++++-- test-data/unit/native-parser.test | 32 +++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 49d3e98e5badf..96bb2e7074ccf 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -26,6 +26,8 @@ END_TAG, LIST_GEN, LIST_INT, + LITERAL_NONE, + LITERAL_STR, LOCATION, ReadBuffer, Tag, @@ -35,6 +37,7 @@ read_tag, read_bool, ) +from librt.internal import read_str as read_str_bare from mypy.nodes import ( ARG_POS, ARG_OPT, @@ -254,8 +257,23 @@ def read_expression(data: ReadBuffer) -> Expression: if tag == nodes.CALL_EXPR: callee = read_expression(data) args = read_expression_list(data) - n = len(args) - ce = CallExpr(callee, args, [ARG_POS] * n, [None] * n) + # Read argument kinds + expect_tag(data, LIST_INT) + n_kinds = read_int_bare(data) + arg_kinds = [ARG_KINDS[read_int_bare(data)] for _ in range(n_kinds)] + # Read argument names + expect_tag(data, LIST_GEN) + n_names = read_int_bare(data) + arg_names = [] + for _ in range(n_names): + tag = read_tag(data) + if tag == LITERAL_NONE: + arg_names.append(None) + elif tag == LITERAL_STR: + arg_names.append(read_str_bare(data)) + else: + assert False, f"Unexpected tag for arg_name: {tag}" + ce = CallExpr(callee, args, arg_kinds, arg_names) read_loc(data, ce) expect_end_tag(data) return ce diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 6675906b9ebb2..e87e1ddbc3ed5 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -442,3 +442,35 @@ MypyFile:1( IntExpr(42))) Block:2( PassStmt:2()))) + +[case testCallWithKeywordArgs] +foo(x=1, y=2) +[out] +MypyFile:1( + ExpressionStmt:1( + CallExpr:1( + NameExpr(foo) + Args() + KwArgs( + x + IntExpr(1)) + KwArgs( + y + IntExpr(2))))) + +[case testCallMixedPositionalAndKeyword] +bar(1, 2, x=3, y=4) +[out] +MypyFile:1( + ExpressionStmt:1( + CallExpr:1( + NameExpr(bar) + Args( + IntExpr(1) + IntExpr(2)) + KwArgs( + x + IntExpr(3)) + KwArgs( + y + IntExpr(4))))) From 5cf0bef0ed9454152d3de1fedf3e07f549ee827e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 30 Dec 2025 12:51:46 +0000 Subject: [PATCH 025/192] Test *args and **kwargs in calls --- test-data/unit/native-parser.test | 49 +++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index e87e1ddbc3ed5..ad239da255db7 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -474,3 +474,52 @@ MypyFile:1( KwArgs( y IntExpr(4))))) + +[case testCallWithStarArgs] +f(1, *x, 2) +[out] +MypyFile:1( + ExpressionStmt:1( + CallExpr:1( + NameExpr(f) + Args( + IntExpr(1) + NameExpr(x) + IntExpr(2)) + VarArg))) + +[case testCallWithDictStarArgs] +f(**x) +f(1, **y) +[out] +MypyFile:1( + ExpressionStmt:1( + CallExpr:1( + NameExpr(f) + Args() + DictVarArg( + NameExpr(x)))) + ExpressionStmt:2( + CallExpr:2( + NameExpr(f) + Args( + IntExpr(1)) + DictVarArg( + NameExpr(y))))) + +[case testCallWithAllArgTypes] +f(1, *x, a=2, **y) +[out] +MypyFile:1( + ExpressionStmt:1( + CallExpr:1( + NameExpr(f) + Args( + IntExpr(1) + NameExpr(x)) + VarArg + KwArgs( + a + IntExpr(2)) + DictVarArg( + NameExpr(y))))) From 7ff2f480d0ce5961a7382341a9bfb5ee319ccb0e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 30 Dec 2025 12:59:38 +0000 Subject: [PATCH 026/192] Minimal deserialization of class defs --- mypy/nativeparse.py | 36 +++++++++++++++++++++++++++++++ test-data/unit/native-parser.test | 25 +++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 96bb2e7074ccf..4733b72e248ee 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -23,6 +23,7 @@ from mypy import nodes from mypy.cache import ( + DICT_STR_GEN, END_TAG, LIST_GEN, LIST_INT, @@ -50,6 +51,7 @@ AssignmentStmt, Block, CallExpr, + ClassDef, ComparisonExpr, Expression, ExpressionStmt, @@ -210,6 +212,40 @@ def read_statement(data: ReadBuffer) -> Statement: read_loc(data, stmt) expect_end_tag(data) return stmt + elif tag == nodes.CLASS_DEF: + # Class name + name = read_str(data) + + # Body + body = read_block(data) + + # TODO: Base classes (skip for now) + expect_tag(data, LIST_GEN) + n_bases = read_int_bare(data) + assert n_bases == 0, "Base classes not yet supported" + + # TODO: Decorators (skip for now) + expect_tag(data, LIST_GEN) + n_decorators = read_int_bare(data) + assert n_decorators == 0, "Decorators not yet supported" + + # TODO: Type parameters (skip for now) + has_type_params = read_bool(data) + assert not has_type_params, "Type parameters not yet supported" + + # TODO: Metaclass (skip for now) + has_metaclass = read_bool(data) + assert not has_metaclass, "Metaclass not yet supported" + + # TODO: Keywords (skip for now) + expect_tag(data, DICT_STR_GEN) + n_keywords = read_int_bare(data) + assert n_keywords == 0, "Keywords not yet supported" + + class_def = ClassDef(name, body) + read_loc(data, class_def) + expect_end_tag(data) + return class_def else: assert False, tag diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index ad239da255db7..8bb9bea6c1efd 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -523,3 +523,28 @@ MypyFile:1( IntExpr(2)) DictVarArg( NameExpr(y))))) + +[case testSimpleClass] +class A: + pass +[out] +MypyFile:1( + ClassDef:1( + A + PassStmt:2())) + +[case testClassWithMethod] +class Foo: + def bar(self): + return 1 +[out] +MypyFile:1( + ClassDef:1( + Foo + FuncDef:2( + bar + Args( + Var:nil(self)) + Block:3( + ReturnStmt:3( + IntExpr(1)))))) From 7d50e25a8d971fd8a7bf6c2e691cde339888c47a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 30 Dec 2025 13:48:31 +0000 Subject: [PATCH 027/192] Support base classes --- mypy/nativeparse.py | 8 +++----- test-data/unit/native-parser.test | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 4733b72e248ee..e25a2139cc9d3 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -219,10 +219,8 @@ def read_statement(data: ReadBuffer) -> Statement: # Body body = read_block(data) - # TODO: Base classes (skip for now) - expect_tag(data, LIST_GEN) - n_bases = read_int_bare(data) - assert n_bases == 0, "Base classes not yet supported" + # Base classes + base_type_exprs = read_expression_list(data) # TODO: Decorators (skip for now) expect_tag(data, LIST_GEN) @@ -242,7 +240,7 @@ def read_statement(data: ReadBuffer) -> Statement: n_keywords = read_int_bare(data) assert n_keywords == 0, "Keywords not yet supported" - class_def = ClassDef(name, body) + class_def = ClassDef(name, body, base_type_exprs=base_type_exprs if base_type_exprs else None) read_loc(data, class_def) expect_end_tag(data) return class_def diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 8bb9bea6c1efd..c6b7e3be31de1 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -548,3 +548,28 @@ MypyFile:1( Block:3( ReturnStmt:3( IntExpr(1)))))) + +[case testClassWithSingleBase] +class A(B): + pass +[out] +MypyFile:1( + ClassDef:1( + A + BaseTypeExpr( + NameExpr(B)) + PassStmt:2())) + +[case testClassWithMultipleBases] +class Child(Base1, Base2): + x = 1 +[out] +MypyFile:1( + ClassDef:1( + Child + BaseTypeExpr( + NameExpr(Base1) + NameExpr(Base2)) + AssignmentStmt:2( + NameExpr(x) + IntExpr(1)))) From 321c215ec3d65436eaad394ff9901c2750e5400a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 30 Dec 2025 13:59:30 +0000 Subject: [PATCH 028/192] Deserialize floats --- mypy/nativeparse.py | 11 ++++++++++- mypy/nodes.py | 1 + test-data/unit/native-parser.test | 8 ++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index e25a2139cc9d3..a32f11a2c59d1 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -27,6 +27,7 @@ END_TAG, LIST_GEN, LIST_INT, + LITERAL_FLOAT, LITERAL_NONE, LITERAL_STR, LOCATION, @@ -38,7 +39,7 @@ read_tag, read_bool, ) -from librt.internal import read_str as read_str_bare +from librt.internal import read_str as read_str_bare, read_float as read_float_bare from mypy.nodes import ( ARG_POS, ARG_OPT, @@ -55,6 +56,7 @@ ComparisonExpr, Expression, ExpressionStmt, + FloatExpr, FuncDef, IfStmt, IndexExpr, @@ -334,6 +336,13 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, ie) expect_end_tag(data) return ie + elif tag == nodes.FLOAT_EXPR: + expect_tag(data, LITERAL_FLOAT) + value = read_float_bare(data) + fe = FloatExpr(value) + read_loc(data, fe) + expect_end_tag(data) + return fe elif tag == nodes.LIST_EXPR: items = read_expression_list(data) expr = ListExpr(items) diff --git a/mypy/nodes.py b/mypy/nodes.py index 17746ed119148..293e3a964b36d 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5038,6 +5038,7 @@ def local_definitions( BOOL_OP_EXPR: Final[Tag] = 178 FUNC_DEF_STMT: Final[Tag] = 179 PASS_STMT: Final[Tag] = 180 +FLOAT_EXPR: Final[Tag] = 181 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index c6b7e3be31de1..e125a99f245e0 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -573,3 +573,11 @@ MypyFile:1( AssignmentStmt:2( NameExpr(x) IntExpr(1)))) + +[case testFloatLiteral] +x = 3.14 +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + FloatExpr(3.14))) From 4e27e8f1864202c4e8d33fb8fe7a7cf4bd89117d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 30 Dec 2025 14:07:24 +0000 Subject: [PATCH 029/192] Deserialize unary expressions --- mypy/nativeparse.py | 9 +++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 24 ++++++++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index a32f11a2c59d1..24dd520e765dd 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -73,6 +73,7 @@ Statement, StrExpr, TupleExpr, + UnaryExpr, Var, WhileStmt, ) @@ -285,6 +286,7 @@ def read_optional_block(data: ReadBuffer) -> Block | None: bin_ops: Final = ["+", "-", "*", "@", "/", "%", "**", "<<", ">>", "|", "^", "&", "//"] bool_ops: Final = ["and", "or"] cmp_ops: Final = ["==", "!=", "<", "<=", ">", ">=", "is", "is not", "in", "not in"] +unary_ops: Final = ["~", "not", "+", "-"] def read_expression(data: ReadBuffer) -> Expression: @@ -409,6 +411,13 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, expr) expect_end_tag(data) return expr + elif tag == nodes.UNARY_EXPR: + op = unary_ops[read_int(data)] + operand = read_expression(data) + expr = UnaryExpr(op, operand) + read_loc(data, expr) + expect_end_tag(data) + return expr else: assert False, tag diff --git a/mypy/nodes.py b/mypy/nodes.py index 293e3a964b36d..5c744f7c9aa9c 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5039,6 +5039,7 @@ def local_definitions( FUNC_DEF_STMT: Final[Tag] = 179 PASS_STMT: Final[Tag] = 180 FLOAT_EXPR: Final[Tag] = 181 +UNARY_EXPR: Final[Tag] = 182 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index e125a99f245e0..60fb65005a13b 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -581,3 +581,27 @@ MypyFile:1( AssignmentStmt:1( NameExpr(x) FloatExpr(3.14))) + +[case testUnaryExpressions] +-x ++y +~z +not w +[out] +MypyFile:1( + ExpressionStmt:1( + UnaryExpr:1( + - + NameExpr(x))) + ExpressionStmt:2( + UnaryExpr:2( + + + NameExpr(y))) + ExpressionStmt:3( + UnaryExpr:3( + ~ + NameExpr(z))) + ExpressionStmt:4( + UnaryExpr:4( + not + NameExpr(w)))) From e5859df66567d94a5132fb140b5d23b2c8864d96 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 30 Dec 2025 15:44:42 +0000 Subject: [PATCH 030/192] Deserialize dict expressions --- mypy/nativeparse.py | 20 ++++++++++++++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 12 ++++++++++++ 3 files changed, 33 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 24dd520e765dd..5aa7ebd0422f8 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -54,6 +54,7 @@ CallExpr, ClassDef, ComparisonExpr, + DictExpr, Expression, ExpressionStmt, FloatExpr, @@ -418,6 +419,25 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, expr) expect_end_tag(data) return expr + elif tag == nodes.DICT_EXPR: + # Read keys + expect_tag(data, LIST_GEN) + n_keys = read_int_bare(data) + keys = [] + for _ in range(n_keys): + has_key = read_bool(data) + if has_key: + keys.append(read_expression(data)) + else: + keys.append(None) + # Read values + values = read_expression_list(data) + # Zip keys and values into items + items = list(zip(keys, values)) + expr = DictExpr(items) + read_loc(data, expr) + expect_end_tag(data) + return expr else: assert False, tag diff --git a/mypy/nodes.py b/mypy/nodes.py index 5c744f7c9aa9c..a082aad5d75f9 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5040,6 +5040,7 @@ def local_definitions( PASS_STMT: Final[Tag] = 180 FLOAT_EXPR: Final[Tag] = 181 UNARY_EXPR: Final[Tag] = 182 +DICT_EXPR: Final[Tag] = 183 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 60fb65005a13b..a69a2e82dda16 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -605,3 +605,15 @@ MypyFile:1( UnaryExpr:4( not NameExpr(w)))) + +[case testDictionaryExpression] +{} +{1:x} +[out] +MypyFile:1( + ExpressionStmt:1( + DictExpr:1()) + ExpressionStmt:2( + DictExpr:2( + IntExpr(1) + NameExpr(x)))) From b7b694335e9de930ab90e15399b15eea65fbc56d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 30 Dec 2025 15:51:24 +0000 Subject: [PATCH 031/192] Deserialize complex literals --- mypy/nativeparse.py | 14 ++++++++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 10 ++++++++++ 3 files changed, 25 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 5aa7ebd0422f8..fb2e7d98ccb74 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -54,6 +54,7 @@ CallExpr, ClassDef, ComparisonExpr, + ComplexExpr, DictExpr, Expression, ExpressionStmt, @@ -438,6 +439,19 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, expr) expect_end_tag(data) return expr + elif tag == nodes.COMPLEX_EXPR: + # Read real part + expect_tag(data, LITERAL_FLOAT) + real = read_float_bare(data) + # Read imaginary part + expect_tag(data, LITERAL_FLOAT) + imag = read_float_bare(data) + # Create complex value + value = complex(real, imag) + expr = ComplexExpr(value) + read_loc(data, expr) + expect_end_tag(data) + return expr else: assert False, tag diff --git a/mypy/nodes.py b/mypy/nodes.py index a082aad5d75f9..b170074568da8 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5041,6 +5041,7 @@ def local_definitions( FLOAT_EXPR: Final[Tag] = 181 UNARY_EXPR: Final[Tag] = 182 DICT_EXPR: Final[Tag] = 183 +COMPLEX_EXPR: Final[Tag] = 184 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index a69a2e82dda16..db67a11d8888f 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -617,3 +617,13 @@ MypyFile:1( DictExpr:2( IntExpr(1) NameExpr(x)))) + +[case testComplexLiteral] +1j +2.5j +[out] +MypyFile:1( + ExpressionStmt:1( + ComplexExpr(1j)) + ExpressionStmt:2( + ComplexExpr(2.5j))) From df5d22af047a06727f7df0ad72535a5bcb6fda6d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 31 Dec 2025 10:52:22 +0000 Subject: [PATCH 032/192] Deserialize slice expressions --- mypy/nativeparse.py | 15 +++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 68 +++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index fb2e7d98ccb74..646137e1ceb27 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -72,6 +72,7 @@ PassStmt, ReturnStmt, SetExpr, + SliceExpr, Statement, StrExpr, TupleExpr, @@ -452,6 +453,20 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, expr) expect_end_tag(data) return expr + elif tag == nodes.SLICE_EXPR: + # Read begin_index (lower in Ruff) + has_begin = read_bool(data) + begin_index = read_expression(data) if has_begin else None + # Read end_index (upper in Ruff) + has_end = read_bool(data) + end_index = read_expression(data) if has_end else None + # Read stride (step in Ruff) + has_stride = read_bool(data) + stride = read_expression(data) if has_stride else None + expr = SliceExpr(begin_index, end_index, stride) + read_loc(data, expr) + expect_end_tag(data) + return expr else: assert False, tag diff --git a/mypy/nodes.py b/mypy/nodes.py index b170074568da8..d7d0ccea71d31 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5042,6 +5042,7 @@ def local_definitions( UNARY_EXPR: Final[Tag] = 182 DICT_EXPR: Final[Tag] = 183 COMPLEX_EXPR: Final[Tag] = 184 +SLICE_EXPR: Final[Tag] = 185 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index db67a11d8888f..02984eb84d9af 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -627,3 +627,71 @@ MypyFile:1( ComplexExpr(1j)) ExpressionStmt:2( ComplexExpr(2.5j))) + +[case testSlices] +x[1:2] +x[:1] +x[1:] +x[:] +[out] +MypyFile:1( + ExpressionStmt:1( + IndexExpr:1( + NameExpr(x) + SliceExpr:1( + IntExpr(1) + IntExpr(2)))) + ExpressionStmt:2( + IndexExpr:2( + NameExpr(x) + SliceExpr:2( + + IntExpr(1)))) + ExpressionStmt:3( + IndexExpr:3( + NameExpr(x) + SliceExpr:3( + IntExpr(1) + ))) + ExpressionStmt:4( + IndexExpr:4( + NameExpr(x) + SliceExpr:4( + + )))) + +[case testSliceWithStride] +x[1:2:3] +x[1::2] +x[:1:2] +x[::2] +[out] +MypyFile:1( + ExpressionStmt:1( + IndexExpr:1( + NameExpr(x) + SliceExpr:1( + IntExpr(1) + IntExpr(2) + IntExpr(3)))) + ExpressionStmt:2( + IndexExpr:2( + NameExpr(x) + SliceExpr:2( + IntExpr(1) + + IntExpr(2)))) + ExpressionStmt:3( + IndexExpr:3( + NameExpr(x) + SliceExpr:3( + + IntExpr(1) + IntExpr(2)))) + ExpressionStmt:4( + IndexExpr:4( + NameExpr(x) + SliceExpr:4( + + + IntExpr(2))))) From 19e7be133d8d9f625d5157d52e9d712816ba4db4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 31 Dec 2025 12:17:45 +0000 Subject: [PATCH 033/192] Partial support for deserializing function return types --- mypy/nativeparse.py | 47 ++++++++++++++++++++++++++++--- test-data/unit/native-parser.test | 18 ++++++++++++ 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 646137e1ceb27..bfc6ec9c8085a 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -21,7 +21,7 @@ import subprocess from typing import Final -from mypy import nodes +from mypy import nodes, types from mypy.cache import ( DICT_STR_GEN, END_TAG, @@ -55,6 +55,7 @@ ClassDef, ComparisonExpr, ComplexExpr, + Context, DictExpr, Expression, ExpressionStmt, @@ -79,7 +80,14 @@ UnaryExpr, Var, WhileStmt, + MISSING_FALLBACK, ) +from mypy.types import CallableType, UnboundType, NoneType, UnionType, AnyType, TypeOfAny, Instance, Type + + +# There is no way to create reasonable fallbacks at this stage, +# they must be patched later. +_dummy_fallback: Final = Instance(MISSING_FALLBACK, [], -1) def expect_end_tag(data: ReadBuffer) -> None: @@ -156,9 +164,23 @@ def read_statement(data: ReadBuffer) -> Statement: # TODO: Return type annotation has_return_type = read_bool(data) - assert not has_return_type, "Return type annotations not yet supported" + if has_return_type: + return_type = read_type(data) + else: + return_type = None + + if return_type is not None: + typ = CallableType( + [AnyType(TypeOfAny.unannotated) for arg in arguments], + [arg.kind for arg in arguments], + [arg.name for arg in arguments], + return_type if return_type else AnyType(TypeOfAny.unannotated), + _dummy_fallback + ) + else: + typ = None - func_def = FuncDef(name, arguments, body) + func_def = FuncDef(name, arguments, body, typ=typ) if is_async: func_def.is_coroutine = True read_loc(data, func_def) @@ -254,6 +276,23 @@ def read_statement(data: ReadBuffer) -> Statement: assert False, tag +def read_type(data: ReadBuffer) -> Type: + tag = read_tag(data) + if tag == types.UNBOUND_TYPE: + name = read_str(data) + expect_tag(data, LIST_GEN) + n = read_int_bare(data) + args = tuple(read_type(data) for i in range(n)) + expect_tag(data, LITERAL_NONE) # TODO + expect_tag(data, LITERAL_NONE) # TODO + unbound = UnboundType(name, args) + read_loc(data, unbound) + expect_end_tag(data) + return unbound + else: + assert False, tag + + def read_block(data: ReadBuffer) -> Block: expect_tag(data, nodes.BLOCK) expect_tag(data, LIST_GEN) @@ -477,7 +516,7 @@ def read_expression_list(data: ReadBuffer) -> list[Expression]: return [read_expression(data) for i in range(n)] -def read_loc(data: ReadBuffer, node: Node) -> None: +def read_loc(data: ReadBuffer, node: Context) -> None: expect_tag(data, LOCATION) line = read_int_bare(data) node.line = line diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 02984eb84d9af..c9910fda9673f 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -695,3 +695,21 @@ MypyFile:1( IntExpr(2))))) + +[case testFuncionSignature] +def f() -> int: + pass +def g() -> None: + pass +[out] +MypyFile:1( + FuncDef:1( + f + def () -> int? + Block:2( + PassStmt:2())) + FuncDef:3( + g + def () -> None? + Block:4( + PassStmt:4()))) From 48635fa681a5e7473c40133233d0160945341690 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 31 Dec 2025 13:16:04 +0000 Subject: [PATCH 034/192] Deserialize parameter types --- mypy/nativeparse.py | 17 ++++++++++++----- test-data/unit/native-parser.test | 15 +++++++++++++-- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index bfc6ec9c8085a..e38bce06b490d 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -127,6 +127,7 @@ def read_statement(data: ReadBuffer) -> Statement: expect_tag(data, LIST_GEN) n_args = read_int_bare(data) arguments = [] + has_ann = False for _ in range(n_args): arg_name = read_str(data) arg_kind_int = read_int(data) @@ -134,7 +135,11 @@ def read_statement(data: ReadBuffer) -> Statement: arg_kind = ARG_KINDS[arg_kind_int] # TODO: Read type annotation when implemented has_type = read_bool(data) - assert not has_type, "Type annotations not yet supported" + if has_type: + ann = read_type(data) + has_ann = True + else: + ann = None # Read default value has_default = read_bool(data) if has_default: @@ -144,7 +149,7 @@ def read_statement(data: ReadBuffer) -> Statement: pos_only = read_bool(data) var = Var(arg_name) - arg = Argument(var, None, default, arg_kind, pos_only) + arg = Argument(var, ann, default, arg_kind, pos_only) arguments.append(arg) # Body @@ -166,14 +171,16 @@ def read_statement(data: ReadBuffer) -> Statement: has_return_type = read_bool(data) if has_return_type: return_type = read_type(data) + has_ann = True else: return_type = None - if return_type is not None: + if has_ann: typ = CallableType( - [AnyType(TypeOfAny.unannotated) for arg in arguments], + [arg.type_annotation if arg.type_annotation else AnyType(TypeOfAny.unannotated) + for arg in arguments], [arg.kind for arg in arguments], - [arg.name for arg in arguments], + [arg.variable.name for arg in arguments], return_type if return_type else AnyType(TypeOfAny.unannotated), _dummy_fallback ) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index c9910fda9673f..c0c013d9d5e2e 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -696,11 +696,13 @@ MypyFile:1( IntExpr(2))))) -[case testFuncionSignature] +[case testFunctionSignature] def f() -> int: pass def g() -> None: pass +def h(x: int, y): + pass [out] MypyFile:1( FuncDef:1( @@ -712,4 +714,13 @@ MypyFile:1( g def () -> None? Block:4( - PassStmt:4()))) + PassStmt:4())) + FuncDef:5( + h + Args( + Var:nil(x) + Var:nil(y)) + def (x: int?, y: Any) -> Any + Block:6( + PassStmt:6()))) + From 45f18625a0c90685d2e6b78edbf38d67edf348d0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 31 Dec 2025 13:30:22 +0000 Subject: [PATCH 035/192] Add test for qualified type names --- test-data/unit/native-parser.test | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index c0c013d9d5e2e..acd2365df72c2 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -701,7 +701,7 @@ def f() -> int: pass def g() -> None: pass -def h(x: int, y): +def h(x: foo.bar, y, z: x.y.z): pass [out] MypyFile:1( @@ -719,8 +719,9 @@ MypyFile:1( h Args( Var:nil(x) - Var:nil(y)) - def (x: int?, y: Any) -> Any + Var:nil(y) + Var:nil(z)) + def (x: foo.bar?, y: Any, z: x.y.z?) -> Any Block:6( PassStmt:6()))) From 4ff18091b30e7b73293bc5caafc92daa6c8af6c3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 31 Dec 2025 13:55:19 +0000 Subject: [PATCH 036/192] Test deserialization of generic types --- test-data/unit/native-parser.test | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index acd2365df72c2..e89e0f43e7c31 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -725,3 +725,15 @@ MypyFile:1( Block:6( PassStmt:6()))) +[case testFunctionSignature2] +def f(x: a[b]) -> c.d[e.f.g[h], i]: + pass +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var:nil(x)) + def (x: a?[b?]) -> c.d?[e.f.g?[h?], i?] + Block:2( + PassStmt:2()))) From 69e5b9fa5097198d2190e36593a2d351eb5d1a03 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 31 Dec 2025 14:06:35 +0000 Subject: [PATCH 037/192] Deserialize union types --- mypy/nativeparse.py | 11 +++++++++++ test-data/unit/native-parser.test | 13 +++++++++++++ 2 files changed, 24 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index e38bce06b490d..a05fbe4e8384f 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -296,6 +296,17 @@ def read_type(data: ReadBuffer) -> Type: read_loc(data, unbound) expect_end_tag(data) return unbound + elif tag == types.UNION_TYPE: + # Read items list + expect_tag(data, LIST_GEN) + n = read_int_bare(data) + items = [read_type(data) for i in range(n)] + # Read uses_pep604_syntax flag + uses_pep604_syntax = read_bool(data) + union = UnionType(items, uses_pep604_syntax=uses_pep604_syntax) + read_loc(data, union) + expect_end_tag(data) + return union else: assert False, tag diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index e89e0f43e7c31..823b52b1cb17f 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -737,3 +737,16 @@ MypyFile:1( def (x: a?[b?]) -> c.d?[e.f.g?[h?], i?] Block:2( PassStmt:2()))) + +[case testUnionTypes] +def f(x: int | str) -> bool | None: + pass +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var:nil(x)) + def (x: Union[int?, str?]) -> Union[bool?, None?] + Block:2( + PassStmt:2()))) From 22f64e0fe4d68250df6d003266c3316e95fdc875 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 31 Dec 2025 14:20:47 +0000 Subject: [PATCH 038/192] Deserialize assignment with type annotation --- mypy/nativeparse.py | 22 +++++++++++++++++++++- mypy/nodes.py | 1 + test-data/unit/native-parser.test | 15 +++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index a05fbe4e8384f..8b4018e59a027 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -76,6 +76,7 @@ SliceExpr, Statement, StrExpr, + TempNode, TupleExpr, UnaryExpr, Var, @@ -204,8 +205,22 @@ def read_statement(data: ReadBuffer) -> Statement: elif tag == nodes.ASSIGNMENT_STMT: lvalues = read_expression_list(data) rvalue = read_expression(data) - a = AssignmentStmt(lvalues, rvalue) + # Read type annotation + has_type = read_bool(data) + if has_type: + type_annotation = read_type(data) + else: + type_annotation = None + # Read new_syntax flag + new_syntax = read_bool(data) + a = AssignmentStmt(lvalues, rvalue, type=type_annotation, new_syntax=new_syntax) read_loc(data, a) + # If rvalue is TempNode, copy location from AssignmentStmt + if isinstance(rvalue, TempNode): + rvalue.line = a.line + rvalue.column = a.column + rvalue.end_line = a.end_line + rvalue.end_column = a.end_column expect_end_tag(data) return a elif tag == nodes.IF_STMT: @@ -524,6 +539,11 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, expr) expect_end_tag(data) return expr + elif tag == nodes.TEMP_NODE: + # TempNode with no attributes + temp = TempNode(AnyType(TypeOfAny.special_form), no_rhs=True) + expect_end_tag(data) + return temp else: assert False, tag diff --git a/mypy/nodes.py b/mypy/nodes.py index d7d0ccea71d31..5e4ebdb81e90e 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5043,6 +5043,7 @@ def local_definitions( DICT_EXPR: Final[Tag] = 183 COMPLEX_EXPR: Final[Tag] = 184 SLICE_EXPR: Final[Tag] = 185 +TEMP_NODE: Final[Tag] = 186 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 823b52b1cb17f..eadcd81fd1fd2 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -750,3 +750,18 @@ MypyFile:1( def (x: Union[int?, str?]) -> Union[bool?, None?] Block:2( PassStmt:2()))) + +[case testAnnotatedAssignment] +x: int +y: str = "hello" +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + TempNode:1( + Any) + int?) + AssignmentStmt:2( + NameExpr(y) + StrExpr(hello) + str?)) From 2744a2083fcc123bb05612d2224ff96e266c59b0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 31 Dec 2025 14:57:20 +0000 Subject: [PATCH 039/192] Deserialize import statements --- mypy/nativeparse.py | 19 +++++++++++++++++++ test-data/unit/native-parser.test | 10 ++++++++++ 2 files changed, 29 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 8b4018e59a027..1a0f08fe34e47 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -62,6 +62,7 @@ FloatExpr, FuncDef, IfStmt, + Import, IndexExpr, IntExpr, ListExpr, @@ -262,6 +263,24 @@ def read_statement(data: ReadBuffer) -> Statement: read_loc(data, stmt) expect_end_tag(data) return stmt + elif tag == nodes.IMPORT: + # Read number of imports + n = read_int(data) + ids = [] + for _ in range(n): + # Read import name + name = read_str(data) + # Read as_name (optional) + has_asname = read_bool(data) + if has_asname: + asname = read_str(data) + else: + asname = None + ids.append((name, asname)) + stmt = Import(ids) + read_loc(data, stmt) + expect_end_tag(data) + return stmt elif tag == nodes.CLASS_DEF: # Class name name = read_str(data) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index eadcd81fd1fd2..51d6eeb688f4d 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -765,3 +765,13 @@ MypyFile:1( NameExpr(y) StrExpr(hello) str?)) + +[case testImportStatements] +import x +import a, b +import y as z +[out] +MypyFile:1( + Import:1(x) + Import:2(a, b) + Import:3(y : z)) From ff7e14bc188db4e43832293fadc64f4c2ac1af06 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 31 Dec 2025 17:49:15 +0000 Subject: [PATCH 040/192] Support raise --- mypy/nativeparse.py | 18 ++++++++++++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 19 +++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 1a0f08fe34e47..f854c79a9e8d2 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -72,6 +72,7 @@ Node, OpExpr, PassStmt, + RaiseStmt, ReturnStmt, SetExpr, SliceExpr, @@ -250,6 +251,23 @@ def read_statement(data: ReadBuffer) -> Statement: read_loc(data, stmt) expect_end_tag(data) return stmt + elif tag == nodes.RAISE_STMT: + # Read exception expression (optional) + has_exc = read_bool(data) + if has_exc: + exc = read_expression(data) + else: + exc = None + # Read from expression (optional) + has_from = read_bool(data) + if has_from: + from_expr = read_expression(data) + else: + from_expr = None + stmt = RaiseStmt(exc, from_expr) + read_loc(data, stmt) + expect_end_tag(data) + return stmt elif tag == nodes.WHILE_STMT: expr = read_expression(data) body = read_block(data) diff --git a/mypy/nodes.py b/mypy/nodes.py index 5e4ebdb81e90e..e6f08402cb54e 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5044,6 +5044,7 @@ def local_definitions( COMPLEX_EXPR: Final[Tag] = 184 SLICE_EXPR: Final[Tag] = 185 TEMP_NODE: Final[Tag] = 186 +RAISE_STMT: Final[Tag] = 187 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 51d6eeb688f4d..ef22972187858 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -775,3 +775,22 @@ MypyFile:1( Import:1(x) Import:2(a, b) Import:3(y : z)) + +[case testRaiseStatements] +raise +raise ValueError +raise ValueError("error message") +raise KeyError from e +[out] +MypyFile:1( + RaiseStmt:1() + RaiseStmt:2( + NameExpr(ValueError)) + RaiseStmt:3( + CallExpr:3( + NameExpr(ValueError) + Args( + StrExpr(error message)))) + RaiseStmt:4( + NameExpr(KeyError) + NameExpr(e))) From dcedc1f675f5751073742fc4215e62bbc5083b6e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 31 Dec 2025 17:52:57 +0000 Subject: [PATCH 041/192] Support break and continue --- mypy/nativeparse.py | 12 ++++++++++++ mypy/nodes.py | 2 ++ test-data/unit/native-parser.test | 31 +++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index f854c79a9e8d2..bc19d55308341 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -51,9 +51,11 @@ Argument, AssignmentStmt, Block, + BreakStmt, CallExpr, ClassDef, ComparisonExpr, + ContinueStmt, ComplexExpr, Context, DictExpr, @@ -281,6 +283,16 @@ def read_statement(data: ReadBuffer) -> Statement: read_loc(data, stmt) expect_end_tag(data) return stmt + elif tag == nodes.BREAK_STMT: + stmt = BreakStmt() + read_loc(data, stmt) + expect_end_tag(data) + return stmt + elif tag == nodes.CONTINUE_STMT: + stmt = ContinueStmt() + read_loc(data, stmt) + expect_end_tag(data) + return stmt elif tag == nodes.IMPORT: # Read number of imports n = read_int(data) diff --git a/mypy/nodes.py b/mypy/nodes.py index e6f08402cb54e..cef786e179811 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5045,6 +5045,8 @@ def local_definitions( SLICE_EXPR: Final[Tag] = 185 TEMP_NODE: Final[Tag] = 186 RAISE_STMT: Final[Tag] = 187 +BREAK_STMT: Final[Tag] = 188 +CONTINUE_STMT: Final[Tag] = 189 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index ef22972187858..650013bd89a96 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -794,3 +794,34 @@ MypyFile:1( RaiseStmt:4( NameExpr(KeyError) NameExpr(e))) + +[case testBreakAndContinue] +while True: + break +while x: + continue +while y: + if z: + break + else: + continue +[out] +MypyFile:1( + WhileStmt:1( + NameExpr(True) + Block:2( + BreakStmt:2())) + WhileStmt:3( + NameExpr(x) + Block:4( + ContinueStmt:4())) + WhileStmt:5( + NameExpr(y) + Block:6( + IfStmt:6( + If( + NameExpr(z)) + Then( + BreakStmt:7()) + Else( + ContinueStmt:9()))))) From 48d0379be0bb8604e60334f0dd04ec4f18ff4b1d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 31 Dec 2025 18:04:20 +0000 Subject: [PATCH 042/192] Support generator expressions --- mypy/nativeparse.py | 18 +++++++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 43 +++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index bc19d55308341..9c93978d965d0 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -63,6 +63,7 @@ ExpressionStmt, FloatExpr, FuncDef, + GeneratorExpr, IfStmt, Import, IndexExpr, @@ -487,6 +488,23 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, expr) expect_end_tag(data) return expr + elif tag == nodes.GENERATOR_EXPR: + # Read element expression + left_expr = read_expression(data) + # Read number of generators + n_generators = read_int(data) + # Read all indices (targets) + indices = [read_expression(data) for _ in range(n_generators)] + # Read all sequences (iters) + sequences = [read_expression(data) for _ in range(n_generators)] + # Read all condlists (ifs for each generator) + condlists = [read_expression_list(data) for _ in range(n_generators)] + # Read all is_async flags + is_async = [read_bool(data) for _ in range(n_generators)] + expr = GeneratorExpr(left_expr, indices, sequences, condlists, is_async) + read_loc(data, expr) + expect_end_tag(data) + return expr elif tag == nodes.OP_EXPR: op = bin_ops[read_int(data)] left = read_expression(data) diff --git a/mypy/nodes.py b/mypy/nodes.py index cef786e179811..27d49e2e32f33 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5047,6 +5047,7 @@ def local_definitions( RAISE_STMT: Final[Tag] = 187 BREAK_STMT: Final[Tag] = 188 CONTINUE_STMT: Final[Tag] = 189 +GENERATOR_EXPR: Final[Tag] = 190 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 650013bd89a96..1776b4b3c67e8 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -825,3 +825,46 @@ MypyFile:1( BreakStmt:7()) Else( ContinueStmt:9()))))) + +[case testGeneratorExpressions] +(x for x in items) +(x * 2 for x in range(10) if x > 5) +(x + y for x in range(3) for y in range(2)) +[out] +MypyFile:1( + ExpressionStmt:1( + GeneratorExpr:1( + NameExpr(x) + NameExpr(x) + NameExpr(items))) + ExpressionStmt:2( + GeneratorExpr:2( + OpExpr:2( + * + NameExpr(x) + IntExpr(2)) + NameExpr(x) + CallExpr:2( + NameExpr(range) + Args( + IntExpr(10))) + ComparisonExpr:2( + > + NameExpr(x) + IntExpr(5)))) + ExpressionStmt:3( + GeneratorExpr:3( + OpExpr:3( + + + NameExpr(x) + NameExpr(y)) + NameExpr(x) + NameExpr(y) + CallExpr:3( + NameExpr(range) + Args( + IntExpr(3))) + CallExpr:3( + NameExpr(range) + Args( + IntExpr(2)))))) From 56ecbda3c394aa42dd1135272934bc89ae6f6d01 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 31 Dec 2025 18:07:38 +0000 Subject: [PATCH 043/192] Support yield and yield from --- mypy/nativeparse.py | 20 ++++++++++++++++++++ mypy/nodes.py | 2 ++ test-data/unit/native-parser.test | 19 +++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 9c93978d965d0..1c9bf63df5621 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -86,6 +86,8 @@ UnaryExpr, Var, WhileStmt, + YieldExpr, + YieldFromExpr, MISSING_FALLBACK, ) from mypy.types import CallableType, UnboundType, NoneType, UnionType, AnyType, TypeOfAny, Instance, Type @@ -505,6 +507,24 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, expr) expect_end_tag(data) return expr + elif tag == nodes.YIELD_EXPR: + # Read optional value expression + has_value = read_bool(data) + if has_value: + value = read_expression(data) + else: + value = None + expr = YieldExpr(value) + read_loc(data, expr) + expect_end_tag(data) + return expr + elif tag == nodes.YIELD_FROM_EXPR: + # Read value expression (required for yield from) + value = read_expression(data) + expr = YieldFromExpr(value) + read_loc(data, expr) + expect_end_tag(data) + return expr elif tag == nodes.OP_EXPR: op = bin_ops[read_int(data)] left = read_expression(data) diff --git a/mypy/nodes.py b/mypy/nodes.py index 27d49e2e32f33..2ca75337b17dc 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5048,6 +5048,8 @@ def local_definitions( BREAK_STMT: Final[Tag] = 188 CONTINUE_STMT: Final[Tag] = 189 GENERATOR_EXPR: Final[Tag] = 190 +YIELD_EXPR: Final[Tag] = 191 +YIELD_FROM_EXPR: Final[Tag] = 192 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 1776b4b3c67e8..eff74f4316f61 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -868,3 +868,22 @@ MypyFile:1( NameExpr(range) Args( IntExpr(2)))))) + +[case testYieldExpressions] +def gen(): + yield + yield 1 + yield from items +[out] +MypyFile:1( + FuncDef:1( + gen + Block:2( + ExpressionStmt:2( + YieldExpr:2()) + ExpressionStmt:3( + YieldExpr:3( + IntExpr(1))) + ExpressionStmt:4( + YieldFromExpr:4( + NameExpr(items)))))) From dc14c80ecd6911498245984a39145be7119fb39b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 31 Dec 2025 18:14:01 +0000 Subject: [PATCH 044/192] Support list and set comprehensions --- mypy/nativeparse.py | 55 +++++++++++++++++++++++-------- mypy/nodes.py | 2 ++ test-data/unit/native-parser.test | 53 +++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 13 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 1c9bf63df5621..a0a0d87b48303 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -66,6 +66,8 @@ GeneratorExpr, IfStmt, Import, + ListComprehension, + SetComprehension, IndexExpr, IntExpr, ListExpr, @@ -491,22 +493,32 @@ def read_expression(data: ReadBuffer) -> Expression: expect_end_tag(data) return expr elif tag == nodes.GENERATOR_EXPR: - # Read element expression - left_expr = read_expression(data) - # Read number of generators - n_generators = read_int(data) - # Read all indices (targets) - indices = [read_expression(data) for _ in range(n_generators)] - # Read all sequences (iters) - sequences = [read_expression(data) for _ in range(n_generators)] - # Read all condlists (ifs for each generator) - condlists = [read_expression_list(data) for _ in range(n_generators)] - # Read all is_async flags - is_async = [read_bool(data) for _ in range(n_generators)] - expr = GeneratorExpr(left_expr, indices, sequences, condlists, is_async) + expr = read_generator_expr(data) read_loc(data, expr) expect_end_tag(data) return expr + elif tag == nodes.LIST_COMPREHENSION: + generator = read_generator_expr(data) + expr = ListComprehension(generator) + read_loc(data, expr) + # Also copy location to the inner generator + generator.line = expr.line + generator.column = expr.column + generator.end_line = expr.end_line + generator.end_column = expr.end_column + expect_end_tag(data) + return expr + elif tag == nodes.SET_COMPREHENSION: + generator = read_generator_expr(data) + expr = SetComprehension(generator) + read_loc(data, expr) + # Also copy location to the inner generator + generator.line = expr.line + generator.column = expr.column + generator.end_line = expr.end_line + generator.end_column = expr.end_column + expect_end_tag(data) + return expr elif tag == nodes.YIELD_EXPR: # Read optional value expression has_value = read_bool(data) @@ -641,6 +653,23 @@ def read_expression_list(data: ReadBuffer) -> list[Expression]: return [read_expression(data) for i in range(n)] +def read_generator_expr(data: ReadBuffer) -> GeneratorExpr: + """Helper function to read comprehension data (shared by Generator, ListComp, SetComp)""" + # Read element expression + left_expr = read_expression(data) + # Read number of generators + n_generators = read_int(data) + # Read all indices (targets) + indices = [read_expression(data) for _ in range(n_generators)] + # Read all sequences (iters) + sequences = [read_expression(data) for _ in range(n_generators)] + # Read all condlists (ifs for each generator) + condlists = [read_expression_list(data) for _ in range(n_generators)] + # Read all is_async flags + is_async = [read_bool(data) for _ in range(n_generators)] + return GeneratorExpr(left_expr, indices, sequences, condlists, is_async) + + def read_loc(data: ReadBuffer, node: Context) -> None: expect_tag(data, LOCATION) line = read_int_bare(data) diff --git a/mypy/nodes.py b/mypy/nodes.py index 2ca75337b17dc..2e1099c38f066 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5050,6 +5050,8 @@ def local_definitions( GENERATOR_EXPR: Final[Tag] = 190 YIELD_EXPR: Final[Tag] = 191 YIELD_FROM_EXPR: Final[Tag] = 192 +LIST_COMPREHENSION: Final[Tag] = 193 +SET_COMPREHENSION: Final[Tag] = 194 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index eff74f4316f61..1a21399fa1ccc 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -887,3 +887,56 @@ MypyFile:1( ExpressionStmt:4( YieldFromExpr:4( NameExpr(items)))))) + +[case testListAndSetComprehensions] +a = [x for x in items] +b = [x * 2 for x in range(10) if x > 5] +c = {y for y in data} +d = {z + 1 for z in nums if z < 100} +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(a) + ListComprehension:1( + GeneratorExpr:1( + NameExpr(x) + NameExpr(x) + NameExpr(items)))) + AssignmentStmt:2( + NameExpr(b) + ListComprehension:2( + GeneratorExpr:2( + OpExpr:2( + * + NameExpr(x) + IntExpr(2)) + NameExpr(x) + CallExpr:2( + NameExpr(range) + Args( + IntExpr(10))) + ComparisonExpr:2( + > + NameExpr(x) + IntExpr(5))))) + AssignmentStmt:3( + NameExpr(c) + SetComprehension:3( + GeneratorExpr:3( + NameExpr(y) + NameExpr(y) + NameExpr(data)))) + AssignmentStmt:4( + NameExpr(d) + SetComprehension:4( + GeneratorExpr:4( + OpExpr:4( + + + NameExpr(z) + IntExpr(1)) + NameExpr(z) + NameExpr(nums) + ComparisonExpr:4( + < + NameExpr(z) + IntExpr(100)))))) From 60a5aeb1a1a74f029c838b20432de521913d8d0e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 1 Jan 2026 10:20:16 +0000 Subject: [PATCH 045/192] Support dict comprehensions --- mypy/nativeparse.py | 20 +++++++++++++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 32 +++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index a0a0d87b48303..7af8306a67fc7 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -59,6 +59,7 @@ ComplexExpr, Context, DictExpr, + DictionaryComprehension, Expression, ExpressionStmt, FloatExpr, @@ -519,6 +520,25 @@ def read_expression(data: ReadBuffer) -> Expression: generator.end_column = expr.end_column expect_end_tag(data) return expr + elif tag == nodes.DICT_COMPREHENSION: + # Read key expression + key = read_expression(data) + # Read value expression + value = read_expression(data) + # Read number of generators + n_generators = read_int(data) + # Read all indices (targets) + indices = [read_expression(data) for _ in range(n_generators)] + # Read all sequences (iters) + sequences = [read_expression(data) for _ in range(n_generators)] + # Read all condlists (ifs for each generator) + condlists = [read_expression_list(data) for _ in range(n_generators)] + # Read all is_async flags + is_async = [read_bool(data) for _ in range(n_generators)] + expr = DictionaryComprehension(key, value, indices, sequences, condlists, is_async) + read_loc(data, expr) + expect_end_tag(data) + return expr elif tag == nodes.YIELD_EXPR: # Read optional value expression has_value = read_bool(data) diff --git a/mypy/nodes.py b/mypy/nodes.py index 2e1099c38f066..62818d2405ca7 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5052,6 +5052,7 @@ def local_definitions( YIELD_FROM_EXPR: Final[Tag] = 192 LIST_COMPREHENSION: Final[Tag] = 193 SET_COMPREHENSION: Final[Tag] = 194 +DICT_COMPREHENSION: Final[Tag] = 195 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 1a21399fa1ccc..0bfff43774ae1 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -940,3 +940,35 @@ MypyFile:1( < NameExpr(z) IntExpr(100)))))) + +[case testDictComprehensions] +a = {k: v for k, v in items} +b = {x: x * 2 for x in range(10) if x > 5} +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(a) + DictionaryComprehension:1( + NameExpr(k) + NameExpr(v) + TupleExpr:1( + NameExpr(k) + NameExpr(v)) + NameExpr(items))) + AssignmentStmt:2( + NameExpr(b) + DictionaryComprehension:2( + NameExpr(x) + OpExpr:2( + * + NameExpr(x) + IntExpr(2)) + NameExpr(x) + CallExpr:2( + NameExpr(range) + Args( + IntExpr(10))) + ComparisonExpr:2( + > + NameExpr(x) + IntExpr(5))))) From e35f8f5cf4fe22d0027e5d33ec8ff228dd4b4472 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 1 Jan 2026 11:43:48 +0000 Subject: [PATCH 046/192] Use the C extension in tests instead of subprocess --- mypy/nativeparse.py | 8 +++----- mypy/test/test_nativeparse.py | 5 ++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 7af8306a67fc7..1e4e5f1bb5972 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -17,10 +17,10 @@ from __future__ import annotations -import os -import subprocess from typing import Final +import ast_serialize + from mypy import nodes, types from mypy.cache import ( DICT_STR_GEN, @@ -122,9 +122,7 @@ def native_parse(filename: str) -> MypyFile: def parse_to_binary_ast(filename: str) -> bytes: - binpath = os.path.expanduser("~/src/ruff/target/release/mypy_parser") - result = subprocess.run([binpath, "serialize-ast", filename], capture_output=True, check=True) - return result.stdout + return ast_serialize.parse(filename) def read_statement(data: ReadBuffer) -> Statement: diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index 501d2fc533b43..4159124b9887b 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -5,7 +5,6 @@ import contextlib import gc import os -import subprocess import tempfile import unittest from collections.abc import Iterator @@ -63,8 +62,8 @@ def test_parser(testcase: DataDrivenTestCase) -> None: with temp_source(source) as fnam: try: node = native_parse(fnam) - except subprocess.CalledProcessError as e: - print(f"Parse failed\nstdout: {e.stdout.decode()}\nstderr: {e.stderr.decode()}") + except ValueError as e: + print(f"Parse failed: {e}") assert False node.path = "main" a = node.str_with_options(options).split("\n") From fc1fc68a903c183c8ecc1e5736531b1aff4d80f3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 1 Jan 2026 12:10:14 +0000 Subject: [PATCH 047/192] Support from ... import --- mypy/nativeparse.py | 26 ++++++++++++++++++++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 16 ++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 1e4e5f1bb5972..6330a9f59e9d3 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -67,6 +67,7 @@ GeneratorExpr, IfStmt, Import, + ImportFrom, ListComprehension, SetComprehension, IndexExpr, @@ -315,6 +316,31 @@ def read_statement(data: ReadBuffer) -> Statement: read_loc(data, stmt) expect_end_tag(data) return stmt + elif tag == nodes.IMPORT_FROM: + # Read relative import level + relative = read_int(data) + + # Read module name (empty string for "from . import x") + module_id = read_str(data) + + # Read number of imported names + n = read_int(data) + names = [] + for _ in range(n): + # Read imported name + name = read_str(data) + # Read optional alias + has_asname = read_bool(data) + if has_asname: + asname = read_str(data) + else: + asname = None + names.append((name, asname)) + + stmt = ImportFrom(module_id, relative, names) + read_loc(data, stmt) + expect_end_tag(data) + return stmt elif tag == nodes.CLASS_DEF: # Class name name = read_str(data) diff --git a/mypy/nodes.py b/mypy/nodes.py index 62818d2405ca7..46acadae3efea 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5053,6 +5053,7 @@ def local_definitions( LIST_COMPREHENSION: Final[Tag] = 193 SET_COMPREHENSION: Final[Tag] = 194 DICT_COMPREHENSION: Final[Tag] = 195 +IMPORT_FROM: Final[Tag] = 196 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 0bfff43774ae1..18b3ede010e30 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -776,6 +776,22 @@ MypyFile:1( Import:2(a, b) Import:3(y : z)) +[case testImportFromStatements] +from os import path +from sys import argv, exit +from typing import List as L +from . import foo +from .. import bar +from .pkg import baz +[out] +MypyFile:1( + ImportFrom:1(os, [path]) + ImportFrom:2(sys, [argv, exit]) + ImportFrom:3(typing, [List : L]) + ImportFrom:4(., [foo]) + ImportFrom:5(.., [bar]) + ImportFrom:6(.pkg, [baz])) + [case testRaiseStatements] raise raise ValueError From db7d848afbab41dfe0f0bc32681a6f2efab5a83d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 1 Jan 2026 12:15:47 +0000 Subject: [PATCH 048/192] Support assert statements --- mypy/nativeparse.py | 14 ++++++++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 17 +++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 6330a9f59e9d3..3c719ec9d3668 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -49,6 +49,7 @@ ARG_NAMED_OPT, ARG_KINDS, Argument, + AssertStmt, AssignmentStmt, Block, BreakStmt, @@ -275,6 +276,19 @@ def read_statement(data: ReadBuffer) -> Statement: read_loc(data, stmt) expect_end_tag(data) return stmt + elif tag == nodes.ASSERT_STMT: + # Read test expression + test = read_expression(data) + # Read optional message expression + has_msg = read_bool(data) + if has_msg: + msg = read_expression(data) + else: + msg = None + stmt = AssertStmt(test, msg) + read_loc(data, stmt) + expect_end_tag(data) + return stmt elif tag == nodes.WHILE_STMT: expr = read_expression(data) body = read_block(data) diff --git a/mypy/nodes.py b/mypy/nodes.py index 46acadae3efea..904fa708e2e14 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5054,6 +5054,7 @@ def local_definitions( SET_COMPREHENSION: Final[Tag] = 194 DICT_COMPREHENSION: Final[Tag] = 195 IMPORT_FROM: Final[Tag] = 196 +ASSERT_STMT: Final[Tag] = 197 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 18b3ede010e30..a46dccff966d6 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -811,6 +811,23 @@ MypyFile:1( NameExpr(KeyError) NameExpr(e))) +[case testAssertStatements] +assert x +assert x == y +assert condition, "error message" +[out] +MypyFile:1( + AssertStmt:1( + NameExpr(x)) + AssertStmt:2( + ComparisonExpr:2( + == + NameExpr(x) + NameExpr(y))) + AssertStmt:3( + NameExpr(condition) + StrExpr(error message))) + [case testBreakAndContinue] while True: break From 1e0d1629f93ee1a024b75356a031ff653b131e5b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 1 Jan 2026 12:21:04 +0000 Subject: [PATCH 049/192] Support 'for' statements --- mypy/nativeparse.py | 14 ++++++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 36 +++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 3c719ec9d3668..e70cc50160421 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -64,6 +64,7 @@ Expression, ExpressionStmt, FloatExpr, + ForStmt, FuncDef, GeneratorExpr, IfStmt, @@ -297,6 +298,19 @@ def read_statement(data: ReadBuffer) -> Statement: read_loc(data, stmt) expect_end_tag(data) return stmt + elif tag == nodes.FOR_STMT: + # Read index (target) + index = read_expression(data) + # Read iterator expression + expr = read_expression(data) + # Read body + body = read_block(data) + # Read else clause + else_body = read_optional_block(data) + stmt = ForStmt(index, expr, body, else_body) + read_loc(data, stmt) + expect_end_tag(data) + return stmt elif tag == nodes.PASS_STMT: stmt = PassStmt() read_loc(data, stmt) diff --git a/mypy/nodes.py b/mypy/nodes.py index 904fa708e2e14..6d3ca85b5de10 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5055,6 +5055,7 @@ def local_definitions( DICT_COMPREHENSION: Final[Tag] = 195 IMPORT_FROM: Final[Tag] = 196 ASSERT_STMT: Final[Tag] = 197 +FOR_STMT: Final[Tag] = 198 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index a46dccff966d6..785263d50115e 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -828,6 +828,42 @@ MypyFile:1( NameExpr(condition) StrExpr(error message))) +[case testForStatements] +for x in items: + pass +for i, j in pairs: + x = 1 +for y in range(10): + break +else: + continue +[out] +MypyFile:1( + ForStmt:1( + NameExpr(x) + NameExpr(items) + Block:2( + PassStmt:2())) + ForStmt:3( + TupleExpr:3( + NameExpr(i) + NameExpr(j)) + NameExpr(pairs) + Block:4( + AssignmentStmt:4( + NameExpr(x) + IntExpr(1)))) + ForStmt:5( + NameExpr(y) + CallExpr:5( + NameExpr(range) + Args( + IntExpr(10))) + Block:6( + BreakStmt:6()) + Else( + ContinueStmt:8()))) + [case testBreakAndContinue] while True: break From badeb1af439ca3e39eef2db262e835b570fce649 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 1 Jan 2026 12:26:40 +0000 Subject: [PATCH 050/192] Support 'with' statements --- mypy/nativeparse.py | 24 +++++++++++++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 38 +++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index e70cc50160421..7d96b8619d80e 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -92,6 +92,7 @@ UnaryExpr, Var, WhileStmt, + WithStmt, YieldExpr, YieldFromExpr, MISSING_FALLBACK, @@ -311,6 +312,29 @@ def read_statement(data: ReadBuffer) -> Statement: read_loc(data, stmt) expect_end_tag(data) return stmt + elif tag == nodes.WITH_STMT: + # Read number of items + n = read_int(data) + expr_list = [] + target_list = [] + # Read each item + for _ in range(n): + # Read context expression + context_expr = read_expression(data) + expr_list.append(context_expr) + # Read optional target + has_target = read_bool(data) + if has_target: + target = read_expression(data) + target_list.append(target) + else: + target_list.append(None) + # Read body + body = read_block(data) + stmt = WithStmt(expr_list, target_list, body) + read_loc(data, stmt) + expect_end_tag(data) + return stmt elif tag == nodes.PASS_STMT: stmt = PassStmt() read_loc(data, stmt) diff --git a/mypy/nodes.py b/mypy/nodes.py index 6d3ca85b5de10..dba80cc1087e1 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5056,6 +5056,7 @@ def local_definitions( IMPORT_FROM: Final[Tag] = 196 ASSERT_STMT: Final[Tag] = 197 FOR_STMT: Final[Tag] = 198 +WITH_STMT: Final[Tag] = 199 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 785263d50115e..cadeb533b005a 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -864,6 +864,44 @@ MypyFile:1( Else( ContinueStmt:8()))) +[case testWithStatements] +with f: + pass +with open("file") as f: + x = 1 +with a as x, b as y: + break +[out] +MypyFile:1( + WithStmt:1( + Expr( + NameExpr(f)) + Block:2( + PassStmt:2())) + WithStmt:3( + Expr( + CallExpr:3( + NameExpr(open) + Args( + StrExpr(file)))) + Target( + NameExpr(f)) + Block:4( + AssignmentStmt:4( + NameExpr(x) + IntExpr(1)))) + WithStmt:5( + Expr( + NameExpr(a)) + Target( + NameExpr(x)) + Expr( + NameExpr(b)) + Target( + NameExpr(y)) + Block:6( + BreakStmt:6()))) + [case testBreakAndContinue] while True: break From 9f7481f8b31141c01164be2213029af2c21adf1d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 1 Jan 2026 12:48:13 +0000 Subject: [PATCH 051/192] Support function decorators (non-overloads) --- mypy/nativeparse.py | 17 +++++++++++++++++ test-data/unit/native-parser.test | 31 +++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 7d96b8619d80e..6e9d65f9ae4ce 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -59,6 +59,7 @@ ContinueStmt, ComplexExpr, Context, + Decorator, DictExpr, DictionaryComprehension, Expression, @@ -206,6 +207,22 @@ def read_statement(data: ReadBuffer) -> Statement: read_loc(data, func_def) expect_end_tag(data) return func_def + elif tag == nodes.DECORATOR: + expect_tag(data, LIST_GEN) + n_decorators = read_int_bare(data) + decorators = [read_expression(data) for i in range(n_decorators)] + fdef = read_statement(data) + assert isinstance(fdef, FuncDef) + var = Var(fdef.name) + # Create Decorator wrapping the FuncDef + stmt = Decorator(fdef, decorators, var) + stmt.line = fdef.line + stmt.column = fdef.column + stmt.end_line = fdef.end_line + stmt.end_column = fdef.end_column + # TODO: Adjust funcdef location to start after decorator? + expect_end_tag(data) + return stmt elif tag == nodes.EXPR_STMT: es = ExpressionStmt(read_expression(data)) es.line = es.expr.line diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index cadeb533b005a..ee826b4f79b8c 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1079,3 +1079,34 @@ MypyFile:1( > NameExpr(x) IntExpr(5))))) + +[case testDecoratedFuncDef] +@dec +def foo(): + pass +class C: + @dec() + @staticmethod + def foo(): + pass +[out] +MypyFile:1( + Decorator:1( + Var:nil(foo) + NameExpr(dec) + FuncDef:1( + foo + Block:3( + PassStmt:3()))) + ClassDef:4( + C + Decorator:5( + Var:nil(foo) + CallExpr:5( + NameExpr(dec) + Args()) + NameExpr(staticmethod) + FuncDef:5( + foo + Block:8( + PassStmt:8()))))) From acba8bb7b2fd164909747c550c7f9d8e7c661743 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 1 Jan 2026 14:37:01 +0000 Subject: [PATCH 052/192] Fix self check --- mypy/nativeparse.py | 14 +++++++------- mypy/test/test_nativeparse.py | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 6e9d65f9ae4ce..7e70c4470ad22 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -1,3 +1,4 @@ +# mypy: allow-redefinition-new, local-partial-types """Python parser that directly constructs a native AST (when compiled). Use a Rust extension to generate a serialized AST, and deserialize the AST directly @@ -19,7 +20,7 @@ from typing import Final -import ast_serialize +import ast_serialize # type: ignore[import-untyped] from mypy import nodes, types from mypy.cache import ( @@ -34,12 +35,11 @@ ReadBuffer, Tag, read_int, - read_int_bare, read_str, read_tag, read_bool, ) -from librt.internal import read_str as read_str_bare, read_float as read_float_bare +from librt.internal import read_str as read_str_bare, read_float as read_float_bare, read_int as read_int_bare from mypy.nodes import ( ARG_POS, ARG_OPT, @@ -127,7 +127,7 @@ def native_parse(filename: str) -> MypyFile: def parse_to_binary_ast(filename: str) -> bytes: - return ast_serialize.parse(filename) + return ast_serialize.parse(filename) # type: ignore[no-any-return] def read_statement(data: ReadBuffer) -> Statement: @@ -333,7 +333,7 @@ def read_statement(data: ReadBuffer) -> Statement: # Read number of items n = read_int(data) expr_list = [] - target_list = [] + target_list: list[Expression | None] = [] # Read each item for _ in range(n): # Read context expression @@ -525,7 +525,7 @@ def read_expression(data: ReadBuffer) -> Expression: # Read argument names expect_tag(data, LIST_GEN) n_names = read_int_bare(data) - arg_names = [] + arg_names: list[str | None] = [] for _ in range(n_names): tag = read_tag(data) if tag == LITERAL_NONE: @@ -709,7 +709,7 @@ def read_expression(data: ReadBuffer) -> Expression: # Read keys expect_tag(data, LIST_GEN) n_keys = read_int_bare(data) - keys = [] + keys: list[Expression | None] = [] for _ in range(n_keys): has_key = read_bool(data) if has_key: diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index 4159124b9887b..b6e0a76b35d30 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -79,7 +79,7 @@ def test_trivial_binary_data(self) -> None: def int_enc(n: int) -> int: return (n + 10) << 1 - def locs(start_line: int, start_column: int, end_line: int, end_column) -> list[int]: + def locs(start_line: int, start_column: int, end_line: int, end_column: int) -> list[int]: return [ LOCATION, int_enc(start_line), @@ -151,8 +151,8 @@ def test_parse_bench(self) -> None: with open(fnam, "rb") as f: data = f.read() node = parse(data, fnam, "__main__", Errors(o), o) - assert False, 1 / ((time.time() - t0) / 100000) assert isinstance(node, MypyFile) + assert False, 1 / ((time.time() - t0) / 100000) @contextlib.contextmanager From 2d698a967eaec99d40d3c8df8ea28e2b84f597d1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 1 Jan 2026 15:19:42 +0000 Subject: [PATCH 053/192] Support operator assignment --- mypy/nativeparse.py | 12 ++++++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 7e70c4470ad22..310135094d8a5 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -81,6 +81,7 @@ NameExpr, Node, OpExpr, + OperatorAssignmentStmt, PassStmt, RaiseStmt, ReturnStmt, @@ -252,6 +253,17 @@ def read_statement(data: ReadBuffer) -> Statement: rvalue.end_column = a.end_column expect_end_tag(data) return a + elif tag == nodes.OPERATOR_ASSIGNMENT_STMT: + # Read operator string + op = read_str(data) + # Read lvalue (target) + lvalue = read_expression(data) + # Read rvalue (value) + rvalue = read_expression(data) + stmt = OperatorAssignmentStmt(op, lvalue, rvalue) + read_loc(data, stmt) + expect_end_tag(data) + return stmt elif tag == nodes.IF_STMT: expr = [read_expression(data)] body = [read_block(data)] diff --git a/mypy/nodes.py b/mypy/nodes.py index dba80cc1087e1..2d99ddbe3ba25 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5057,6 +5057,7 @@ def local_definitions( ASSERT_STMT: Final[Tag] = 197 FOR_STMT: Final[Tag] = 198 WITH_STMT: Final[Tag] = 199 +OPERATOR_ASSIGNMENT_STMT: Final[Tag] = 200 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index ee826b4f79b8c..7b6c13f4fe559 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -69,6 +69,25 @@ MypyFile:1( NameExpr(c)) IntExpr(6))) +[case testOperatorAssignment] +x += 1 +y -= 2 +z *= 3 +[out] +MypyFile:1( + OperatorAssignmentStmt:1( + + + NameExpr(x) + IntExpr(1)) + OperatorAssignmentStmt:2( + - + NameExpr(y) + IntExpr(2)) + OperatorAssignmentStmt:3( + * + NameExpr(z) + IntExpr(3))) + [case testIfStmt] if x: y From ad21b7865ef83e613ae54447585ab97892173226 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 1 Jan 2026 15:25:48 +0000 Subject: [PATCH 054/192] Support try statements --- mypy/nativeparse.py | 53 ++++++++++++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 91 +++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 310135094d8a5..e68b55c6d7117 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -90,6 +90,7 @@ Statement, StrExpr, TempNode, + TryStmt, TupleExpr, UnaryExpr, Var, @@ -454,6 +455,58 @@ def read_statement(data: ReadBuffer) -> Statement: read_loc(data, class_def) expect_end_tag(data) return class_def + elif tag == nodes.TRY_STMT: + # Read try body + body = read_block(data) + + # Read number of except handlers + num_handlers = read_int(data) + + # Read exception types for each handler + types_list = [] + for _ in range(num_handlers): + has_type = read_bool(data) + if has_type: + exc_type = read_expression(data) + types_list.append(exc_type) + else: + types_list.append(None) + + # Read variable names for each handler + vars_list = [] + for _ in range(num_handlers): + has_name = read_bool(data) + if has_name: + var_name = read_str(data) + var_expr = NameExpr(var_name) + vars_list.append(var_expr) + else: + vars_list.append(None) + + # Read handler bodies + handlers = [] + for _ in range(num_handlers): + handler_body = read_block(data) + handlers.append(handler_body) + + # Read else body (optional) + has_else = read_bool(data) + if has_else: + else_body = read_block(data) + else: + else_body = None + + # Read finally body (optional) + has_finally = read_bool(data) + if has_finally: + finally_body = read_block(data) + else: + finally_body = None + + stmt = TryStmt(body, vars_list, types_list, handlers, else_body, finally_body) + read_loc(data, stmt) + expect_end_tag(data) + return stmt else: assert False, tag diff --git a/mypy/nodes.py b/mypy/nodes.py index 2d99ddbe3ba25..6ea5a9d59c2fa 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5058,6 +5058,7 @@ def local_definitions( FOR_STMT: Final[Tag] = 198 WITH_STMT: Final[Tag] = 199 OPERATOR_ASSIGNMENT_STMT: Final[Tag] = 200 +TRY_STMT: Final[Tag] = 201 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 7b6c13f4fe559..f8bfdebdc3ac9 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1129,3 +1129,94 @@ MypyFile:1( foo Block:8( PassStmt:8()))))) +[case testTryFinally] +try: + x +finally: + y +[out] +MypyFile:1( + TryStmt:1( + Block:2( + ExpressionStmt:2( + NameExpr(x))) + Finally( + ExpressionStmt:4( + NameExpr(y))))) + +[case testTryExcept] +try: + x +except Exception: + y +[out] +MypyFile:1( + TryStmt:1( + Block:2( + ExpressionStmt:2( + NameExpr(x))) + NameExpr(Exception) + Block:4( + ExpressionStmt:4( + NameExpr(y))))) + +[case testTryExceptWithVar] +try: + x +except Exception as e: + y +[out] +MypyFile:1( + TryStmt:1( + Block:2( + ExpressionStmt:2( + NameExpr(x))) + NameExpr(Exception) + NameExpr(e) + Block:4( + ExpressionStmt:4( + NameExpr(y))))) + +[case testTryExceptElse] +try: + x +except Exception: + y +else: + z +[out] +MypyFile:1( + TryStmt:1( + Block:2( + ExpressionStmt:2( + NameExpr(x))) + NameExpr(Exception) + Block:4( + ExpressionStmt:4( + NameExpr(y))) + Else( + ExpressionStmt:6( + NameExpr(z))))) + +[case testTryMultipleExcept] +try: + x +except ValueError: + y +except KeyError as e: + z +[out] +MypyFile:1( + TryStmt:1( + Block:2( + ExpressionStmt:2( + NameExpr(x))) + NameExpr(ValueError) + Block:4( + ExpressionStmt:4( + NameExpr(y))) + NameExpr(KeyError) + NameExpr(e) + Block:6( + ExpressionStmt:6( + NameExpr(z))))) From 8d797d6d4992c9c610b3d0d207b8a08aceb372ee Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 1 Jan 2026 15:30:52 +0000 Subject: [PATCH 055/192] Fix test --- test-data/unit/native-parser.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index f8bfdebdc3ac9..1af4f1b7570f1 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -766,7 +766,7 @@ MypyFile:1( f Args( Var:nil(x)) - def (x: Union[int?, str?]) -> Union[bool?, None?] + def (x: int? | str?) -> bool? | None? Block:2( PassStmt:2()))) From da7e92fada8f34c272dede069291f09b5552fe79 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 1 Jan 2026 15:36:26 +0000 Subject: [PATCH 056/192] Support ellipsis expressions --- mypy/nativeparse.py | 6 ++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 5 +++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index e68b55c6d7117..3b5680dd3d4fe 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -62,6 +62,7 @@ Decorator, DictExpr, DictionaryComprehension, + EllipsisExpr, Expression, ExpressionStmt, FloatExpr, @@ -821,6 +822,11 @@ def read_expression(data: ReadBuffer) -> Expression: temp = TempNode(AnyType(TypeOfAny.special_form), no_rhs=True) expect_end_tag(data) return temp + elif tag == nodes.ELLIPSIS_EXPR: + expr = EllipsisExpr() + read_loc(data, expr) + expect_end_tag(data) + return expr else: assert False, tag diff --git a/mypy/nodes.py b/mypy/nodes.py index 6ea5a9d59c2fa..09295145897d9 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5059,6 +5059,7 @@ def local_definitions( WITH_STMT: Final[Tag] = 199 OPERATOR_ASSIGNMENT_STMT: Final[Tag] = 200 TRY_STMT: Final[Tag] = 201 +ELLIPSIS_EXPR: Final[Tag] = 202 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 1af4f1b7570f1..3e7eb4f92262c 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -291,14 +291,15 @@ MypyFile:1( NameExpr(r)))) [case testLiterals] -(None, True, False) +(None, True, False, ...) [out] MypyFile:1( ExpressionStmt:1( TupleExpr:1( NameExpr(None) NameExpr(True) - NameExpr(False)))) + NameExpr(False) + Ellipsis))) [case testSimpleFunction] def foo(): From dbffbf1e3b490f096f8190137d5bfb02b25919af Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 1 Jan 2026 15:45:24 +0000 Subject: [PATCH 057/192] Support 'if' expressions --- mypy/nativeparse.py | 12 ++++++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 11 +++++++++++ 3 files changed, 24 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 3b5680dd3d4fe..018b5b9f8c8f2 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -56,6 +56,7 @@ CallExpr, ClassDef, ComparisonExpr, + ConditionalExpr, ContinueStmt, ComplexExpr, Context, @@ -827,6 +828,17 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, expr) expect_end_tag(data) return expr + elif tag == nodes.CONDITIONAL_EXPR: + # Read if_expr (value when condition is true) + if_expr = read_expression(data) + # Read cond (the condition) + cond = read_expression(data) + # Read else_expr (value when condition is false) + else_expr = read_expression(data) + expr = ConditionalExpr(cond, if_expr, else_expr) + read_loc(data, expr) + expect_end_tag(data) + return expr else: assert False, tag diff --git a/mypy/nodes.py b/mypy/nodes.py index 09295145897d9..f47a04993bbf6 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5060,6 +5060,7 @@ def local_definitions( OPERATOR_ASSIGNMENT_STMT: Final[Tag] = 200 TRY_STMT: Final[Tag] = 201 ELLIPSIS_EXPR: Final[Tag] = 202 +CONDITIONAL_EXPR: Final[Tag] = 203 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 3e7eb4f92262c..0857c29b0bf55 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1221,3 +1221,14 @@ MypyFile:1( Block:6( ExpressionStmt:6( NameExpr(z))))) + +[case testConditionalExprSimple] +x if y else z +[out] +MypyFile:1( + ExpressionStmt:1( + ConditionalExpr:1( + Condition( + NameExpr(y)) + NameExpr(x) + NameExpr(z)))) From 558181c802ade679e3315720f653e7fda407bfcc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 1 Jan 2026 15:57:07 +0000 Subject: [PATCH 058/192] Support type list --- mypy/nativeparse.py | 11 ++++++++- mypy/types.py | 1 + test-data/unit/native-parser.test | 41 +++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 018b5b9f8c8f2..34c5921bab2d1 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -102,7 +102,7 @@ YieldFromExpr, MISSING_FALLBACK, ) -from mypy.types import CallableType, UnboundType, NoneType, UnionType, AnyType, TypeOfAny, Instance, Type +from mypy.types import CallableType, UnboundType, NoneType, UnionType, AnyType, TypeOfAny, Instance, Type, TypeList # There is no way to create reasonable fallbacks at this stage, @@ -537,6 +537,15 @@ def read_type(data: ReadBuffer) -> Type: read_loc(data, union) expect_end_tag(data) return union + elif tag == types.LIST_TYPE: + # Read items list + expect_tag(data, LIST_GEN) + n = read_int_bare(data) + items = [read_type(data) for i in range(n)] + type_list = TypeList(items) + read_loc(data, type_list) + expect_end_tag(data) + return type_list else: assert False, tag diff --git a/mypy/types.py b/mypy/types.py index 207e87984bed9..32e0113567308 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -4320,6 +4320,7 @@ def type_vars_as_args(type_vars: Sequence[TypeVarLikeType]) -> tuple[Type, ...]: UNION_TYPE: Final[Tag] = 115 TYPE_TYPE: Final[Tag] = 116 PARAMETERS: Final[Tag] = 117 +LIST_TYPE: Final[Tag] = 118 # Only valid in serialized ASTs def read_type(data: ReadBuffer, tag: Tag | None = None) -> Type: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 0857c29b0bf55..b047b13c46d7c 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1232,3 +1232,44 @@ MypyFile:1( NameExpr(y)) NameExpr(x) NameExpr(z)))) + +[case testFunctionWithCallableType] +def f(x: Callable[[int, str], None]) -> None: + pass +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var:nil(x)) + def (x: Callable?[, None?]) -> None? + Block:2( + PassStmt:2()))) + +[case testFunctionWithEmptyCallableType] +def g(callback: Callable[[], bool]) -> int: + return 1 +[out] +MypyFile:1( + FuncDef:1( + g + Args( + Var:nil(callback)) + def (callback: Callable?[, bool?]) -> int? + Block:2( + ReturnStmt:2( + IntExpr(1))))) + +[case testFunctionWithComplexCallableType] +def h(f: Callable[[List[int], Dict[str, int]], Tuple[int, str]]) -> None: + pass +[out] +MypyFile:1( + FuncDef:1( + h + Args( + Var:nil(f)) + def (f: Callable?[, Tuple?[int?, str?]]) -> None? + Block:2( + PassStmt:2()))) + From 7853cc155935c747c9234a94646ef2dc78f9cc8d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 1 Jan 2026 16:04:00 +0000 Subject: [PATCH 059/192] Support ellipsis types --- mypy/nativeparse.py | 8 +++++++- mypy/types.py | 1 + test-data/unit/native-parser.test | 12 ++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 34c5921bab2d1..f0b949676fd90 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -102,7 +102,7 @@ YieldFromExpr, MISSING_FALLBACK, ) -from mypy.types import CallableType, UnboundType, NoneType, UnionType, AnyType, TypeOfAny, Instance, Type, TypeList +from mypy.types import CallableType, UnboundType, NoneType, UnionType, AnyType, TypeOfAny, Instance, Type, TypeList, EllipsisType # There is no way to create reasonable fallbacks at this stage, @@ -546,6 +546,12 @@ def read_type(data: ReadBuffer) -> Type: read_loc(data, type_list) expect_end_tag(data) return type_list + elif tag == types.ELLIPSIS_TYPE: + # EllipsisType has no attributes + ellipsis_type = EllipsisType() + read_loc(data, ellipsis_type) + expect_end_tag(data) + return ellipsis_type else: assert False, tag diff --git a/mypy/types.py b/mypy/types.py index 32e0113567308..326ec66212ddf 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -4321,6 +4321,7 @@ def type_vars_as_args(type_vars: Sequence[TypeVarLikeType]) -> tuple[Type, ...]: TYPE_TYPE: Final[Tag] = 116 PARAMETERS: Final[Tag] = 117 LIST_TYPE: Final[Tag] = 118 # Only valid in serialized ASTs +ELLIPSIS_TYPE: Final[Tag] = 119 # Only valid in serialized ASTs def read_type(data: ReadBuffer, tag: Tag | None = None) -> Type: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index b047b13c46d7c..4be01cd2b3915 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1273,3 +1273,15 @@ MypyFile:1( Block:2( PassStmt:2()))) +[case testFunctionWithEllipsisCallableType] +def f(callback: Callable[..., int]) -> None: + pass +[out] +MypyFile:1( + FuncDef:1( + f + Args( + Var:nil(callback)) + def (callback: Callable?[..., int?]) -> None? + Block:2( + PassStmt:2()))) From 66d0cd1bca1620cf3525f6e37cb80db429ef3046 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 2 Jan 2026 09:33:50 +0000 Subject: [PATCH 060/192] Support del statements --- mypy/nativeparse.py | 8 ++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 21 +++++++++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index f0b949676fd90..d1dc7f65b2b49 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -61,6 +61,7 @@ ComplexExpr, Context, Decorator, + DelStmt, DictExpr, DictionaryComprehension, EllipsisExpr, @@ -509,6 +510,13 @@ def read_statement(data: ReadBuffer) -> Statement: read_loc(data, stmt) expect_end_tag(data) return stmt + elif tag == nodes.DEL_STMT: + # Read the target expression + expr = read_expression(data) + stmt = DelStmt(expr) + read_loc(data, stmt) + expect_end_tag(data) + return stmt else: assert False, tag diff --git a/mypy/nodes.py b/mypy/nodes.py index f47a04993bbf6..a64feec2e8486 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5061,6 +5061,7 @@ def local_definitions( TRY_STMT: Final[Tag] = 201 ELLIPSIS_EXPR: Final[Tag] = 202 CONDITIONAL_EXPR: Final[Tag] = 203 +DEL_STMT: Final[Tag] = 204 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 4be01cd2b3915..fa511b1bb6d7b 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1285,3 +1285,24 @@ MypyFile:1( def (callback: Callable?[..., int?]) -> None? Block:2( PassStmt:2()))) + +[case testDelStmt] +del x +del x[0], y[1] +del a.b +[out] +MypyFile:1( + DelStmt:1( + NameExpr(x)) + DelStmt:2( + TupleExpr:2( + IndexExpr:2( + NameExpr(x) + IntExpr(0)) + IndexExpr:2( + NameExpr(y) + IntExpr(1)))) + DelStmt:3( + MemberExpr:3( + NameExpr(a) + b))) From ac8fca50cc98a19b421a2f5acf36b1502641e587 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 2 Jan 2026 10:52:50 +0000 Subject: [PATCH 061/192] Minimal support for f-strings --- mypy/nativeparse.py | 39 +++++++++++++++++++++++++++++++ mypy/nodes.py | 2 ++ test-data/unit/native-parser.test | 20 ++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index d1dc7f65b2b49..7265d600e2f66 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -862,10 +862,49 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, expr) expect_end_tag(data) return expr + elif tag == nodes.FSTRING_EXPR: + # F-strings are converted into nodes representing "".join([...]), to match + # pre-existing behavior. + items = [] + expect_tag(data, LIST_GEN) + n = read_int_bare(data) + for _ in range(n): + t = read_tag(data) + if t == LITERAL_STR: + str_expr = StrExpr(read_str_bare(data)) + items.append(str_expr) + read_loc(data, str_expr) + elif t == nodes.FSTRING_INTERPOLATION: + expr = read_expression(data) + member = MemberExpr(StrExpr("{:{}}"), "format") + set_line_column(member, expr) + call = CallExpr(member, [expr, StrExpr("")], [ARG_POS, ARG_POS], [None, None]) + set_line_column(call, expr) + items.append(call) + b = read_bool(data) + assert not b + expect_end_tag(data) + else: + raise ValueError(f"Unexpected tag {t}") + l = ListExpr(items) + str_expr = StrExpr("") + member = MemberExpr(str_expr, "join") + call = CallExpr(member, [l], [ARG_POS], [None]) + read_loc(data, call) + set_line_column(l, call) + set_line_column(str_expr, call) + set_line_column(member, call) + expect_end_tag(data) + return call else: assert False, tag +def set_line_column(target: Context, src: Context) -> None: + target.line = src.line + target.column = src.column + + def read_expression_list(data: ReadBuffer) -> list[Expression]: expect_tag(data, LIST_GEN) n = read_int_bare(data) diff --git a/mypy/nodes.py b/mypy/nodes.py index a64feec2e8486..d7383dbfa37b5 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5062,6 +5062,8 @@ def local_definitions( ELLIPSIS_EXPR: Final[Tag] = 202 CONDITIONAL_EXPR: Final[Tag] = 203 DEL_STMT: Final[Tag] = 204 +FSTRING_EXPR: Final[Tag] = 205 +FSTRING_INTERPOLATION: Final[Tag] = 206 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index fa511b1bb6d7b..612abfe2ad77f 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1306,3 +1306,23 @@ MypyFile:1( MemberExpr:3( NameExpr(a) b))) + +[case testFString] +f"foo {x}" +[out] +MypyFile:1( + ExpressionStmt:1( + CallExpr:1( + MemberExpr:1( + StrExpr() + join) + Args( + ListExpr:1( + StrExpr(foo ) + CallExpr:1( + MemberExpr:1( + StrExpr({:{}}) + format) + Args( + NameExpr(x) + StrExpr()))))))) From 363b0dfb6061a400f047f944933d4426e75bf42a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 2 Jan 2026 11:13:13 +0000 Subject: [PATCH 062/192] Support f-string conversion flags --- mypy/nativeparse.py | 8 +++++++- test-data/unit/native-parser.test | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 7265d600e2f66..aec4fd263b242 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -876,7 +876,13 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, str_expr) elif t == nodes.FSTRING_INTERPOLATION: expr = read_expression(data) - member = MemberExpr(StrExpr("{:{}}"), "format") + has_conv = read_bool(data) + if has_conv: + c = read_str(data) + fmt = "{" + c + ":{}}" + else: + fmt = "{:{}}" + member = MemberExpr(StrExpr(fmt), "format") set_line_column(member, expr) call = CallExpr(member, [expr, StrExpr("")], [ARG_POS, ARG_POS], [None, None]) set_line_column(call, expr) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 612abfe2ad77f..d6b28e7dc723a 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1326,3 +1326,27 @@ MypyFile:1( Args( NameExpr(x) StrExpr()))))))) + +[case testFStringWithConversion] +x = 'mypy' +F'Hello {x!r}' +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + StrExpr(mypy)) + ExpressionStmt:2( + CallExpr:2( + MemberExpr:2( + StrExpr() + join) + Args( + ListExpr:2( + StrExpr(Hello ) + CallExpr:2( + MemberExpr:2( + StrExpr({!r:{}}) + format) + Args( + NameExpr(x) + StrExpr()))))))) From 1902bc5d018f2a7ef45e7b452ab8e098b7f4699e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 2 Jan 2026 11:53:47 +0000 Subject: [PATCH 063/192] Support f-string format specs and optimize simple f-strings --- mypy/nativeparse.py | 94 +++++++++++++++++++------------ test-data/unit/native-parser.test | 31 ++++++++++ 2 files changed, 88 insertions(+), 37 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index aec4fd263b242..4bc0998052dd4 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -196,7 +196,7 @@ def read_statement(data: ReadBuffer) -> Statement: if has_ann: typ = CallableType( - [arg.type_annotation if arg.type_annotation else AnyType(TypeOfAny.unannotated) + [arg.type_annotation if arg.type_annotation else AnyType(TypeOfAny.unannotated) for arg in arguments], [arg.kind for arg in arguments], [arg.variable.name for arg in arguments], @@ -865,47 +865,67 @@ def read_expression(data: ReadBuffer) -> Expression: elif tag == nodes.FSTRING_EXPR: # F-strings are converted into nodes representing "".join([...]), to match # pre-existing behavior. - items = [] - expect_tag(data, LIST_GEN) - n = read_int_bare(data) - for _ in range(n): - t = read_tag(data) - if t == LITERAL_STR: - str_expr = StrExpr(read_str_bare(data)) - items.append(str_expr) - read_loc(data, str_expr) - elif t == nodes.FSTRING_INTERPOLATION: - expr = read_expression(data) - has_conv = read_bool(data) - if has_conv: - c = read_str(data) - fmt = "{" + c + ":{}}" - else: - fmt = "{:{}}" - member = MemberExpr(StrExpr(fmt), "format") - set_line_column(member, expr) - call = CallExpr(member, [expr, StrExpr("")], [ARG_POS, ARG_POS], [None, None]) - set_line_column(call, expr) - items.append(call) - b = read_bool(data) - assert not b - expect_end_tag(data) - else: - raise ValueError(f"Unexpected tag {t}") - l = ListExpr(items) - str_expr = StrExpr("") - member = MemberExpr(str_expr, "join") - call = CallExpr(member, [l], [ARG_POS], [None]) - read_loc(data, call) - set_line_column(l, call) - set_line_column(str_expr, call) - set_line_column(member, call) + expr = read_fstring_items(data) expect_end_tag(data) - return call + return expr else: assert False, tag +def read_fstring_items(data: ReadBuffer) -> Expression: + items = [] + expect_tag(data, LIST_GEN) + n = read_int_bare(data) + items = [read_fstring_item(data) for i in range(n)] + if len(items) == 1 and isinstance(items[0], StrExpr): + str_expr = items[0] + read_loc(data, str_expr) + return str_expr + args = ListExpr(items) + str_expr = StrExpr("") + member = MemberExpr(str_expr, "join") + call = CallExpr(member, [args], [ARG_POS], [None]) + read_loc(data, call) + set_line_column(args, call) + set_line_column(str_expr, call) + set_line_column(member, call) + return call + + +def read_fstring_item(data: ReadBuffer) -> Expression: + t = read_tag(data) + if t == LITERAL_STR: + str_expr = StrExpr(read_str_bare(data)) + read_loc(data, str_expr) + return str_expr + elif t == nodes.FSTRING_INTERPOLATION: + expr = read_expression(data) + + # Read conversion flag such as !r + has_conv = read_bool(data) + if has_conv: + c = read_str(data) + fmt = "{" + c + ":{}}" + else: + fmt = "{:{}}" + + # Read format spec such as <30 (which may have nested {...}) + has_spec = read_bool(data) + if has_spec: + spec = read_fstring_items(data) + else: + spec = StrExpr("") + + member = MemberExpr(StrExpr(fmt), "format") + set_line_column(member, expr) + call = CallExpr(member, [expr, spec], [ARG_POS, ARG_POS], [None, None]) + set_line_column(call, expr) + expect_end_tag(data) + return call + else: + raise ValueError(f"Unexpected tag {t}") + + def set_line_column(target: Context, src: Context) -> None: target.line = src.line target.column = src.column diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index d6b28e7dc723a..3a36bdbde2cd9 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1350,3 +1350,34 @@ MypyFile:1( Args( NameExpr(x) StrExpr()))))))) + +[case testFStringTrivial] +f"foo" +[out] +MypyFile:1( + ExpressionStmt:1( + StrExpr(foo))) + +[case testFStringWithOnlyFormatSpecifier] +x = 'mypy' +f'Hello {x:<30}' +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + StrExpr(mypy)) + ExpressionStmt:2( + CallExpr:2( + MemberExpr:2( + StrExpr() + join) + Args( + ListExpr:2( + StrExpr(Hello ) + CallExpr:2( + MemberExpr:2( + StrExpr({:{}}) + format) + Args( + NameExpr(x) + StrExpr(<30)))))))) \ No newline at end of file From 1ea538ee9f885eabc4bba2c6679c15c31b454088 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 2 Jan 2026 11:54:32 +0000 Subject: [PATCH 064/192] Add f-string test case --- test-data/unit/native-parser.test | 46 ++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 3a36bdbde2cd9..bf6be7b45cf8a 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1380,4 +1380,48 @@ MypyFile:1( format) Args( NameExpr(x) - StrExpr(<30)))))))) \ No newline at end of file + StrExpr(<30)))))))) + +[case testFStringWithFormatSpecifierExpression] +x = 'mypy' +y = 30 +f'Hello {x!s:<{y+y}}' +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + StrExpr(mypy)) + AssignmentStmt:2( + NameExpr(y) + IntExpr(30)) + ExpressionStmt:3( + CallExpr:3( + MemberExpr:3( + StrExpr() + join) + Args( + ListExpr:3( + StrExpr(Hello ) + CallExpr:3( + MemberExpr:3( + StrExpr({!s:{}}) + format) + Args( + NameExpr(x) + CallExpr:3( + MemberExpr:3( + StrExpr() + join) + Args( + ListExpr:3( + StrExpr(<) + CallExpr:3( + MemberExpr:3( + StrExpr({:{}}) + format) + Args( + OpExpr:3( + + + NameExpr(y) + NameExpr(y)) + StrExpr())))))))))))) From b396a3dc2021218617c38caca0133af30835fc48 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 2 Jan 2026 12:06:39 +0000 Subject: [PATCH 065/192] Optimize more simple cases --- mypy/nativeparse.py | 8 ++++---- test-data/unit/native-parser.test | 13 +++++++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 4bc0998052dd4..33ab85e304c16 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -877,10 +877,10 @@ def read_fstring_items(data: ReadBuffer) -> Expression: expect_tag(data, LIST_GEN) n = read_int_bare(data) items = [read_fstring_item(data) for i in range(n)] - if len(items) == 1 and isinstance(items[0], StrExpr): - str_expr = items[0] - read_loc(data, str_expr) - return str_expr + if len(items) == 1: + expr = items[0] + read_loc(data, expr) + return expr args = ListExpr(items) str_expr = StrExpr("") member = MemberExpr(str_expr, "join") diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index bf6be7b45cf8a..0d78a343491a1 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1353,10 +1353,19 @@ MypyFile:1( [case testFStringTrivial] f"foo" +f"{x}" [out] MypyFile:1( ExpressionStmt:1( - StrExpr(foo))) + StrExpr(foo)) + ExpressionStmt:2( + CallExpr:2( + MemberExpr:2( + StrExpr({:{}}) + format) + Args( + NameExpr(x) + StrExpr())))) [case testFStringWithOnlyFormatSpecifier] x = 'mypy' @@ -1381,7 +1390,7 @@ MypyFile:1( Args( NameExpr(x) StrExpr(<30)))))))) - + [case testFStringWithFormatSpecifierExpression] x = 'mypy' y = 30 From cf538c68f30eb8945a58a64485f49ba7a6f23e32 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 2 Jan 2026 12:37:34 +0000 Subject: [PATCH 066/192] Fix string literals mixed with f-strings --- mypy/nativeparse.py | 28 ++++++++++++++++++++++++---- test-data/unit/native-parser.test | 5 ++++- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 33ab85e304c16..9e44f027b7530 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -18,7 +18,7 @@ from __future__ import annotations -from typing import Final +from typing import Final, cast import ast_serialize # type: ignore[import-untyped] @@ -865,7 +865,19 @@ def read_expression(data: ReadBuffer) -> Expression: elif tag == nodes.FSTRING_EXPR: # F-strings are converted into nodes representing "".join([...]), to match # pre-existing behavior. - expr = read_fstring_items(data) + nparts = read_int(data) + items = [] + for _ in range(nparts): + b = read_bool(data) + if b: + n = read_int(data) + for i in range(n): + items.append(read_fstring_item(data)) + else: + s = StrExpr(read_str(data)) + read_loc(data, s) + items.append(s) + expr = build_fstring_join(data, items) expect_end_tag(data) return expr else: @@ -874,13 +886,21 @@ def read_expression(data: ReadBuffer) -> Expression: def read_fstring_items(data: ReadBuffer) -> Expression: items = [] - expect_tag(data, LIST_GEN) - n = read_int_bare(data) + n = read_int(data) items = [read_fstring_item(data) for i in range(n)] + return build_fstring_join(data, items) + + +def build_fstring_join(data: ReadBuffer, items: list[Expression]) -> Expression: if len(items) == 1: expr = items[0] read_loc(data, expr) return expr + if all(isinstance(item, StrExpr) for item in items): + s = "".join([cast(StrExpr, item).value for item in items]) + expr = StrExpr(s) + read_loc(data, expr) + return expr args = ListExpr(items) str_expr = StrExpr("") member = MemberExpr(str_expr, "join") diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 0d78a343491a1..168fcc9c2a9d4 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1354,6 +1354,7 @@ MypyFile:1( [case testFStringTrivial] f"foo" f"{x}" +f"foo" "bar" [out] MypyFile:1( ExpressionStmt:1( @@ -1365,7 +1366,9 @@ MypyFile:1( format) Args( NameExpr(x) - StrExpr())))) + StrExpr()))) + ExpressionStmt:3( + StrExpr(foobar))) [case testFStringWithOnlyFormatSpecifier] x = 'mypy' From 05fa03eebb5d220bbd239c9a5ce8e0809a200eb2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 2 Jan 2026 13:11:40 +0000 Subject: [PATCH 067/192] Support lambda expressions --- mypy/nativeparse.py | 51 +++++++++++++++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 74 +++++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 9e44f027b7530..1a2fa99837062 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -78,6 +78,7 @@ SetComprehension, IndexExpr, IntExpr, + LambdaExpr, ListExpr, MemberExpr, MypyFile, @@ -880,6 +881,56 @@ def read_expression(data: ReadBuffer) -> Expression: expr = build_fstring_join(data, items) expect_end_tag(data) return expr + elif tag == nodes.LAMBDA_EXPR: + # Read arguments + expect_tag(data, LIST_GEN) + n_args = read_int_bare(data) + arguments = [] + has_ann = False + for _ in range(n_args): + arg_name = read_str(data) + arg_kind_int = read_int(data) + arg_kind = ARG_KINDS[arg_kind_int] + # Read type annotation + has_type = read_bool(data) + if has_type: + ann = read_type(data) + has_ann = True + else: + ann = None + # Read default value + has_default = read_bool(data) + if has_default: + default = read_expression(data) + else: + default = None + pos_only = read_bool(data) + + var = Var(arg_name) + arg = Argument(var, ann, default, arg_kind, pos_only) + arguments.append(arg) + + # Read body block + body = read_block(data) + + # Create lambda expression + if has_ann: + typ = CallableType( + [arg.type_annotation if arg.type_annotation else AnyType(TypeOfAny.unannotated) + for arg in arguments], + [arg.kind for arg in arguments], + [arg.variable.name for arg in arguments], + AnyType(TypeOfAny.unannotated), + _dummy_fallback, + ) + else: + typ = None + + expr = LambdaExpr(arguments, body) + expr.type = typ + read_loc(data, expr) + expect_end_tag(data) + return expr else: assert False, tag diff --git a/mypy/nodes.py b/mypy/nodes.py index d7383dbfa37b5..4134e43477720 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5064,6 +5064,7 @@ def local_definitions( DEL_STMT: Final[Tag] = 204 FSTRING_EXPR: Final[Tag] = 205 FSTRING_INTERPOLATION: Final[Tag] = 206 +LAMBDA_EXPR: Final[Tag] = 207 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 168fcc9c2a9d4..177a18253a502 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1437,3 +1437,77 @@ MypyFile:1( NameExpr(y) NameExpr(y)) StrExpr())))))))))))) + +[case testSimpleLambda] +lambda: 1 +[out] +MypyFile:1( + ExpressionStmt:1( + LambdaExpr:1( + Block:1( + ReturnStmt:1( + IntExpr(1)))))) + +[case testLambdaWithOneArg] +lambda x: y + 1 +[out] +MypyFile:1( + ExpressionStmt:1( + LambdaExpr:1( + Args( + Var:nil(x)) + Block:1( + ReturnStmt:1( + OpExpr:1( + + + NameExpr(y) + IntExpr(1))))))) + +[case testLambdaWithMultipleArgs] +lambda x, y: x + y +[out] +MypyFile:1( + ExpressionStmt:1( + LambdaExpr:1( + Args( + Var:nil(x) + Var:nil(y)) + Block:1( + ReturnStmt:1( + OpExpr:1( + + + NameExpr(x) + NameExpr(y))))))) + +[case testLambdaWithDefault] +lambda x=2: x +[out] +MypyFile:1( + ExpressionStmt:1( + LambdaExpr:1( + Args( + default( + Var:nil(x) + IntExpr(2))) + Block:1( + ReturnStmt:1( + NameExpr(x)))))) + +[case testLambdaInCall] +map(lambda x: x * 2, items) +[out] +MypyFile:1( + ExpressionStmt:1( + CallExpr:1( + NameExpr(map) + Args( + LambdaExpr:1( + Args( + Var:nil(x)) + Block:1( + ReturnStmt:1( + OpExpr:1( + * + NameExpr(x) + IntExpr(2))))) + NameExpr(items))))) From a72892a8d893ce729a9873f779d6639c5501cf14 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 2 Jan 2026 13:20:14 +0000 Subject: [PATCH 068/192] Support assignment expressions (:=) --- mypy/nativeparse.py | 15 +++++++++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 13 +++++++++++++ 3 files changed, 29 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 1a2fa99837062..3543689ba865e 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -50,6 +50,7 @@ ARG_KINDS, Argument, AssertStmt, + AssignmentExpr, AssignmentStmt, Block, BreakStmt, @@ -931,6 +932,20 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, expr) expect_end_tag(data) return expr + elif tag == nodes.NAMED_EXPR: + # Read target expression + target = read_expression(data) + # Read value expression + value = read_expression(data) + # AssignmentExpr expects target to be a NameExpr + if not isinstance(target, NameExpr): + # In case target is not a NameExpr, we need to handle this + # For now, we'll assert since the grammar should ensure it's a NameExpr + assert isinstance(target, NameExpr), f"Expected NameExpr for target, got {type(target)}" + expr = AssignmentExpr(target, value) + read_loc(data, expr) + expect_end_tag(data) + return expr else: assert False, tag diff --git a/mypy/nodes.py b/mypy/nodes.py index 4134e43477720..6b4306b609255 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5065,6 +5065,7 @@ def local_definitions( FSTRING_EXPR: Final[Tag] = 205 FSTRING_INTERPOLATION: Final[Tag] = 206 LAMBDA_EXPR: Final[Tag] = 207 +NAMED_EXPR: Final[Tag] = 208 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 177a18253a502..d728f7ae2d277 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1511,3 +1511,16 @@ MypyFile:1( NameExpr(x) IntExpr(2))))) NameExpr(items))))) + +[case testNamedExprSimple] +if (x := 1): + pass +[out] +MypyFile:1( + IfStmt:1( + If( + AssignmentExpr:1( + NameExpr(x) + IntExpr(1))) + Then( + PassStmt:2()))) From 7360a999736a9e46b3a76256c8e60ae4ac9c5e5e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 2 Jan 2026 13:27:20 +0000 Subject: [PATCH 069/192] Support star expressions --- mypy/nativeparse.py | 8 ++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 20 ++++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 3543689ba865e..52d1e279dcf10 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -92,6 +92,7 @@ ReturnStmt, SetExpr, SliceExpr, + StarExpr, Statement, StrExpr, TempNode, @@ -946,6 +947,13 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, expr) expect_end_tag(data) return expr + elif tag == nodes.STAR_EXPR: + # Read the wrapped expression + wrapped_expr = read_expression(data) + expr = StarExpr(wrapped_expr) + read_loc(data, expr) + expect_end_tag(data) + return expr else: assert False, tag diff --git a/mypy/nodes.py b/mypy/nodes.py index 6b4306b609255..027dbf47a9b94 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5066,6 +5066,7 @@ def local_definitions( FSTRING_INTERPOLATION: Final[Tag] = 206 LAMBDA_EXPR: Final[Tag] = 207 NAMED_EXPR: Final[Tag] = 208 +STAR_EXPR: Final[Tag] = 209 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index d728f7ae2d277..69b97b1e1f893 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1524,3 +1524,23 @@ MypyFile:1( IntExpr(1))) Then( PassStmt:2()))) + +[case testStarExpression] +*a +*a, b +a, *b +[out] +MypyFile:1( + ExpressionStmt:1( + StarExpr:1( + NameExpr(a))) + ExpressionStmt:2( + TupleExpr:2( + StarExpr:2( + NameExpr(a)) + NameExpr(b))) + ExpressionStmt:3( + TupleExpr:3( + NameExpr(a) + StarExpr:3( + NameExpr(b))))) From dc6eab38fb228ca04a4c59d1ee4d0c0f40086af2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 2 Jan 2026 13:56:36 +0000 Subject: [PATCH 070/192] Support bytes literals --- mypy/nativeparse.py | 8 ++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 27 +++++++++++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 52d1e279dcf10..54977ba7ac2ea 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -54,6 +54,7 @@ AssignmentStmt, Block, BreakStmt, + BytesExpr, CallExpr, ClassDef, ComparisonExpr, @@ -954,6 +955,13 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, expr) expect_end_tag(data) return expr + elif tag == nodes.BYTES_EXPR: + # Read bytes literal as string + value = read_str(data) + expr = BytesExpr(value) + read_loc(data, expr) + expect_end_tag(data) + return expr else: assert False, tag diff --git a/mypy/nodes.py b/mypy/nodes.py index 027dbf47a9b94..73a96d991524c 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5067,6 +5067,7 @@ def local_definitions( LAMBDA_EXPR: Final[Tag] = 207 NAMED_EXPR: Final[Tag] = 208 STAR_EXPR: Final[Tag] = 209 +BYTES_EXPR: Final[Tag] = 210 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 69b97b1e1f893..06c7c4e4f88aa 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1544,3 +1544,30 @@ MypyFile:1( NameExpr(a) StarExpr:3( NameExpr(b))))) + +[case testBytesLiteral] +b"foo" +b"foobar" +b"x\\n\\'" +[out] +MypyFile:1( + ExpressionStmt:1( + BytesExpr(foo)) + ExpressionStmt:2( + BytesExpr(foobar)) + ExpressionStmt:3( + BytesExpr(x\\n\\\'))) + +[case testBytesLiteralWithEscapes] +b'\r\n\t\x2f\xbe' +[out] +MypyFile:1( + ExpressionStmt:1( + BytesExpr(\r\n\t/\xbe))) + +[case testBytesLiteralOctalEscapes] +b'\1\476' +[out] +MypyFile:1( + ExpressionStmt:1( + BytesExpr(\x01>))) From 1ee58dd873ea37660ef5a9223ef6d75d7a0f11f9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 2 Jan 2026 14:11:24 +0000 Subject: [PATCH 071/192] Support 'global' and 'nonlocal' statements --- mypy/nativeparse.py | 22 +++++++++++++++++ mypy/nodes.py | 2 ++ test-data/unit/native-parser.test | 40 +++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 54977ba7ac2ea..ee20211e275ae 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -73,6 +73,7 @@ ForStmt, FuncDef, GeneratorExpr, + GlobalDecl, IfStmt, Import, ImportFrom, @@ -86,6 +87,7 @@ MypyFile, NameExpr, Node, + NonlocalDecl, OpExpr, OperatorAssignmentStmt, PassStmt, @@ -521,6 +523,26 @@ def read_statement(data: ReadBuffer) -> Statement: read_loc(data, stmt) expect_end_tag(data) return stmt + elif tag == nodes.GLOBAL_DECL: + # Read number of names + n = read_int(data) + names = [] + for _ in range(n): + names.append(read_str(data)) + stmt = GlobalDecl(names) + read_loc(data, stmt) + expect_end_tag(data) + return stmt + elif tag == nodes.NONLOCAL_DECL: + # Read number of names + n = read_int(data) + names = [] + for _ in range(n): + names.append(read_str(data)) + stmt = NonlocalDecl(names) + read_loc(data, stmt) + expect_end_tag(data) + return stmt else: assert False, tag diff --git a/mypy/nodes.py b/mypy/nodes.py index 73a96d991524c..3249c95257e9b 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5068,6 +5068,8 @@ def local_definitions( NAMED_EXPR: Final[Tag] = 208 STAR_EXPR: Final[Tag] = 209 BYTES_EXPR: Final[Tag] = 210 +GLOBAL_DECL: Final[Tag] = 211 +NONLOCAL_DECL: Final[Tag] = 212 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 06c7c4e4f88aa..3eeb0b51300e0 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1571,3 +1571,43 @@ b'\1\476' MypyFile:1( ExpressionStmt:1( BytesExpr(\x01>))) +[case testGlobalAndNonlocal] +x = 1 +y = 2 +def outer(): + global x + def inner(): + nonlocal y + global z + y = 3 + z = 4 + x = 5 +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + IntExpr(1)) + AssignmentStmt:2( + NameExpr(y) + IntExpr(2)) + FuncDef:3( + outer + Block:4( + GlobalDecl:4( + x) + FuncDef:5( + inner + Block:6( + NonlocalDecl:6( + y) + GlobalDecl:7( + z) + AssignmentStmt:8( + NameExpr(y) + IntExpr(3)) + AssignmentStmt:9( + NameExpr(z) + IntExpr(4)))) + AssignmentStmt:10( + NameExpr(x) + IntExpr(5))))) From 3ceb623a69c01dc6052e348367ba2abed90ba481 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 2 Jan 2026 14:20:42 +0000 Subject: [PATCH 072/192] Support await expressions --- mypy/nativeparse.py | 8 ++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 22 ++++++++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index ee20211e275ae..e8515b8095918 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -51,6 +51,7 @@ Argument, AssertStmt, AssignmentExpr, + AwaitExpr, AssignmentStmt, Block, BreakStmt, @@ -984,6 +985,13 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, expr) expect_end_tag(data) return expr + elif tag == nodes.AWAIT_EXPR: + # Read awaited expression + value = read_expression(data) + expr = AwaitExpr(value) + read_loc(data, expr) + expect_end_tag(data) + return expr else: assert False, tag diff --git a/mypy/nodes.py b/mypy/nodes.py index 3249c95257e9b..466f05b9e1521 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5070,6 +5070,7 @@ def local_definitions( BYTES_EXPR: Final[Tag] = 210 GLOBAL_DECL: Final[Tag] = 211 NONLOCAL_DECL: Final[Tag] = 212 +AWAIT_EXPR: Final[Tag] = 213 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 3eeb0b51300e0..03f2bce9a5d9c 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1611,3 +1611,25 @@ MypyFile:1( AssignmentStmt:10( NameExpr(x) IntExpr(5))))) +[case testAwaitExpression] +async def foo(): + x = await bar() + return await baz(x) +[out] +MypyFile:1( + FuncDef:1( + foo + Async + Block:2( + AssignmentStmt:2( + NameExpr(x) + AwaitExpr:2( + CallExpr:2( + NameExpr(bar) + Args()))) + ReturnStmt:3( + AwaitExpr:3( + CallExpr:3( + NameExpr(baz) + Args( + NameExpr(x)))))))) From a4ab43cab6442dd34861a9d7f00f27ba9baf1c39 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 2 Jan 2026 14:37:09 +0000 Subject: [PATCH 073/192] Support bool literal types --- mypy/nativeparse.py | 13 ++++++++++++- mypy/types.py | 1 + test-data/unit/native-parser.test | 10 ++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index e8515b8095918..d9344dc38f583 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -110,7 +110,7 @@ YieldFromExpr, MISSING_FALLBACK, ) -from mypy.types import CallableType, UnboundType, NoneType, UnionType, AnyType, TypeOfAny, Instance, Type, TypeList, EllipsisType +from mypy.types import CallableType, UnboundType, NoneType, UnionType, AnyType, TypeOfAny, Instance, Type, TypeList, EllipsisType, RawExpressionType # There is no way to create reasonable fallbacks at this stage, @@ -587,6 +587,17 @@ def read_type(data: ReadBuffer) -> Type: read_loc(data, ellipsis_type) expect_end_tag(data) return ellipsis_type + elif tag == types.RAW_EXPRESSION_TYPE: + type_name = read_str(data) + value: types.LiteralValue + if type_name == "builtins.bool": + value = read_bool(data) + else: + assert False # TODO + raw_type = RawExpressionType(value, type_name) + read_loc(data, raw_type) + expect_end_tag(data) + return raw_type else: assert False, tag diff --git a/mypy/types.py b/mypy/types.py index 326ec66212ddf..a865879ca5529 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -4322,6 +4322,7 @@ def type_vars_as_args(type_vars: Sequence[TypeVarLikeType]) -> tuple[Type, ...]: PARAMETERS: Final[Tag] = 117 LIST_TYPE: Final[Tag] = 118 # Only valid in serialized ASTs ELLIPSIS_TYPE: Final[Tag] = 119 # Only valid in serialized ASTs +RAW_EXPRESSION_TYPE: Final[Tag] = 120 # Only valid in serialized ASTs def read_type(data: ReadBuffer, tag: Tag | None = None) -> Type: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 03f2bce9a5d9c..bf86cb63eefce 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1286,6 +1286,16 @@ MypyFile:1( Block:2( PassStmt:2()))) +[case testLiteralTypes] +x: Literal[True, False] +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + TempNode:1( + Any) + Literal?[True, False])) + [case testDelStmt] del x del x[0], y[1] From b5d73eba16d4b41d8c2c0b5c9bd3bff6443896a0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 2 Jan 2026 15:35:38 +0000 Subject: [PATCH 074/192] Handle syntax errors --- mypy/nativeparse.py | 10 +++++----- mypy/test/test_nativeparse.py | 9 ++++++++- test-data/unit/native-parser.test | 15 +++++++++++++++ 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index d9344dc38f583..14a91485d0639 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -18,7 +18,7 @@ from __future__ import annotations -from typing import Final, cast +from typing import Final, cast, Any import ast_serialize # type: ignore[import-untyped] @@ -126,8 +126,8 @@ def expect_tag(data: ReadBuffer, tag: Tag) -> None: assert read_tag(data) == tag -def native_parse(filename: str) -> MypyFile: - b = parse_to_binary_ast(filename) +def native_parse(filename: str) -> tuple[MypyFile, list[dict[str, Any]]]: + b, errors = parse_to_binary_ast(filename) data = ReadBuffer(b) n = read_int(data) defs = [] @@ -135,10 +135,10 @@ def native_parse(filename: str) -> MypyFile: defs.append(read_statement(data)) node = MypyFile(defs, []) node.path = filename - return node + return node, errors -def parse_to_binary_ast(filename: str) -> bytes: +def parse_to_binary_ast(filename: str) -> tuple[bytes, list[dict[str, Any]]]: return ast_serialize.parse(filename) # type: ignore[no-any-return] diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index b6e0a76b35d30..a50c6ad45cab9 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -9,6 +9,8 @@ import unittest from collections.abc import Iterator +from typing import Any + from mypy import defaults, nodes from mypy.cache import END_TAG, LIST_GEN, LITERAL_INT, LITERAL_STR, LOCATION from mypy.config_parser import parse_mypy_comments @@ -61,12 +63,13 @@ def test_parser(testcase: DataDrivenTestCase) -> None: try: with temp_source(source) as fnam: try: - node = native_parse(fnam) + node, errors = native_parse(fnam) except ValueError as e: print(f"Parse failed: {e}") assert False node.path = "main" a = node.str_with_options(options).split("\n") + a = [format_error(err) for err in errors] + a except CompileError as e: a = e.messages assert_string_arrays_equal( @@ -74,6 +77,10 @@ def test_parser(testcase: DataDrivenTestCase) -> None: ) +def format_error(err: dict[str, Any]) -> str: + return f"{err['line']}:{err['column']}: error: {err['message']}" + + class TestNativeParse(unittest.TestCase): def test_trivial_binary_data(self) -> None: def int_enc(n: int) -> int: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index bf86cb63eefce..41942eab9ccbd 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1643,3 +1643,18 @@ MypyFile:1( NameExpr(baz) Args( NameExpr(x)))))))) + +[case testSyntaxError] +x = [1 2] +y = 2 +[out] +1:8: error: Expected ',', found int +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + ListExpr:1( + IntExpr(1) + IntExpr(2))) + AssignmentStmt:2( + NameExpr(y) + IntExpr(2))) From 34b76f4d424cc159abadff09e53bb1d4fe030ab7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 3 Jan 2026 11:13:30 +0000 Subject: [PATCH 075/192] Refactor: split statement parsing in read_statement --- mypy/nativeparse.py | 320 +++++++++++++++++++++++--------------------- 1 file changed, 165 insertions(+), 155 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 14a91485d0639..1a2d65c749e33 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -146,79 +146,7 @@ def read_statement(data: ReadBuffer) -> Statement: tag = read_tag(data) stmt: Statement if tag == nodes.FUNC_DEF_STMT: - # Function name - name = read_str(data) - - # Arguments - expect_tag(data, LIST_GEN) - n_args = read_int_bare(data) - arguments = [] - has_ann = False - for _ in range(n_args): - arg_name = read_str(data) - arg_kind_int = read_int(data) - # Convert integer to ArgKind enum using ARG_KINDS tuple - arg_kind = ARG_KINDS[arg_kind_int] - # TODO: Read type annotation when implemented - has_type = read_bool(data) - if has_type: - ann = read_type(data) - has_ann = True - else: - ann = None - # Read default value - has_default = read_bool(data) - if has_default: - default = read_expression(data) - else: - default = None - pos_only = read_bool(data) - - var = Var(arg_name) - arg = Argument(var, ann, default, arg_kind, pos_only) - arguments.append(arg) - - # Body - body = read_block(data) - - # Decorators - expect_tag(data, LIST_GEN) - n_decorators = read_int_bare(data) - assert n_decorators == 0, "Decorators not yet supported" - - # is_async - is_async = read_bool(data) - - # TODO: type_params - has_type_params = read_bool(data) - assert not has_type_params, "Type params not yet supported" - - # TODO: Return type annotation - has_return_type = read_bool(data) - if has_return_type: - return_type = read_type(data) - has_ann = True - else: - return_type = None - - if has_ann: - typ = CallableType( - [arg.type_annotation if arg.type_annotation else AnyType(TypeOfAny.unannotated) - for arg in arguments], - [arg.kind for arg in arguments], - [arg.variable.name for arg in arguments], - return_type if return_type else AnyType(TypeOfAny.unannotated), - _dummy_fallback - ) - else: - typ = None - - func_def = FuncDef(name, arguments, body, typ=typ) - if is_async: - func_def.is_coroutine = True - read_loc(data, func_def) - expect_end_tag(data) - return func_def + return read_func_def(data) elif tag == nodes.DECORATOR: expect_tag(data, LIST_GEN) n_decorators = read_int_bare(data) @@ -434,89 +362,9 @@ def read_statement(data: ReadBuffer) -> Statement: expect_end_tag(data) return stmt elif tag == nodes.CLASS_DEF: - # Class name - name = read_str(data) - - # Body - body = read_block(data) - - # Base classes - base_type_exprs = read_expression_list(data) - - # TODO: Decorators (skip for now) - expect_tag(data, LIST_GEN) - n_decorators = read_int_bare(data) - assert n_decorators == 0, "Decorators not yet supported" - - # TODO: Type parameters (skip for now) - has_type_params = read_bool(data) - assert not has_type_params, "Type parameters not yet supported" - - # TODO: Metaclass (skip for now) - has_metaclass = read_bool(data) - assert not has_metaclass, "Metaclass not yet supported" - - # TODO: Keywords (skip for now) - expect_tag(data, DICT_STR_GEN) - n_keywords = read_int_bare(data) - assert n_keywords == 0, "Keywords not yet supported" - - class_def = ClassDef(name, body, base_type_exprs=base_type_exprs if base_type_exprs else None) - read_loc(data, class_def) - expect_end_tag(data) - return class_def + return read_class_def(data) elif tag == nodes.TRY_STMT: - # Read try body - body = read_block(data) - - # Read number of except handlers - num_handlers = read_int(data) - - # Read exception types for each handler - types_list = [] - for _ in range(num_handlers): - has_type = read_bool(data) - if has_type: - exc_type = read_expression(data) - types_list.append(exc_type) - else: - types_list.append(None) - - # Read variable names for each handler - vars_list = [] - for _ in range(num_handlers): - has_name = read_bool(data) - if has_name: - var_name = read_str(data) - var_expr = NameExpr(var_name) - vars_list.append(var_expr) - else: - vars_list.append(None) - - # Read handler bodies - handlers = [] - for _ in range(num_handlers): - handler_body = read_block(data) - handlers.append(handler_body) - - # Read else body (optional) - has_else = read_bool(data) - if has_else: - else_body = read_block(data) - else: - else_body = None - - # Read finally body (optional) - has_finally = read_bool(data) - if has_finally: - finally_body = read_block(data) - else: - finally_body = None - - stmt = TryStmt(body, vars_list, types_list, handlers, else_body, finally_body) - read_loc(data, stmt) - expect_end_tag(data) - return stmt + return read_try_stmt(data) elif tag == nodes.DEL_STMT: # Read the target expression expr = read_expression(data) @@ -548,6 +396,168 @@ def read_statement(data: ReadBuffer) -> Statement: assert False, tag +def read_func_def(data: ReadBuffer) -> FuncDef: + # Function name + name = read_str(data) + + # Arguments + expect_tag(data, LIST_GEN) + n_args = read_int_bare(data) + arguments = [] + has_ann = False + for _ in range(n_args): + arg_name = read_str(data) + arg_kind_int = read_int(data) + # Convert integer to ArgKind enum using ARG_KINDS tuple + arg_kind = ARG_KINDS[arg_kind_int] + # TODO: Read type annotation when implemented + has_type = read_bool(data) + if has_type: + ann = read_type(data) + has_ann = True + else: + ann = None + # Read default value + has_default = read_bool(data) + if has_default: + default = read_expression(data) + else: + default = None + pos_only = read_bool(data) + + var = Var(arg_name) + arg = Argument(var, ann, default, arg_kind, pos_only) + arguments.append(arg) + + body = read_block(data) + + # Decorators + expect_tag(data, LIST_GEN) + n_decorators = read_int_bare(data) + assert n_decorators == 0, "Decorators not yet supported" + + is_async = read_bool(data) + + # TODO: type_params + has_type_params = read_bool(data) + assert not has_type_params, "Type params not yet supported" + + # TODO: Return type annotation + has_return_type = read_bool(data) + if has_return_type: + return_type = read_type(data) + has_ann = True + else: + return_type = None + + if has_ann: + typ = CallableType( + [arg.type_annotation if arg.type_annotation else AnyType(TypeOfAny.unannotated) + for arg in arguments], + [arg.kind for arg in arguments], + [arg.variable.name for arg in arguments], + return_type if return_type else AnyType(TypeOfAny.unannotated), + _dummy_fallback + ) + else: + typ = None + + func_def = FuncDef(name, arguments, body, typ=typ) + if is_async: + func_def.is_coroutine = True + read_loc(data, func_def) + expect_end_tag(data) + return func_def + + +def read_class_def(data: ReadBuffer) -> ClassDef: + # Class name + name = read_str(data) + + # Body + body = read_block(data) + + # Base classes + base_type_exprs = read_expression_list(data) + + # TODO: Decorators (skip for now) + expect_tag(data, LIST_GEN) + n_decorators = read_int_bare(data) + assert n_decorators == 0, "Decorators not yet supported" + + # TODO: Type parameters (skip for now) + has_type_params = read_bool(data) + assert not has_type_params, "Type parameters not yet supported" + + # TODO: Metaclass (skip for now) + has_metaclass = read_bool(data) + assert not has_metaclass, "Metaclass not yet supported" + + # TODO: Keywords (skip for now) + expect_tag(data, DICT_STR_GEN) + n_keywords = read_int_bare(data) + assert n_keywords == 0, "Keywords not yet supported" + + class_def = ClassDef(name, body, base_type_exprs=base_type_exprs if base_type_exprs else None) + read_loc(data, class_def) + expect_end_tag(data) + return class_def + + +def read_try_stmt(data: ReadBuffer) -> TryStmt: + # Read try body + body = read_block(data) + + # Read number of except handlers + num_handlers = read_int(data) + + # Read exception types for each handler + types_list = [] + for _ in range(num_handlers): + has_type = read_bool(data) + if has_type: + exc_type = read_expression(data) + types_list.append(exc_type) + else: + types_list.append(None) + + # Read variable names for each handler + vars_list = [] + for _ in range(num_handlers): + has_name = read_bool(data) + if has_name: + var_name = read_str(data) + var_expr = NameExpr(var_name) + vars_list.append(var_expr) + else: + vars_list.append(None) + + # Read handler bodies + handlers = [] + for _ in range(num_handlers): + handler_body = read_block(data) + handlers.append(handler_body) + + # Read else body (optional) + has_else = read_bool(data) + if has_else: + else_body = read_block(data) + else: + else_body = None + + # Read finally body (optional) + has_finally = read_bool(data) + if has_finally: + finally_body = read_block(data) + else: + finally_body = None + + stmt = TryStmt(body, vars_list, types_list, handlers, else_body, finally_body) + read_loc(data, stmt) + expect_end_tag(data) + return stmt + + def read_type(data: ReadBuffer) -> Type: tag = read_tag(data) if tag == types.UNBOUND_TYPE: From a510e5abed2c0157c71a84287518fdf2b96dab40 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 3 Jan 2026 11:37:11 +0000 Subject: [PATCH 076/192] Support overloaded functions (no tests yet) --- mypy/nativeparse.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 1a2d65c749e33..cdb3b826f6c04 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -91,6 +91,7 @@ NonlocalDecl, OpExpr, OperatorAssignmentStmt, + OverloadedFuncDef, PassStmt, RaiseStmt, ReturnStmt, @@ -130,14 +131,36 @@ def native_parse(filename: str) -> tuple[MypyFile, list[dict[str, Any]]]: b, errors = parse_to_binary_ast(filename) data = ReadBuffer(b) n = read_int(data) - defs = [] - for i in range(n): - defs.append(read_statement(data)) + defs = read_statements(data, n) node = MypyFile(defs, []) node.path = filename return node, errors +def read_statements(data: ReadBuffer, n: int) -> list[Statement]: + defs = [] + prev_func = False + prev_name = "" + for _ in range(n): + stmt = read_statement(data) + if isinstance(stmt, (FuncDef, Decorator)): + if prev_func and stmt.name == prev_name: + # Merge into overloaded function definition + prev = defs[-1] + if isinstance(prev, OverloadedFuncDef): + prev.items.append(stmt) + else: + defs[-1] = OverloadedFuncDef([prev, stmt]) + else: + defs.append(stmt) + prev_name = stmt.name + prev_func = True + else: + defs.append(stmt) + prev_func = False + return defs + + def parse_to_binary_ast(filename: str) -> tuple[bytes, list[dict[str, Any]]]: return ast_serialize.parse(filename) # type: ignore[no-any-return] @@ -617,7 +640,7 @@ def read_block(data: ReadBuffer) -> Block: expect_tag(data, LIST_GEN) n = read_int_bare(data) assert n > 0 - a = [read_statement(data) for i in range(n)] + a = read_statements(data, n) expect_end_tag(data) b = Block(a) b.line = a[0].line From b9a63f65f9598172f697ba911785e27fe5f9e6be Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 3 Jan 2026 11:49:14 +0000 Subject: [PATCH 077/192] Simplify --- mypy/nativeparse.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index cdb3b826f6c04..509499674c4ee 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -454,11 +454,6 @@ def read_func_def(data: ReadBuffer) -> FuncDef: body = read_block(data) - # Decorators - expect_tag(data, LIST_GEN) - n_decorators = read_int_bare(data) - assert n_decorators == 0, "Decorators not yet supported" - is_async = read_bool(data) # TODO: type_params From acaee8cc4a8b6e3d3f6dc35553c0a6095bc899e0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 3 Jan 2026 12:00:50 +0000 Subject: [PATCH 078/192] Deserialize function parameter locations --- mypy/nativeparse.py | 12 +++++- test-data/unit/native-parser.test | 66 +++++++++++++++---------------- 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 509499674c4ee..eaa03e486617b 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -423,7 +423,7 @@ def read_func_def(data: ReadBuffer) -> FuncDef: # Function name name = read_str(data) - # Arguments + # Parameters expect_tag(data, LIST_GEN) n_args = read_int_bare(data) arguments = [] @@ -450,6 +450,11 @@ def read_func_def(data: ReadBuffer) -> FuncDef: var = Var(arg_name) arg = Argument(var, ann, default, arg_kind, pos_only) + read_loc(data, arg) + var.line = arg.line + var.column = arg.column + var.end_line = arg.end_line + var.end_column = arg.end_column arguments.append(arg) body = read_block(data) @@ -973,6 +978,11 @@ def read_expression(data: ReadBuffer) -> Expression: var = Var(arg_name) arg = Argument(var, ann, default, arg_kind, pos_only) + read_loc(data, arg) + var.line = arg.line + var.column = arg.column + var.end_line = arg.end_line + var.end_column = arg.end_column arguments.append(arg) # Read body block diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 41942eab9ccbd..120edd651ba68 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -321,8 +321,8 @@ MypyFile:1( FuncDef:1( add Args( - Var:nil(x) - Var:nil(y)) + Var(x) + Var(y)) Block:2( ReturnStmt:2( OpExpr:2( @@ -338,7 +338,7 @@ MypyFile:1( FuncDef:1( foo VarArg( - Var:nil(args)) + Var(args)) Block:2( PassStmt:2()))) @@ -350,7 +350,7 @@ MypyFile:1( FuncDef:1( bar DictVarArg( - Var:nil(kwargs)) + Var(kwargs)) Block:2( PassStmt:2()))) @@ -363,8 +363,8 @@ MypyFile:1( baz MaxPos(1) Args( - Var:nil(x) - Var:nil(y)) + Var(x) + Var(y)) Block:2( PassStmt:2()))) @@ -377,13 +377,13 @@ MypyFile:1( complex_fn MaxPos(2) Args( - Var:nil(a) - Var:nil(b) - Var:nil(c)) + Var(a) + Var(b) + Var(c)) VarArg( - Var:nil(args)) + Var(args)) DictVarArg( - Var:nil(kwargs)) + Var(kwargs)) Block:2( PassStmt:2()))) @@ -407,7 +407,7 @@ MypyFile:1( greet Args( default( - Var:nil(name) + Var(name) StrExpr(World))) Block:2( PassStmt:2()))) @@ -421,10 +421,10 @@ MypyFile:1( add Args( default( - Var:nil(x) + Var(x) IntExpr(1)) default( - Var:nil(y) + Var(y) IntExpr(2))) Block:2( ReturnStmt:2( @@ -438,12 +438,12 @@ MypyFile:1( FuncDef:1( func Args( - Var:nil(a) + Var(a) default( - Var:nil(b) + Var(b) IntExpr(5)) default( - Var:nil(c) + Var(c) IntExpr(10))) Block:2( PassStmt:2()))) @@ -458,7 +458,7 @@ MypyFile:1( MaxPos(0) Args( default( - Var:nil(x) + Var(x) IntExpr(42))) Block:2( PassStmt:2()))) @@ -564,7 +564,7 @@ MypyFile:1( FuncDef:2( bar Args( - Var:nil(self)) + Var(self)) Block:3( ReturnStmt:3( IntExpr(1)))))) @@ -738,9 +738,9 @@ MypyFile:1( FuncDef:5( h Args( - Var:nil(x) - Var:nil(y) - Var:nil(z)) + Var(x) + Var(y) + Var(z)) def (x: foo.bar?, y: Any, z: x.y.z?) -> Any Block:6( PassStmt:6()))) @@ -753,7 +753,7 @@ MypyFile:1( FuncDef:1( f Args( - Var:nil(x)) + Var(x)) def (x: a?[b?]) -> c.d?[e.f.g?[h?], i?] Block:2( PassStmt:2()))) @@ -766,7 +766,7 @@ MypyFile:1( FuncDef:1( f Args( - Var:nil(x)) + Var(x)) def (x: int? | str?) -> bool? | None? Block:2( PassStmt:2()))) @@ -1241,7 +1241,7 @@ MypyFile:1( FuncDef:1( f Args( - Var:nil(x)) + Var(x)) def (x: Callable?[, None?]) -> None? Block:2( PassStmt:2()))) @@ -1254,7 +1254,7 @@ MypyFile:1( FuncDef:1( g Args( - Var:nil(callback)) + Var(callback)) def (callback: Callable?[, bool?]) -> int? Block:2( ReturnStmt:2( @@ -1268,7 +1268,7 @@ MypyFile:1( FuncDef:1( h Args( - Var:nil(f)) + Var(f)) def (f: Callable?[, Tuple?[int?, str?]]) -> None? Block:2( PassStmt:2()))) @@ -1281,7 +1281,7 @@ MypyFile:1( FuncDef:1( f Args( - Var:nil(callback)) + Var(callback)) def (callback: Callable?[..., int?]) -> None? Block:2( PassStmt:2()))) @@ -1465,7 +1465,7 @@ MypyFile:1( ExpressionStmt:1( LambdaExpr:1( Args( - Var:nil(x)) + Var(x)) Block:1( ReturnStmt:1( OpExpr:1( @@ -1480,8 +1480,8 @@ MypyFile:1( ExpressionStmt:1( LambdaExpr:1( Args( - Var:nil(x) - Var:nil(y)) + Var(x) + Var(y)) Block:1( ReturnStmt:1( OpExpr:1( @@ -1497,7 +1497,7 @@ MypyFile:1( LambdaExpr:1( Args( default( - Var:nil(x) + Var(x) IntExpr(2))) Block:1( ReturnStmt:1( @@ -1513,7 +1513,7 @@ MypyFile:1( Args( LambdaExpr:1( Args( - Var:nil(x)) + Var(x)) Block:1( ReturnStmt:1( OpExpr:1( From 8a7c78bf9005df5c6d435ea2ca5f597df19c5b47 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 3 Jan 2026 12:04:21 +0000 Subject: [PATCH 079/192] Refactor function parameter deserialization --- mypy/nativeparse.py | 55 +++++++++++++++------------------------------ 1 file changed, 18 insertions(+), 37 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index eaa03e486617b..bb510d28a5103 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -419,11 +419,12 @@ def read_statement(data: ReadBuffer) -> Statement: assert False, tag -def read_func_def(data: ReadBuffer) -> FuncDef: - # Function name - name = read_str(data) +def read_parameters(data: ReadBuffer) -> tuple[list[Argument], bool]: + """Read function/lambda parameters from the buffer. - # Parameters + Returns: + A tuple of (arguments list, has_annotations flag) + """ expect_tag(data, LIST_GEN) n_args = read_int_bare(data) arguments = [] @@ -433,7 +434,7 @@ def read_func_def(data: ReadBuffer) -> FuncDef: arg_kind_int = read_int(data) # Convert integer to ArgKind enum using ARG_KINDS tuple arg_kind = ARG_KINDS[arg_kind_int] - # TODO: Read type annotation when implemented + # Read type annotation has_type = read_bool(data) if has_type: ann = read_type(data) @@ -457,6 +458,16 @@ def read_func_def(data: ReadBuffer) -> FuncDef: var.end_column = arg.end_column arguments.append(arg) + return arguments, has_ann + + +def read_func_def(data: ReadBuffer) -> FuncDef: + # Function name + name = read_str(data) + + # Parameters + arguments, has_ann = read_parameters(data) + body = read_block(data) is_async = read_bool(data) @@ -952,38 +963,8 @@ def read_expression(data: ReadBuffer) -> Expression: expect_end_tag(data) return expr elif tag == nodes.LAMBDA_EXPR: - # Read arguments - expect_tag(data, LIST_GEN) - n_args = read_int_bare(data) - arguments = [] - has_ann = False - for _ in range(n_args): - arg_name = read_str(data) - arg_kind_int = read_int(data) - arg_kind = ARG_KINDS[arg_kind_int] - # Read type annotation - has_type = read_bool(data) - if has_type: - ann = read_type(data) - has_ann = True - else: - ann = None - # Read default value - has_default = read_bool(data) - if has_default: - default = read_expression(data) - else: - default = None - pos_only = read_bool(data) - - var = Var(arg_name) - arg = Argument(var, ann, default, arg_kind, pos_only) - read_loc(data, arg) - var.line = arg.line - var.column = arg.column - var.end_line = arg.end_line - var.end_column = arg.end_column - arguments.append(arg) + # Read parameters + arguments, has_ann = read_parameters(data) # Read body block body = read_block(data) From 8aa6bf0e560a876bf4326aa2e065b076c652ebca Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 3 Jan 2026 13:03:08 +0000 Subject: [PATCH 080/192] Improve decorators --- mypy/nativeparse.py | 2 ++ test-data/unit/native-parser.test | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index bb510d28a5103..88434b7a44dc8 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -177,6 +177,8 @@ def read_statement(data: ReadBuffer) -> Statement: fdef = read_statement(data) assert isinstance(fdef, FuncDef) var = Var(fdef.name) + var.line = fdef.line + var.is_ready = False # Create Decorator wrapping the FuncDef stmt = Decorator(fdef, decorators, var) stmt.line = fdef.line diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 120edd651ba68..c87579986dcfe 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1112,7 +1112,7 @@ class C: [out] MypyFile:1( Decorator:1( - Var:nil(foo) + Var(foo) NameExpr(dec) FuncDef:1( foo @@ -1121,7 +1121,7 @@ MypyFile:1( ClassDef:4( C Decorator:5( - Var:nil(foo) + Var(foo) CallExpr:5( NameExpr(dec) Args()) From ac18a2bc6baddeea29da9db55193e7b075db7e25 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 3 Jan 2026 13:21:48 +0000 Subject: [PATCH 081/192] Fix decorator locations and add overload test cases --- mypy/nativeparse.py | 6 ++- test-data/unit/native-parser.test | 67 ++++++++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 88434b7a44dc8..d9c179f192084 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -174,6 +174,8 @@ def read_statement(data: ReadBuffer) -> Statement: expect_tag(data, LIST_GEN) n_decorators = read_int_bare(data) decorators = [read_expression(data) for i in range(n_decorators)] + line = read_int(data) + column = read_int(data) fdef = read_statement(data) assert isinstance(fdef, FuncDef) var = Var(fdef.name) @@ -181,8 +183,8 @@ def read_statement(data: ReadBuffer) -> Statement: var.is_ready = False # Create Decorator wrapping the FuncDef stmt = Decorator(fdef, decorators, var) - stmt.line = fdef.line - stmt.column = fdef.column + stmt.line = line + stmt.column = column stmt.end_line = fdef.end_line stmt.end_column = fdef.end_column # TODO: Adjust funcdef location to start after decorator? diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index c87579986dcfe..dc03564eb88f1 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1114,7 +1114,7 @@ MypyFile:1( Decorator:1( Var(foo) NameExpr(dec) - FuncDef:1( + FuncDef:2( foo Block:3( PassStmt:3()))) @@ -1126,7 +1126,7 @@ MypyFile:1( NameExpr(dec) Args()) NameExpr(staticmethod) - FuncDef:5( + FuncDef:7( foo Block:8( PassStmt:8()))))) @@ -1658,3 +1658,66 @@ MypyFile:1( AssignmentStmt:2( NameExpr(y) IntExpr(2))) + +[case testFunctionOverload] +@overload +def f() -> x: pass +@overload +def f() -> y: pass +[out] +MypyFile:1( + OverloadedFuncDef:1( + Decorator:1( + Var(f) + NameExpr(overload) + FuncDef:2( + f + def () -> x? + Block:2( + PassStmt:2()))) + Decorator:3( + Var(f) + NameExpr(overload) + FuncDef:4( + f + def () -> y? + Block:4( + PassStmt:4()))))) + +[case testFunctionOverloadWithImplementation] +@overload +def f(a: x) -> x: ... +@overload +def f(a: y) -> y: ... +def f(a): pass +[out] +MypyFile:1( + OverloadedFuncDef:1( + Decorator:1( + Var(f) + NameExpr(overload) + FuncDef:2( + f + Args( + Var(a)) + def (a: x?) -> x? + Block:2( + ExpressionStmt:2( + Ellipsis)))) + Decorator:3( + Var(f) + NameExpr(overload) + FuncDef:4( + f + Args( + Var(a)) + def (a: y?) -> y? + Block:4( + ExpressionStmt:4( + Ellipsis)))) + FuncDef:5( + f + Args( + Var(a)) + Block:5( + PassStmt:5())))) From 9445336e7e6733d18e4df867aa5e2eb3883f0591 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 3 Jan 2026 14:26:52 +0000 Subject: [PATCH 082/192] Basic support for type ignore comments --- mypy/nativeparse.py | 11 +++++++---- mypy/test/test_nativeparse.py | 11 ++++++++++- test-data/unit/native-parser.test | 30 ++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index d9c179f192084..bcb06f27147b4 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -114,6 +114,9 @@ from mypy.types import CallableType, UnboundType, NoneType, UnionType, AnyType, TypeOfAny, Instance, Type, TypeList, EllipsisType, RawExpressionType +TypeIgnores = list[tuple[int, list[str]]] + + # There is no way to create reasonable fallbacks at this stage, # they must be patched later. _dummy_fallback: Final = Instance(MISSING_FALLBACK, [], -1) @@ -127,14 +130,14 @@ def expect_tag(data: ReadBuffer, tag: Tag) -> None: assert read_tag(data) == tag -def native_parse(filename: str) -> tuple[MypyFile, list[dict[str, Any]]]: - b, errors = parse_to_binary_ast(filename) +def native_parse(filename: str) -> tuple[MypyFile, list[dict[str, Any]], TypeIgnores]: + b, errors, ignores = parse_to_binary_ast(filename) data = ReadBuffer(b) n = read_int(data) defs = read_statements(data, n) node = MypyFile(defs, []) node.path = filename - return node, errors + return node, errors, ignores def read_statements(data: ReadBuffer, n: int) -> list[Statement]: @@ -161,7 +164,7 @@ def read_statements(data: ReadBuffer, n: int) -> list[Statement]: return defs -def parse_to_binary_ast(filename: str) -> tuple[bytes, list[dict[str, Any]]]: +def parse_to_binary_ast(filename: str) -> tuple[bytes, list[dict[str, Any]], TypeIgnores]: return ast_serialize.parse(filename) # type: ignore[no-any-return] diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index a50c6ad45cab9..fe0083e58995c 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -63,13 +63,14 @@ def test_parser(testcase: DataDrivenTestCase) -> None: try: with temp_source(source) as fnam: try: - node, errors = native_parse(fnam) + node, errors, type_ignores = native_parse(fnam) except ValueError as e: print(f"Parse failed: {e}") assert False node.path = "main" a = node.str_with_options(options).split("\n") a = [format_error(err) for err in errors] + a + a = [format_ignore(ignore) for ignore in type_ignores] + a except CompileError as e: a = e.messages assert_string_arrays_equal( @@ -81,6 +82,14 @@ def format_error(err: dict[str, Any]) -> str: return f"{err['line']}:{err['column']}: error: {err['message']}" +def format_ignore(ignore: tuple[int, list[str]]) -> str: + line, codes = ignore + if not codes: + return f"ignore: {line}" + else: + return f"ignore: {line} [{', '.join(codes)}]" + + class TestNativeParse(unittest.TestCase): def test_trivial_binary_data(self) -> None: def int_enc(n: int) -> int: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index dc03564eb88f1..b6873109eda93 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1721,3 +1721,33 @@ MypyFile:1( Var(a)) Block:5( PassStmt:5())))) + +[case testTypeIgnores] +x = 1 # type: ignore +y = ( + 2 # type: ignore # Comment +) +y = 1 # foo: ignore +z = 1 #type: ignore[x] +zz = 1 #type: ignore [ foo, arg-type ] +[out] +ignore: 1 +ignore: 3 +ignore: 6 [x] +ignore: 7 [foo, arg-type] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + IntExpr(1)) + AssignmentStmt:2( + NameExpr(y) + IntExpr(2)) + AssignmentStmt:5( + NameExpr(y) + IntExpr(1)) + AssignmentStmt:6( + NameExpr(z) + IntExpr(1)) + AssignmentStmt:7( + NameExpr(zz) + IntExpr(1))) From e2b3b4a8fcb0965fcfd2f75d9f2cc4db51010976 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 3 Jan 2026 14:53:46 +0000 Subject: [PATCH 083/192] Fix self check --- mypy/nativeparse.py | 27 ++++++++++++++------------- mypy/test/test_nativeparse.py | 6 +++--- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index bcb06f27147b4..7985056812a74 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -141,7 +141,7 @@ def native_parse(filename: str) -> tuple[MypyFile, list[dict[str, Any]], TypeIgn def read_statements(data: ReadBuffer, n: int) -> list[Statement]: - defs = [] + defs: list[Statement] = [] prev_func = False prev_name = "" for _ in range(n): @@ -153,6 +153,7 @@ def read_statements(data: ReadBuffer, n: int) -> list[Statement]: if isinstance(prev, OverloadedFuncDef): prev.items.append(stmt) else: + assert isinstance(prev, (FuncDef, Decorator)) defs[-1] = OverloadedFuncDef([prev, stmt]) else: defs.append(stmt) @@ -405,20 +406,20 @@ def read_statement(data: ReadBuffer) -> Statement: elif tag == nodes.GLOBAL_DECL: # Read number of names n = read_int(data) - names = [] + decl_names = [] for _ in range(n): - names.append(read_str(data)) - stmt = GlobalDecl(names) + decl_names.append(read_str(data)) + stmt = GlobalDecl(decl_names) read_loc(data, stmt) expect_end_tag(data) return stmt elif tag == nodes.NONLOCAL_DECL: # Read number of names n = read_int(data) - names = [] + decl_names = [] for _ in range(n): - names.append(read_str(data)) - stmt = NonlocalDecl(names) + decl_names.append(read_str(data)) + stmt = NonlocalDecl(decl_names) read_loc(data, stmt) expect_end_tag(data) return stmt @@ -553,7 +554,7 @@ def read_try_stmt(data: ReadBuffer) -> TryStmt: num_handlers = read_int(data) # Read exception types for each handler - types_list = [] + types_list: list[Expression | None] = [] for _ in range(num_handlers): has_type = read_bool(data) if has_type: @@ -563,7 +564,7 @@ def read_try_stmt(data: ReadBuffer) -> TryStmt: types_list.append(None) # Read variable names for each handler - vars_list = [] + vars_list: list[NameExpr | None] = [] for _ in range(num_handlers): has_name = read_bool(data) if has_name: @@ -955,18 +956,18 @@ def read_expression(data: ReadBuffer) -> Expression: # F-strings are converted into nodes representing "".join([...]), to match # pre-existing behavior. nparts = read_int(data) - items = [] + fitems = [] for _ in range(nparts): b = read_bool(data) if b: n = read_int(data) for i in range(n): - items.append(read_fstring_item(data)) + fitems.append(read_fstring_item(data)) else: s = StrExpr(read_str(data)) read_loc(data, s) - items.append(s) - expr = build_fstring_join(data, items) + fitems.append(s) + expr = build_fstring_join(data, fitems) expect_end_tag(data) return expr elif tag == nodes.LAMBDA_EXPR: diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index fe0083e58995c..80957da7c57d7 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -105,7 +105,7 @@ def locs(start_line: int, start_column: int, end_line: int, end_column: int) -> ] with temp_source("print('hello')") as fnam: - b = parse_to_binary_ast(fnam) + b, _, _ = parse_to_binary_ast(fnam) assert list(b) == ( [LITERAL_INT, 22, nodes.EXPR_STMT, nodes.CALL_EXPR] + [nodes.NAME_EXPR, LITERAL_STR] @@ -128,7 +128,7 @@ def test_deserialize_hello(self) -> None: def test_deserialize_member_expr(self) -> None: with temp_source("foo_bar.xyz2") as fnam: - node = native_parse(fnam) + node, _, _ = native_parse(fnam) assert isinstance(node, MypyFile) assert isinstance(node.defs[0], ExpressionStmt) assert isinstance(node.defs[0].expr, MemberExpr) @@ -141,7 +141,7 @@ def test_deserialize_bench(self) -> None: native_parse(fnam) t0 = time.time() for i in range(25): - node = native_parse(fnam) + node, _, _ = native_parse(fnam) assert isinstance(node, MypyFile) print(len(node.defs)) print((time.time() - t0) * 1000) From 204e66446769ae1f1c53493481f140aa64ae22b5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 4 Jan 2026 11:11:56 +0000 Subject: [PATCH 084/192] Support big integers --- mypy/nativeparse.py | 6 ++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 5 +++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 7985056812a74..935491a444ace 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -1030,6 +1030,12 @@ def read_expression(data: ReadBuffer) -> Expression: read_loc(data, expr) expect_end_tag(data) return expr + elif tag == nodes.BIG_INT_EXPR: + strval = read_str(data) + ie = IntExpr(int(strval)) + read_loc(data, ie) + expect_end_tag(data) + return ie else: assert False, tag diff --git a/mypy/nodes.py b/mypy/nodes.py index 466f05b9e1521..8b82cecf393b6 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5071,6 +5071,7 @@ def local_definitions( GLOBAL_DECL: Final[Tag] = 211 NONLOCAL_DECL: Final[Tag] = 212 AWAIT_EXPR: Final[Tag] = 213 +BIG_INT_EXPR: Final[Tag] = 214 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index b6873109eda93..e740f2d739ed5 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -40,7 +40,7 @@ MypyFile:1( StrExpr(z)))) [case testIntExpr] -(0, 1, 2, 203, 5345, 50123, 1234567890) +(0, 1, 2, 203, 5345, 50123, 1234567890, 9982374892739487239847897928374897298374) [out] MypyFile:1( ExpressionStmt:1( @@ -51,7 +51,8 @@ MypyFile:1( IntExpr(203) IntExpr(5345) IntExpr(50123) - IntExpr(1234567890)))) + IntExpr(1234567890) + IntExpr(9982374892739487239847897928374897298374)))) [case testAssignmentStmt] x = 1 From 77e10cbef5f51e1d6d15dfffb45803a34193ec15 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 5 Jan 2026 18:11:34 +0000 Subject: [PATCH 085/192] Support metaclass in class defs --- mypy/nativeparse.py | 14 +++++++++++--- test-data/unit/native-parser.test | 9 +++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 935491a444ace..5c367229f3693 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -531,16 +531,24 @@ def read_class_def(data: ReadBuffer) -> ClassDef: has_type_params = read_bool(data) assert not has_type_params, "Type parameters not yet supported" - # TODO: Metaclass (skip for now) + # Metaclass has_metaclass = read_bool(data) - assert not has_metaclass, "Metaclass not yet supported" + if has_metaclass: + metaclass = read_expression(data) + else: + metaclass = None # TODO: Keywords (skip for now) expect_tag(data, DICT_STR_GEN) n_keywords = read_int_bare(data) assert n_keywords == 0, "Keywords not yet supported" - class_def = ClassDef(name, body, base_type_exprs=base_type_exprs if base_type_exprs else None) + class_def = ClassDef( + name, + body, + base_type_exprs=base_type_exprs if base_type_exprs else None, + metaclass=metaclass + ) read_loc(data, class_def) expect_end_tag(data) return class_def diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index e740f2d739ed5..463c5222ccf71 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1752,3 +1752,12 @@ MypyFile:1( AssignmentStmt:7( NameExpr(zz) IntExpr(1))) + +[case testMetaclass] +class Foo(metaclass=Bar): pass +[out] +MypyFile:1( + ClassDef:1( + Foo + Metaclass(NameExpr(Bar)) + PassStmt:1())) From 85c94c35e287934e806d42e85b9cc1991b184094 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 5 Jan 2026 18:21:30 +0000 Subject: [PATCH 086/192] Support arbitrary class def keywords --- mypy/nativeparse.py | 21 +++++++++++---------- mypy/strconv.py | 3 +++ test-data/unit/native-parser.test | 10 ++++++++++ 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 5c367229f3693..4ce037c15ae69 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -531,23 +531,24 @@ def read_class_def(data: ReadBuffer) -> ClassDef: has_type_params = read_bool(data) assert not has_type_params, "Type parameters not yet supported" - # Metaclass - has_metaclass = read_bool(data) - if has_metaclass: - metaclass = read_expression(data) - else: - metaclass = None - - # TODO: Keywords (skip for now) + # Keywords (all keyword arguments including metaclass) expect_tag(data, DICT_STR_GEN) n_keywords = read_int_bare(data) - assert n_keywords == 0, "Keywords not yet supported" + keywords = [] + for _ in range(n_keywords): + key = read_str(data) + value = read_expression(data) + keywords.append((key, value)) + + # Extract metaclass from keywords if present + metaclass = dict(keywords).get("metaclass") if keywords else None class_def = ClassDef( name, body, base_type_exprs=base_type_exprs if base_type_exprs else None, - metaclass=metaclass + metaclass=metaclass, + keywords=keywords if keywords else None ) read_loc(data, class_def) expect_end_tag(data) diff --git a/mypy/strconv.py b/mypy/strconv.py index db2b59d49850e..ee4a7feed6ce4 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -185,6 +185,9 @@ def visit_class_def(self, o: mypy.nodes.ClassDef) -> str: a.insert(1, ("TypeVars", o.type_vars)) if o.metaclass: a.insert(1, f"Metaclass({o.metaclass.accept(self)})") + if o.keywords: + keyword_items = [f"{k}={v.accept(self)}" for k, v in o.keywords.items()] + a.insert(1, f"Keywords({', '.join(keyword_items)})") if o.decorators: a.insert(1, ("Decorators", o.decorators)) if o.info and o.info._promote: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 463c5222ccf71..4553f0c55f421 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1759,5 +1759,15 @@ class Foo(metaclass=Bar): pass MypyFile:1( ClassDef:1( Foo + Keywords(metaclass=NameExpr(Bar)) Metaclass(NameExpr(Bar)) PassStmt:1())) + +[case testClassWithKeywordArgs] +class Foo(_root=None, total=False): pass +[out] +MypyFile:1( + ClassDef:1( + Foo + Keywords(_root=NameExpr(None), total=NameExpr(False)) + PassStmt:1())) From 77288a57207e9b852e61ed1bc032dc66fc07d648 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 5 Jan 2026 18:28:24 +0000 Subject: [PATCH 087/192] Support unpack types --- mypy/nativeparse.py | 8 +++++++- test-data/unit/native-parser.test | 23 +++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 4ce037c15ae69..d2ef93849640d 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -111,7 +111,7 @@ YieldFromExpr, MISSING_FALLBACK, ) -from mypy.types import CallableType, UnboundType, NoneType, UnionType, AnyType, TypeOfAny, Instance, Type, TypeList, EllipsisType, RawExpressionType +from mypy.types import CallableType, UnboundType, NoneType, UnionType, AnyType, TypeOfAny, Instance, Type, TypeList, EllipsisType, RawExpressionType, UnpackType TypeIgnores = list[tuple[int, list[str]]] @@ -659,6 +659,12 @@ def read_type(data: ReadBuffer) -> Type: read_loc(data, raw_type) expect_end_tag(data) return raw_type + elif tag == types.UNPACK_TYPE: + inner_type = read_type(data) + unpack = UnpackType(inner_type) + read_loc(data, unpack) + expect_end_tag(data) + return unpack else: assert False, tag diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 4553f0c55f421..943d96257eb25 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1771,3 +1771,26 @@ MypyFile:1( Foo Keywords(_root=NameExpr(None), total=NameExpr(False)) PassStmt:1())) + +[case testUnpackTypeInSignature] +def f(*args: *Ts) -> None: + pass +[out] +MypyFile:1( + FuncDef:1( + f + def (*args: Unpack[Ts?]) -> None? + VarArg( + Var(args)) + Block:2( + PassStmt:2()))) + +[case testUnpackTypeInTuple] +x: tuple[int, *Ts, str] +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + TempNode:1( + Any) + tuple?[int?, Unpack[Ts?], str?])) From 2b89afb9c0f9b46f90878e4a0db18dea5725cfda Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 5 Jan 2026 18:34:07 +0000 Subject: [PATCH 088/192] Support int literal types --- mypy/nativeparse.py | 4 +++- test-data/unit/native-parser.test | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index d2ef93849640d..5803d177ae1c4 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -653,8 +653,10 @@ def read_type(data: ReadBuffer) -> Type: value: types.LiteralValue if type_name == "builtins.bool": value = read_bool(data) + elif type_name == "builtins.int": + value = read_int(data) else: - assert False # TODO + assert False, f"Unsupported RawExpressionType: {type_name}" raw_type = RawExpressionType(value, type_name) read_loc(data, raw_type) expect_end_tag(data) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 943d96257eb25..633797c0d72a2 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1297,6 +1297,22 @@ MypyFile:1( Any) Literal?[True, False])) +[case testLiteralIntTypes] +x: Literal[1] +y: Literal[1, 2, 3] +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + TempNode:1( + Any) + Literal?[1]) + AssignmentStmt:2( + NameExpr(y) + TempNode:2( + Any) + Literal?[1, 2, 3])) + [case testDelStmt] del x del x[0], y[1] From cd30492974b40c84eddbad7d70e43dec0e5b8ae0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 5 Jan 2026 18:42:35 +0000 Subject: [PATCH 089/192] Add negative int literal type test --- test-data/unit/native-parser.test | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 633797c0d72a2..4097e682e7dd5 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1300,6 +1300,8 @@ MypyFile:1( [case testLiteralIntTypes] x: Literal[1] y: Literal[1, 2, 3] +z: Literal[-1] +w: Literal[-1, 0, 1] [out] MypyFile:1( AssignmentStmt:1( @@ -1311,7 +1313,17 @@ MypyFile:1( NameExpr(y) TempNode:2( Any) - Literal?[1, 2, 3])) + Literal?[1, 2, 3]) + AssignmentStmt:3( + NameExpr(z) + TempNode:3( + Any) + Literal?[-1]) + AssignmentStmt:4( + NameExpr(w) + TempNode:4( + Any) + Literal?[-1, 0, 1])) [case testDelStmt] del x From 5a620d0bd81e0bd84d1bde343ba434493e68f724 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 7 Jan 2026 19:10:40 +0000 Subject: [PATCH 090/192] Replace non-ascii character in test --- mypy/test/testcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 5e533e3c245d4..fb36f67ca1191 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -62,7 +62,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: with tempfile.NamedTemporaryFile(prefix="test", dir=".") as temp_file: temp_path = Path(temp_file.name) if not temp_path.with_name(temp_path.name.upper()).exists(): - pytest.skip("File system is not case‐insensitive") + pytest.skip("File system is not case-insensitive") if lxml is None and os.path.basename(testcase.file) == "check-reports.test": pytest.skip("Cannot import lxml. Is it installed?") incremental = ( From bbe0164e8538233e53211bb0c82f67fb07678b9a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 7 Jan 2026 19:17:59 +0000 Subject: [PATCH 091/192] Support star imports --- mypy/nativeparse.py | 12 ++++++++++++ mypy/nodes.py | 1 + test-data/unit/native-parser.test | 12 ++++++++++++ 3 files changed, 25 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 5803d177ae1c4..157597db62612 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -77,6 +77,7 @@ GlobalDecl, IfStmt, Import, + ImportAll, ImportFrom, ListComprehension, SetComprehension, @@ -392,6 +393,17 @@ def read_statement(data: ReadBuffer) -> Statement: read_loc(data, stmt) expect_end_tag(data) return stmt + elif tag == nodes.IMPORT_ALL: + # Read module name (empty string for "from . import *") + module_id = read_str(data) + + # Read relative import level + relative = read_int(data) + + stmt = ImportAll(module_id, relative) + read_loc(data, stmt) + expect_end_tag(data) + return stmt elif tag == nodes.CLASS_DEF: return read_class_def(data) elif tag == nodes.TRY_STMT: diff --git a/mypy/nodes.py b/mypy/nodes.py index 8b82cecf393b6..8556a0af68ae9 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5072,6 +5072,7 @@ def local_definitions( NONLOCAL_DECL: Final[Tag] = 212 AWAIT_EXPR: Final[Tag] = 213 BIG_INT_EXPR: Final[Tag] = 214 +IMPORT_ALL: Final[Tag] = 215 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 4097e682e7dd5..4aa822ab3a477 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -813,6 +813,18 @@ MypyFile:1( ImportFrom:5(.., [bar]) ImportFrom:6(.pkg, [baz])) +[case testImportAllStatements] +from os import * +from . import * +from .. import * +from .pkg import * +[out] +MypyFile:1( + ImportAll:1(os) + ImportAll:2(.) + ImportAll:3(..) + ImportAll:4(.pkg)) + [case testRaiseStatements] raise raise ValueError From 0bd02e1424be5788206f0cd2896ca0a463ea482a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 7 Jan 2026 19:44:27 +0000 Subject: [PATCH 092/192] Support class decorators --- mypy/nativeparse.py | 5 +++-- test-data/unit/native-parser.test | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 157597db62612..7ca0f7a917c3e 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -534,10 +534,10 @@ def read_class_def(data: ReadBuffer) -> ClassDef: # Base classes base_type_exprs = read_expression_list(data) - # TODO: Decorators (skip for now) + # Decorators expect_tag(data, LIST_GEN) n_decorators = read_int_bare(data) - assert n_decorators == 0, "Decorators not yet supported" + decorators = [read_expression(data) for _ in range(n_decorators)] # TODO: Type parameters (skip for now) has_type_params = read_bool(data) @@ -562,6 +562,7 @@ def read_class_def(data: ReadBuffer) -> ClassDef: metaclass=metaclass, keywords=keywords if keywords else None ) + class_def.decorators = decorators read_loc(data, class_def) expect_end_tag(data) return class_def diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 4aa822ab3a477..7a45355d805a0 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1812,6 +1812,31 @@ MypyFile:1( Keywords(_root=NameExpr(None), total=NameExpr(False)) PassStmt:1())) +[case testClassDecorator] +@foo +class X: pass +@foo(bar) +@x.y +class Z: pass +[out] +MypyFile:1( + ClassDef:2( + X + Decorators( + NameExpr(foo)) + PassStmt:2()) + ClassDef:5( + Z + Decorators( + CallExpr:3( + NameExpr(foo) + Args( + NameExpr(bar))) + MemberExpr:4( + NameExpr(x) + y)) + PassStmt:5())) + [case testUnpackTypeInSignature] def f(*args: *Ts) -> None: pass From 545f6115b8cc6d174369a45f1e46a1df88d5ba8f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 10 Jan 2026 16:22:34 +0000 Subject: [PATCH 093/192] Support string literal types --- mypy/nativeparse.py | 43 ++++++++++++++++++-- test-data/unit/native-parser.test | 67 +++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 4 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 7ca0f7a917c3e..0c36f46ce9373 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -629,9 +629,24 @@ def read_type(data: ReadBuffer) -> Type: expect_tag(data, LIST_GEN) n = read_int_bare(data) args = tuple(read_type(data) for i in range(n)) - expect_tag(data, LITERAL_NONE) # TODO - expect_tag(data, LITERAL_NONE) # TODO - unbound = UnboundType(name, args) + # Read optional original_str_expr + t = read_tag(data) + if t == LITERAL_NONE: + original_str_expr = None + elif t == LITERAL_STR: + original_str_expr = read_str_bare(data) + else: + assert False, f"Unexpected tag for original_str_expr: {t}" + # Read optional original_str_fallback + t = read_tag(data) + if t == LITERAL_NONE: + original_str_fallback = None + elif t == LITERAL_STR: + original_str_fallback = read_str_bare(data) + else: + assert False, f"Unexpected tag for original_str_fallback: {t}" + unbound = UnboundType(name, args, original_str_expr=original_str_expr, + original_str_fallback=original_str_fallback) read_loc(data, unbound) expect_end_tag(data) return unbound @@ -642,7 +657,25 @@ def read_type(data: ReadBuffer) -> Type: items = [read_type(data) for i in range(n)] # Read uses_pep604_syntax flag uses_pep604_syntax = read_bool(data) + # Read optional original_str_expr + t = read_tag(data) + if t == LITERAL_NONE: + original_str_expr = None + elif t == LITERAL_STR: + original_str_expr = read_str_bare(data) + else: + assert False, f"Unexpected tag for original_str_expr: {t}" + # Read optional original_str_fallback + t = read_tag(data) + if t == LITERAL_NONE: + original_str_fallback = None + elif t == LITERAL_STR: + original_str_fallback = read_str_bare(data) + else: + assert False, f"Unexpected tag for original_str_fallback: {t}" union = UnionType(items, uses_pep604_syntax=uses_pep604_syntax) + union.original_str_expr = original_str_expr + union.original_str_fallback = original_str_fallback read_loc(data, union) expect_end_tag(data) return union @@ -663,11 +696,13 @@ def read_type(data: ReadBuffer) -> Type: return ellipsis_type elif tag == types.RAW_EXPRESSION_TYPE: type_name = read_str(data) - value: types.LiteralValue + value: types.LiteralValue | str if type_name == "builtins.bool": value = read_bool(data) elif type_name == "builtins.int": value = read_int(data) + elif type_name == "builtins.str": + value = read_str(data) else: assert False, f"Unsupported RawExpressionType: {type_name}" raw_type = RawExpressionType(value, type_name) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 7a45355d805a0..423f6f6a6b2c2 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1859,3 +1859,70 @@ MypyFile:1( TempNode:1( Any) tuple?[int?, Unpack[Ts?], str?])) + + +[case testLiteralStringType] +from typing import Literal +x: Literal["hello"] +[out] +MypyFile:1( + ImportFrom:1(typing, [Literal]) + AssignmentStmt:2( + NameExpr(x) + TempNode:2( + Any) + Literal?[hello?])) + +[case testLiteralStringWithEscapes] +from typing import Literal +x: Literal["hello\nworld"] +[out] +MypyFile:1( + ImportFrom:1(typing, [Literal]) + AssignmentStmt:2( + NameExpr(x) + TempNode:2( + Any) + Literal?['hello\nworld'])) + +[case testForwardReference] +x: "int" +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + TempNode:1( + Any) + int?)) + +[case testForwardReferenceWithUnion] +x: "int | str" +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + TempNode:1( + Any) + int? | str?)) + +[case testForwardReferenceWithComplexType] +x: "list[int]" +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + TempNode:1( + Any) + list?[int?])) + +[case testMultipleLiteralStrings] +from typing import Literal +x: Literal["a", "b", "c"] +[out] +MypyFile:1( + ImportFrom:1(typing, [Literal]) + AssignmentStmt:2( + NameExpr(x) + TempNode:2( + Any) + Literal?[a?, b?, c?])) From b6cc04a322aa3d04cd5836a375dd4a4a68d0c550 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 10 Jan 2026 20:16:53 +0000 Subject: [PATCH 094/192] Support bytes literal types --- mypy/nativeparse.py | 3 +++ mypy/types.py | 5 +++++ test-data/unit/native-parser.test | 18 ++++++++++++++++++ test-data/unit/parse.test | 10 ++++++++++ 4 files changed, 36 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 0c36f46ce9373..5d4c68c4a1124 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -703,6 +703,9 @@ def read_type(data: ReadBuffer) -> Type: value = read_int(data) elif type_name == "builtins.str": value = read_str(data) + elif type_name == "builtins.bytes": + # Bytes literals are serialized as escaped strings + value = read_str(data) else: assert False, f"Unsupported RawExpressionType: {type_name}" raw_type = RawExpressionType(value, type_name) diff --git a/mypy/types.py b/mypy/types.py index a865879ca5529..84eb59ed1ec74 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -3961,6 +3961,11 @@ def item_str(name: str, typ: str) -> str: return f"TypedDict({prefix}{s})" def visit_raw_expression_type(self, t: RawExpressionType, /) -> str: + # For bytes literals, the value is already escaped, just add quotes and b prefix + if t.base_type_name == "builtins.bytes": + # The value is already escaped (e.g., "foo" or "hello\\nworld") + # Just add quotes and b prefix + return f"b'{t.literal_value}'" return repr(t.literal_value) def visit_literal_type(self, t: LiteralType, /) -> str: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 423f6f6a6b2c2..ba3435b08e3fb 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1926,3 +1926,21 @@ MypyFile:1( TempNode:2( Any) Literal?[a?, b?, c?])) + +[case testLiteralBytesType] +from typing import Literal +x: Literal[b"foo"] +y: Literal[b"hello\nworld", b"\x00\xff"] +[out] +MypyFile:1( + ImportFrom:1(typing, [Literal]) + AssignmentStmt:2( + NameExpr(x) + TempNode:2( + Any) + Literal?[b'foo']) + AssignmentStmt:3( + NameExpr(y) + TempNode:3( + Any) + Literal?[b'hello\nworld', b'\x00\xff'])) diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test index b1c0918365a69..b9ed709aabce8 100644 --- a/test-data/unit/parse.test +++ b/test-data/unit/parse.test @@ -2444,6 +2444,16 @@ MypyFile:1( Block:1( PassStmt:1()))) +[case testBytesLiteralType] +x: Literal[b'foo\xfe\x00'] +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + TempNode:1( + Any) + Literal?[b'foo\xfe\x00'])) + [case testMetaclass] class Foo(metaclass=Bar): pass [out] From 2a2d4e93d7f14aff551a12fcde42035c8918ff1e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 17 Jan 2026 13:11:36 +0000 Subject: [PATCH 095/192] Add skip_function_bodies param --- mypy/nativeparse.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 5d4c68c4a1124..1a5e5ce12c1f3 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -166,8 +166,10 @@ def read_statements(data: ReadBuffer, n: int) -> list[Statement]: return defs -def parse_to_binary_ast(filename: str) -> tuple[bytes, list[dict[str, Any]], TypeIgnores]: - return ast_serialize.parse(filename) # type: ignore[no-any-return] +def parse_to_binary_ast( + filename: str, skip_function_bodies: bool = False +) -> tuple[bytes, list[dict[str, Any]], TypeIgnores]: + return ast_serialize.parse(filename, skip_function_bodies) # type: ignore[no-any-return] def read_statement(data: ReadBuffer) -> Statement: From 5191a54ef2bee55f18190206619f5663bd6e89f4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 17 Jan 2026 13:20:20 +0000 Subject: [PATCH 096/192] Add strip function bodies tests (can't run them yet properly) --- test-data/unit/native-parser.test | 355 ++++++++++++++++++++++++++++++ 1 file changed, 355 insertions(+) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index ba3435b08e3fb..cf8e644543926 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1944,3 +1944,358 @@ MypyFile:1( TempNode:3( Any) Literal?[b'hello\nworld', b'\x00\xff'])) + +[case testStripFunctionBodiesIfIgnoringErrors] +# mypy: ignore-errors=True +def f(self): + self.x = 1 # Cannot define an attribute + return 1 +[out] +MypyFile:1( + FuncDef:2( + f + Args( + Var(self)) + Block:3())) + +[case testStripMethodBodiesIfIgnoringErrors] +# mypy: ignore-errors=True +class C: + def f(self): + x = self.x + for x in y: + pass + with a as y: + pass + while self.foo(): + self.bah() + a[self.x] = 1 +[out] +MypyFile:1( + ClassDef:2( + C + FuncDef:3( + f + Args( + Var(self)) + Block:4()))) + +[case testDoNotStripModuleTopLevelOrClassBody] +# mypy: ignore-errors=True +f() +class C: + x = 5 +[out] +MypyFile:1( + ExpressionStmt:2( + CallExpr:2( + NameExpr(f) + Args())) + ClassDef:3( + C + AssignmentStmt:4( + NameExpr(x) + IntExpr(5)))) + +[case testDoNotStripMethodThatAssignsToAttribute] +# mypy: ignore-errors=True +class C: + def m1(self): + self.x = 0 + def m2(self): + a, self.y = 0 +[out] +MypyFile:1( + ClassDef:2( + C + FuncDef:3( + m1 + Args( + Var(self)) + Block:4( + AssignmentStmt:4( + MemberExpr:4( + NameExpr(self) + x) + IntExpr(0)))) + FuncDef:5( + m2 + Args( + Var(self)) + Block:6( + AssignmentStmt:6( + TupleExpr:6( + NameExpr(a) + MemberExpr:6( + NameExpr(self) + y)) + IntExpr(0)))))) + +[case testDoNotStripMethodThatAssignsToAttributeWithinStatement] +# mypy: ignore-errors=True +class C: + def m1(self): + for x in y: + self.x = 0 + def m2(self): + with x: + self.y = 0 + def m3(self): + if x: + self.y = 0 + else: + x = 4 +[out] +MypyFile:1( + ClassDef:2( + C + FuncDef:3( + m1 + Args( + Var(self)) + Block:4( + ForStmt:4( + NameExpr(x) + NameExpr(y) + Block:5( + AssignmentStmt:5( + MemberExpr:5( + NameExpr(self) + x) + IntExpr(0)))))) + FuncDef:6( + m2 + Args( + Var(self)) + Block:7( + WithStmt:7( + Expr( + NameExpr(x)) + Block:8( + AssignmentStmt:8( + MemberExpr:8( + NameExpr(self) + y) + IntExpr(0)))))) + FuncDef:9( + m3 + Args( + Var(self)) + Block:10( + IfStmt:10( + If( + NameExpr(x)) + Then( + AssignmentStmt:11( + MemberExpr:11( + NameExpr(self) + y) + IntExpr(0))) + Else( + AssignmentStmt:13( + NameExpr(x) + IntExpr(4)))))))) + +[case testDoNotStripMethodThatDefinesAttributeWithoutAssignment] +# mypy: ignore-errors=True +class C: + def m1(self): + with y as self.x: + pass + def m2(self): + for self.y in x: + pass +[out] +MypyFile:1( + ClassDef:2( + C + FuncDef:3( + m1 + Args( + Var(self)) + Block:4( + WithStmt:4( + Expr( + NameExpr(y)) + Target( + MemberExpr:4( + NameExpr(self) + x)) + Block:5( + PassStmt:5())))) + FuncDef:6( + m2 + Args( + Var(self)) + Block:7( + ForStmt:7( + MemberExpr:7( + NameExpr(self) + y) + NameExpr(x) + Block:8( + PassStmt:8())))))) + +[case testStripDecoratedFunctionOrMethod] +# mypy: ignore-errors=True +@deco +def f(): + x = 0 + +class C: + @deco + def m1(self): + x = 0 + + @deco + def m2(self): + self.x = 0 +[out] +MypyFile:1( + Decorator:2( + Var(f) + NameExpr(deco) + FuncDef:3( + f + Block:4())) + ClassDef:6( + C + Decorator:7( + Var(m1) + NameExpr(deco) + FuncDef:8( + m1 + Args( + Var(self)) + Block:9())) + Decorator:11( + Var(m2) + NameExpr(deco) + FuncDef:12( + m2 + Args( + Var(self)) + Block:13( + AssignmentStmt:13( + MemberExpr:13( + NameExpr(self) + x) + IntExpr(0))))))) + +[case testStripOverloadedMethod] +# mypy: ignore-errors=True +class C: + @overload + def m1(self, x: int) -> None: ... + @overload + def m1(self, x: str) -> None: ... + def m1(self, x): + x = 0 + + @overload + def m2(self, x: int) -> None: ... + @overload + def m2(self, x: str) -> None: ... + def m2(self, x): + self.x = 0 +[out] +MypyFile:1( + ClassDef:2( + C + OverloadedFuncDef:3( + Decorator:3( + Var(m1) + NameExpr(overload) + FuncDef:4( + m1 + Args( + Var(self) + Var(x)) + def (self: Any, x: int?) -> None? + Block:4( + ExpressionStmt:4( + Ellipsis)))) + Decorator:5( + Var(m1) + NameExpr(overload) + FuncDef:6( + m1 + Args( + Var(self) + Var(x)) + def (self: Any, x: str?) -> None? + Block:6( + ExpressionStmt:6( + Ellipsis)))) + FuncDef:7( + m1 + Args( + Var(self) + Var(x)) + Block:8())) + OverloadedFuncDef:10( + Decorator:10( + Var(m2) + NameExpr(overload) + FuncDef:11( + m2 + Args( + Var(self) + Var(x)) + def (self: Any, x: int?) -> None? + Block:11( + ExpressionStmt:11( + Ellipsis)))) + Decorator:12( + Var(m2) + NameExpr(overload) + FuncDef:13( + m2 + Args( + Var(self) + Var(x)) + def (self: Any, x: str?) -> None? + Block:13( + ExpressionStmt:13( + Ellipsis)))) + FuncDef:14( + m2 + Args( + Var(self) + Var(x)) + Block:15( + AssignmentStmt:15( + MemberExpr:15( + NameExpr(self) + x) + IntExpr(0))))))) + +[case testStripMethodInNestedClass] +# mypy: ignore-errors=True +class C: + class D: + def m1(self): + self.x = 1 + def m2(self): + return self.x +[out] +MypyFile:1( + ClassDef:2( + C + ClassDef:3( + D + FuncDef:4( + m1 + Args( + Var(self)) + Block:5( + AssignmentStmt:5( + MemberExpr:5( + NameExpr(self) + x) + IntExpr(1)))) + FuncDef:6( + m2 + Args( + Var(self)) + Block:7())))) From a6d9adac4082ae6604ef704b27f2e3d28024d9f4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 17 Jan 2026 13:21:29 +0000 Subject: [PATCH 097/192] Support skipping function bodies in the test runner --- mypy/nativeparse.py | 6 ++++-- mypy/test/test_nativeparse.py | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 1a5e5ce12c1f3..6ab2b4f53a41d 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -131,8 +131,10 @@ def expect_tag(data: ReadBuffer, tag: Tag) -> None: assert read_tag(data) == tag -def native_parse(filename: str) -> tuple[MypyFile, list[dict[str, Any]], TypeIgnores]: - b, errors, ignores = parse_to_binary_ast(filename) +def native_parse( + filename: str, skip_function_bodies: bool = False +) -> tuple[MypyFile, list[dict[str, Any]], TypeIgnores]: + b, errors, ignores = parse_to_binary_ast(filename, skip_function_bodies) data = ReadBuffer(b) n = read_int(data) defs = read_statements(data, n) diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index 80957da7c57d7..12ccdbacb4add 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -60,10 +60,13 @@ def test_parser(testcase: DataDrivenTestCase) -> None: changes, _ = parse_mypy_comments(comments, options) options = options.apply_changes(changes) + # Check if we should skip function bodies (when ignoring errors) + skip_function_bodies = "# mypy: ignore-errors=True" in source + try: with temp_source(source) as fnam: try: - node, errors, type_ignores = native_parse(fnam) + node, errors, type_ignores = native_parse(fnam, skip_function_bodies) except ValueError as e: print(f"Parse failed: {e}") assert False From b7c91e336a53ec0dbd4bbe5d66a3e644ffdb7c95 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 17 Jan 2026 13:37:35 +0000 Subject: [PATCH 098/192] Fix empty blocks --- mypy/nativeparse.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 6ab2b4f53a41d..9e9ab94a70f68 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -730,15 +730,22 @@ def read_block(data: ReadBuffer) -> Block: expect_tag(data, nodes.BLOCK) expect_tag(data, LIST_GEN) n = read_int_bare(data) - assert n > 0 - a = read_statements(data, n) - expect_end_tag(data) - b = Block(a) - b.line = a[0].line - b.column = a[0].column - b.end_line = a[-1].end_line - b.end_column = a[-1].end_column - return b + if n == 0: + # Empty block - read explicit location + b = Block([]) + read_loc(data, b) + expect_end_tag(data) + return b + else: + # Non-empty block - read statements and set location from them + a = read_statements(data, n) + expect_end_tag(data) + b = Block(a) + b.line = a[0].line + b.column = a[0].column + b.end_line = a[-1].end_line + b.end_column = a[-1].end_column + return b def read_optional_block(data: ReadBuffer) -> Block | None: From afc93b18f8c6c6d1f86c619ae1511f8795c4da1f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 17 Jan 2026 13:46:39 +0000 Subject: [PATCH 099/192] Add --native-parser flags (unused for now) --- mypy/main.py | 2 ++ mypy/options.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/mypy/main.py b/mypy/main.py index 864fe4b4febf6..5051ccc06dd94 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1259,6 +1259,8 @@ def add_invertible_flag( # --local-partial-types disallows partial types spanning module top level and a function # (implicitly defined in fine-grained incremental mode) add_invertible_flag("--local-partial-types", default=False, help=argparse.SUPPRESS) + # --native-parser enables the native parser (experimental) + add_invertible_flag("--native-parser", default=False, help=argparse.SUPPRESS) # --logical-deps adds some more dependencies that are not semantically needed, but # may be helpful to determine relative importance of classes and functions for overall # type precision in a code base. It also _removes_ some deps, so this flag should be never diff --git a/mypy/options.py b/mypy/options.py index cb5088af7e795..3d54af09e5a72 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -371,6 +371,8 @@ def __init__(self) -> None: self.logical_deps = False # If True, partial types can't span a module top level and a function self.local_partial_types = False + # If True, use the native parser (experimental) + self.native_parser = False # Some behaviors are changed when using Bazel (https://bazel.build). self.bazel = False # If True, export inferred types for all expressions as BuildResult.types From 2072f9eb3a9e9b7269769548c6b520cb41c59967 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 17 Jan 2026 13:58:31 +0000 Subject: [PATCH 100/192] WIP: basic integration of native parser --- mypy/parse.py | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/mypy/parse.py b/mypy/parse.py index ee61760c0ac07..0501acbe6a545 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -1,8 +1,28 @@ from __future__ import annotations from mypy.errors import Errors -from mypy.nodes import MypyFile +from mypy.nodes import MypyFile, ImportBase, Import, ImportFrom, ImportAll from mypy.options import Options +from mypy.traverser import TraverserVisitor + + +class ImportCollector(TraverserVisitor): + """Visitor that collects all import nodes from an AST.""" + + def __init__(self) -> None: + self.imports: list[ImportBase] = [] + + def visit_import(self, node: Import) -> None: + self.imports.append(node) + super().visit_import(node) + + def visit_import_from(self, node: ImportFrom) -> None: + self.imports.append(node) + super().visit_import_from(node) + + def visit_import_all(self, node: ImportAll) -> None: + self.imports.append(node) + super().visit_import_all(node) def parse( @@ -20,6 +40,24 @@ def parse( The python_version (major, minor) option determines the Python syntax variant. """ + if options.native_parser: + import mypy.nativeparse + + errors.set_file(fnam, module, options=options) + tree, parse_errors, type_ignores = mypy.nativeparse.native_parse(fnam) + # Convert type ignores list to dict + tree.ignored_lines = dict(type_ignores) + # Set is_stub based on file extension + tree.is_stub = fnam.endswith(".pyi") + # Collect all import nodes from the tree + collector = ImportCollector() + tree.accept(collector) + tree.imports = collector.imports + # TODO: Report parse_errors to errors object + if raise_on_error and errors.is_errors(): + errors.raise_error() + return tree + if options.transform_source is not None: source = options.transform_source(source) import mypy.fastparse From 661dbe867f50c5e903e2dbefd02fa9085c3fd85a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 17 Jan 2026 14:06:30 +0000 Subject: [PATCH 101/192] Strip function bodies if errors are ignored --- mypy/parse.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mypy/parse.py b/mypy/parse.py index 0501acbe6a545..f76c76ed9c553 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -43,8 +43,14 @@ def parse( if options.native_parser: import mypy.nativeparse + ignore_errors = options.ignore_errors or fnam in errors.ignored_files + # If errors are ignored, we can drop many function bodies to speed up type checking. + strip_function_bodies = ignore_errors and not options.preserve_asts + errors.set_file(fnam, module, options=options) - tree, parse_errors, type_ignores = mypy.nativeparse.native_parse(fnam) + tree, parse_errors, type_ignores = mypy.nativeparse.native_parse( + fnam, skip_function_bodies=strip_function_bodies + ) # Convert type ignores list to dict tree.ignored_lines = dict(type_ignores) # Set is_stub based on file extension From 387f1f2f9a0773e286c2cb7c5d45d32e5311b250 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 17 Jan 2026 14:13:07 +0000 Subject: [PATCH 102/192] Report syntax errors --- mypy/parse.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/mypy/parse.py b/mypy/parse.py index f76c76ed9c553..ced4736bef312 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -1,5 +1,8 @@ from __future__ import annotations +import re + +from mypy import errorcodes as codes from mypy.errors import Errors from mypy.nodes import MypyFile, ImportBase, Import, ImportFrom, ImportAll from mypy.options import Options @@ -59,7 +62,18 @@ def parse( collector = ImportCollector() tree.accept(collector) tree.imports = collector.imports - # TODO: Report parse_errors to errors object + # Report parse errors + for error in parse_errors: + message = error["message"] + # Standardize error message by capitalizing the first word + message = re.sub(r"^(\s*\w)", lambda m: m.group(1).upper(), message) + errors.report( + error["line"], + error["column"], + message, + blocker=True, + code=codes.SYNTAX, + ) if raise_on_error and errors.is_errors(): errors.raise_error() return tree From 937a67e066e0199f81ebf2da411eb47739f543e7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 17 Jan 2026 14:48:45 +0000 Subject: [PATCH 103/192] Fix class keywords --- mypy/nativeparse.py | 4 +++- test-data/unit/native-parser.test | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 9e9ab94a70f68..2d420eb5b9714 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -558,13 +558,15 @@ def read_class_def(data: ReadBuffer) -> ClassDef: # Extract metaclass from keywords if present metaclass = dict(keywords).get("metaclass") if keywords else None + # Remove metaclass from keywords since it's passed as a separate field + filtered_keywords = [(k, v) for k, v in keywords if k != "metaclass"] if keywords else None class_def = ClassDef( name, body, base_type_exprs=base_type_exprs if base_type_exprs else None, metaclass=metaclass, - keywords=keywords if keywords else None + keywords=filtered_keywords ) class_def.decorators = decorators read_loc(data, class_def) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index cf8e644543926..bfb6b5e8100f8 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1799,7 +1799,6 @@ class Foo(metaclass=Bar): pass MypyFile:1( ClassDef:1( Foo - Keywords(metaclass=NameExpr(Bar)) Metaclass(NameExpr(Bar)) PassStmt:1())) From 999dcfc5124a9dd657f0b33b0c55ef3df3f5c98d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 17 Jan 2026 15:21:13 +0000 Subject: [PATCH 104/192] Fix overloads --- mypy/nativeparse.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 2d420eb5b9714..bde36234b94ed 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -155,6 +155,7 @@ def read_statements(data: ReadBuffer, n: int) -> list[Statement]: prev = defs[-1] if isinstance(prev, OverloadedFuncDef): prev.items.append(stmt) + prev.unanalyzed_items.append(stmt) else: assert isinstance(prev, (FuncDef, Decorator)) defs[-1] = OverloadedFuncDef([prev, stmt]) From 075788c290c87c7074845fc2249524c215f5b620 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 18 Jan 2026 10:43:25 +0000 Subject: [PATCH 105/192] Support super() expressions --- mypy/nativeparse.py | 14 ++++++++++++-- test-data/unit/native-parser.test | 11 +++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index bde36234b94ed..7ad4f2903474b 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -101,6 +101,7 @@ StarExpr, Statement, StrExpr, + SuperExpr, TempNode, TryStmt, TupleExpr, @@ -810,9 +811,18 @@ def read_expression(data: ReadBuffer) -> Expression: e = read_expression(data) attr = read_str(data) m = MemberExpr(e, attr) - read_loc(data, m) + # Check if this is a super() call - if so, convert to SuperExpr + if ( + isinstance(e, CallExpr) + and isinstance(e.callee, NameExpr) + and e.callee.name == "super" + ): + result: Expression = SuperExpr(attr, e) + else: + result = m + read_loc(data, result) expect_end_tag(data) - return m + return result elif tag == nodes.STR_EXPR: se = StrExpr(read_str(data)) read_loc(data, se) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index bfb6b5e8100f8..84f8588fe6ffa 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -2298,3 +2298,14 @@ MypyFile:1( Args( Var(self)) Block:7())))) + +[case testSuperExpr] +super().x +[out] +MypyFile:1( + ExpressionStmt:1( + SuperExpr:1( + x + CallExpr:1( + NameExpr(super) + Args())))) From 6317d63d59c4b18a6b2bccd2e16090ea18348904 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 18 Jan 2026 10:49:10 +0000 Subject: [PATCH 106/192] Add support for callable types (still missing error handling) --- mypy/nativeparse.py | 106 +++++++++++++++++++++++++++++- mypy/types.py | 1 + test-data/unit/native-parser.test | 51 ++++++++++++++ 3 files changed, 157 insertions(+), 1 deletion(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 7ad4f2903474b..2c4d46506f061 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -113,7 +113,7 @@ YieldFromExpr, MISSING_FALLBACK, ) -from mypy.types import CallableType, UnboundType, NoneType, UnionType, AnyType, TypeOfAny, Instance, Type, TypeList, EllipsisType, RawExpressionType, UnpackType +from mypy.types import CallableType, UnboundType, NoneType, UnionType, AnyType, TypeOfAny, Instance, Type, TypeList, EllipsisType, RawExpressionType, UnpackType, CallableArgument TypeIgnores = list[tuple[int, list[str]]] @@ -726,6 +726,8 @@ def read_type(data: ReadBuffer) -> Type: read_loc(data, unpack) expect_end_tag(data) return unpack + elif tag == types.CALL_TYPE: + return read_call_type(data) else: assert False, tag @@ -1230,3 +1232,105 @@ def read_loc(data: ReadBuffer, node: Context) -> None: node.column = column node.end_line = line + read_int_bare(data) node.end_column = column + read_int_bare(data) + + +def stringify_type_name(typ: Type) -> str | None: + """Extract qualified name from a type (for Arg constructor detection).""" + if isinstance(typ, UnboundType): + return typ.name + return None + + +def extract_arg_name(typ: Type) -> str | None: + """Extract argument name from a type (for Arg name parameter).""" + if isinstance(typ, RawExpressionType) and typ.base_type_name == "builtins.str": + return typ.literal_value # type: ignore[return-value] + elif isinstance(typ, UnboundType): + # String literals in type context are parsed as UnboundType (forward references) + # For Arg names, these are typically simple names without dots + if typ.name == "None": + return None + # Return the name as-is (it's the argument name) + return typ.name + return None # Invalid, but let validation handle it + + +def read_call_type(data: ReadBuffer) -> Type: + """Read Call in type context - check if it's an Arg/DefaultArg/VarArg/KwArg constructor. + + TODO: This deserialization is lenient and defers error handling to type analysis. + The following validations are missing (see mypy/fastparse.py:1948-1999 for reference): + - ARG_CONSTRUCTOR_NAME_EXPECTED: When constructor name cannot be extracted + - ARG_CONSTRUCTOR_TOO_MANY_ARGS: When more than 2 positional arguments + - MULTIPLE_VALUES_FOR_NAME_KWARG: When 'name' is specified multiple times + - MULTIPLE_VALUES_FOR_TYPE_KWARG: When 'type' is specified multiple times + - ARG_CONSTRUCTOR_UNEXPECTED_ARG: When keyword arg is not 'name' or 'type' + - Invalid argument name validation: When name is not a string literal or None + """ + # Read callee + callee_type = read_type(data) + + # Read positional arguments + expect_tag(data, LIST_GEN) + n_args = read_int_bare(data) + args = [read_type(data) for _ in range(n_args)] + + # Read keyword arguments + expect_tag(data, LIST_GEN) + n_kwargs = read_int_bare(data) + kwargs = [] + for _ in range(n_kwargs): + tag_kw = read_tag(data) + if tag_kw == LITERAL_NONE: + kw_name = None + elif tag_kw == LITERAL_STR: + kw_name = read_str_bare(data) + else: + assert False, f"Unexpected tag for keyword name: {tag_kw}" + kw_value = read_type(data) + kwargs.append((kw_name, kw_value)) + + # Try to detect Arg/DefaultArg/VarArg/KwArg pattern + constructor = stringify_type_name(callee_type) + + if constructor: + # Extract type and name from arguments + name: str | None = None + default_type = AnyType(TypeOfAny.special_form) + typ: Type = default_type + + # Process positional arguments + for i, arg in enumerate(args): + if i == 0: + typ = arg + elif i == 1: + name = extract_arg_name(arg) + # TODO: Emit error for more than 2 positional args (ARG_CONSTRUCTOR_TOO_MANY_ARGS) + + # Process keyword arguments + for kw_name, kw_value in kwargs: + if kw_name == "name": + # TODO: Emit error if name already set (MULTIPLE_VALUES_FOR_NAME_KWARG) + if name is not None: + pass # Duplicate, but defer to type analysis + name = extract_arg_name(kw_value) + elif kw_name == "type": + # TODO: Emit error if type already set (MULTIPLE_VALUES_FOR_TYPE_KWARG) + if typ is not default_type: + pass # Duplicate, but defer to type analysis + typ = kw_value + # TODO: Emit error for unexpected keyword arg (ARG_CONSTRUCTOR_UNEXPECTED_ARG) + + # Create CallableArgument + call_arg = CallableArgument(typ, name, constructor) + read_loc(data, call_arg) + expect_end_tag(data) + return call_arg + + # TODO: Emit error when constructor name is None (ARG_CONSTRUCTOR_NAME_EXPECTED) + # If not a valid constructor, fall back to creating an invalid type + # (validation will report error later) + invalid = AnyType(TypeOfAny.from_error) + read_loc(data, invalid) + expect_end_tag(data) + return invalid diff --git a/mypy/types.py b/mypy/types.py index 84eb59ed1ec74..4faf7a0e9f431 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -4328,6 +4328,7 @@ def type_vars_as_args(type_vars: Sequence[TypeVarLikeType]) -> tuple[Type, ...]: LIST_TYPE: Final[Tag] = 118 # Only valid in serialized ASTs ELLIPSIS_TYPE: Final[Tag] = 119 # Only valid in serialized ASTs RAW_EXPRESSION_TYPE: Final[Tag] = 120 # Only valid in serialized ASTs +CALL_TYPE: Final[Tag] = 121 # Only valid in serialized ASTs def read_type(data: ReadBuffer, tag: Tag | None = None) -> Type: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 84f8588fe6ffa..b1a6c4bf31750 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1299,6 +1299,57 @@ MypyFile:1( Block:2( PassStmt:2()))) +[case testCallableWithArgConstructor] +from typing import Callable +from mypy_extensions import Arg +def f(x: Callable[[Arg(int, 'x'), Arg(str, 'y')], bool]) -> None: + pass +[out] +MypyFile:1( + ImportFrom:1(typing, [Callable]) + ImportFrom:2(mypy_extensions, [Arg]) + FuncDef:3( + f + Args( + Var(x)) + def (x: Callable?[, bool?]) -> None? + Block:4( + PassStmt:4()))) + +[case testCallableWithArgConstructorKeywordSyntax] +from typing import Callable +from mypy_extensions import Arg +def f(callback: Callable[[Arg(type=int, name='value')], str]) -> None: + pass +[out] +MypyFile:1( + ImportFrom:1(typing, [Callable]) + ImportFrom:2(mypy_extensions, [Arg]) + FuncDef:3( + f + Args( + Var(callback)) + def (callback: Callable?[, str?]) -> None? + Block:4( + PassStmt:4()))) + +[case testCallableWithArgNoName] +from typing import Callable +from mypy_extensions import Arg +def f(x: Callable[[Arg(int)], bool]) -> None: + pass +[out] +MypyFile:1( + ImportFrom:1(typing, [Callable]) + ImportFrom:2(mypy_extensions, [Arg]) + FuncDef:3( + f + Args( + Var(x)) + def (x: Callable?[, bool?]) -> None? + Block:4( + PassStmt:4()))) + [case testLiteralTypes] x: Literal[True, False] [out] From a2f84850f414a1d499fa46201abdfe58c7df9280 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 18 Jan 2026 19:07:25 +0000 Subject: [PATCH 107/192] Support positional-only arguments --- mypy/nativeparse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 2c4d46506f061..65f117c04fb99 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -515,7 +515,7 @@ def read_func_def(data: ReadBuffer) -> FuncDef: [arg.type_annotation if arg.type_annotation else AnyType(TypeOfAny.unannotated) for arg in arguments], [arg.kind for arg in arguments], - [arg.variable.name for arg in arguments], + [None if arg.pos_only else arg.variable.name for arg in arguments], return_type if return_type else AnyType(TypeOfAny.unannotated), _dummy_fallback ) @@ -1077,7 +1077,7 @@ def read_expression(data: ReadBuffer) -> Expression: [arg.type_annotation if arg.type_annotation else AnyType(TypeOfAny.unannotated) for arg in arguments], [arg.kind for arg in arguments], - [arg.variable.name for arg in arguments], + [None if arg.pos_only else arg.variable.name for arg in arguments], AnyType(TypeOfAny.unannotated), _dummy_fallback, ) From 8e7687942a1758e631e1051b416c08c277eda219 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 18 Jan 2026 19:09:53 +0000 Subject: [PATCH 108/192] Add some type comment tests (simple cases only) --- test-data/unit/native-parser.test | 41 +++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index b1a6c4bf31750..6f0d42815c030 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -2360,3 +2360,44 @@ MypyFile:1( CallExpr:1( NameExpr(super) Args())))) + +[case testTypeCommentSimple] +x = [] # type: list[int] +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + ListExpr:1() + list?[int?])) + +[case testTypeCommentMultiple] +x = [] # type: list[int] +y = {} # type: dict[str, int] +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + ListExpr:1() + list?[int?]) + AssignmentStmt:2( + NameExpr(y) + DictExpr:2() + dict?[str?, int?])) + +[case testTypeCommentWithTrailingComment] +x = [] # type: list[str] # This is a comment +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + ListExpr:1() + list?[str?])) + +[case testTypeCommentUnion] +x = None # type: str | None +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + NameExpr(None) + str? | None?)) From c400120739eb2efb864bfa1e80dc3e1a73a5e084 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 18 Jan 2026 19:16:09 +0000 Subject: [PATCH 109/192] Support async with and async for --- mypy/nativeparse.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 65f117c04fb99..1c7d1c9fbd5df 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -314,7 +314,10 @@ def read_statement(data: ReadBuffer) -> Statement: body = read_block(data) # Read else clause else_body = read_optional_block(data) + # Read is_async flag + is_async = read_bool(data) stmt = ForStmt(index, expr, body, else_body) + stmt.is_async = is_async read_loc(data, stmt) expect_end_tag(data) return stmt @@ -337,7 +340,10 @@ def read_statement(data: ReadBuffer) -> Statement: target_list.append(None) # Read body body = read_block(data) + # Read is_async flag + is_async = read_bool(data) stmt = WithStmt(expr_list, target_list, body) + stmt.is_async = is_async read_loc(data, stmt) expect_end_tag(data) return stmt From 103ecd5169a358c893dd1511a4d6bec0fab0a0d8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 18 Jan 2026 19:36:42 +0000 Subject: [PATCH 110/192] Added nested list assignment test --- test-data/unit/native-parser.test | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 6f0d42815c030..40ddbbea9f056 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -2401,3 +2401,19 @@ MypyFile:1( NameExpr(x) NameExpr(None) str? | None?)) + +[case testNestedListAssignment] +a1, [b1, c1] = a2, [b2, c2] +[out] +MypyFile:1( + AssignmentStmt:1( + TupleExpr:1( + NameExpr(a1) + TupleExpr:1( + NameExpr(b1) + NameExpr(c1))) + TupleExpr:1( + NameExpr(a2) + ListExpr:1( + NameExpr(b2) + NameExpr(c2))))) From 0a5373bfe3e2fd09950dc6651956d6ca0df3b892 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jan 2026 18:12:06 +0000 Subject: [PATCH 111/192] Add test case for legacy parser --- test-data/unit/parse.test | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test index b9ed709aabce8..4bd3707a8a128 100644 --- a/test-data/unit/parse.test +++ b/test-data/unit/parse.test @@ -194,6 +194,22 @@ MypyFile:1( IntExpr(2) IntExpr(3))))) +[case testNestedListAssignment] +a1, [b1, c1] = a2, [b2, c2] +[out] +MypyFile:1( + AssignmentStmt:1( + TupleExpr:1( + NameExpr(a1) + TupleExpr:1( + NameExpr(b1) + NameExpr(c1))) + TupleExpr:1( + NameExpr(a2) + ListExpr:1( + NameExpr(b2) + NameExpr(c2))))) + [case testSimpleFunction] def main(): 1 @@ -3862,3 +3878,25 @@ MypyFile:1( Args( Var(self)) Block:7())))) + +[case testXXX] +from functools import total_ordering +from typing import Any, Callable, ClassVar + +@total_ordering +class Ord: + __eq__: Callable[[Any, object], bool] = lambda self, other: False + __lt__: Callable[[Any, "Ord"], bool] = lambda self, other: False + +reveal_type(Ord() < Ord()) # N: Revealed type is "builtins.bool" +reveal_type(Ord() <= Ord()) # N: Revealed type is "builtins.bool" +reveal_type(Ord() == Ord()) # N: Revealed type is "builtins.bool" +reveal_type(Ord() > Ord()) # N: Revealed type is "builtins.bool" +reveal_type(Ord() >= Ord()) # N: Revealed type is "builtins.bool" + +Ord() < 1 # E: Argument 1 has incompatible type "int"; expected "Ord" +Ord() <= 1 # E: Unsupported operand types for <= ("Ord" and "int") +Ord() == 1 +Ord() > 1 # E: Unsupported operand types for > ("Ord" and "int") +Ord() >= 1 # E: Unsupported operand types for >= ("Ord" and "int") +[out] \ No newline at end of file From 31d56609386d03b946c8b404cd9e702a4d7f4e51 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jan 2026 19:26:54 +0000 Subject: [PATCH 112/192] Make --native-parser affect cache --- mypy/options.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy/options.py b/mypy/options.py index 3d54af09e5a72..d4f4355569c76 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -67,6 +67,7 @@ class BuildType: | { "platform", "bazel", + "native_parser", "old_type_inference", "plugins", "disable_bytearray_promotion", From 49216b4175e4068ea560601a5cfee6fc0ac34e31 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jan 2026 19:27:14 +0000 Subject: [PATCH 113/192] WIP return empty file for directory --- mypy/nativeparse.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 1c7d1c9fbd5df..0d0989dc455cd 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -18,6 +18,7 @@ from __future__ import annotations +import os from typing import Final, cast, Any import ast_serialize # type: ignore[import-untyped] @@ -135,6 +136,13 @@ def expect_tag(data: ReadBuffer, tag: Tag) -> None: def native_parse( filename: str, skip_function_bodies: bool = False ) -> tuple[MypyFile, list[dict[str, Any]], TypeIgnores]: + # If the path is a directory, return empty AST (matching fastparse behavior) + # This can happen for packages that only contain .pyc files without source + if os.path.isdir(filename): + node = MypyFile([], []) + node.path = filename + return node, [], [] + b, errors, ignores = parse_to_binary_ast(filename, skip_function_bodies) data = ReadBuffer(b) n = read_int(data) From 723213388a965d83643af5f842da3cb9ed1734d4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jan 2026 19:28:08 +0000 Subject: [PATCH 114/192] WIP TEMPORARY ignore some errors caused be native parser issues --- mypy/plugins/attrs.py | 2 +- mypy/test/teststubgen.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mypy/plugins/attrs.py b/mypy/plugins/attrs.py index 47c6ad9f305a2..619315dc68256 100644 --- a/mypy/plugins/attrs.py +++ b/mypy/plugins/attrs.py @@ -1107,7 +1107,7 @@ def _meet_fields(types: list[Mapping[str, Type]]) -> Mapping[str, Type]: return { name: ( - get_proper_type(reduce(meet_types, f_types)) + get_proper_type(reduce(meet_types, f_types)) # type: ignore[call-arg, misc] # HAX if len(f_types) == len(types) else UninhabitedType() ) diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 605409f995232..f8b0cc3185743 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -1153,7 +1153,7 @@ def test(self, arg0: str) -> None: def test_generate_c_type_classmethod(self) -> None: class TestClass: @classmethod - def test(cls, arg0: str) -> None: + def test(cls, arg0: str) -> None: # type: ignore[no-redef] # HAX pass output: list[str] = [] @@ -1171,7 +1171,7 @@ def test(cls, arg0: str) -> None: def test_generate_c_type_classmethod_with_overloads(self) -> None: class TestClass: @classmethod - def test(cls, arg0: str) -> None: + def test(cls, arg0: str) -> None: # type: ignore[no-redef] # HAX """ test(cls, arg0: str) test(cls, arg0: int) @@ -1370,11 +1370,11 @@ def __init__(self) -> None: self._attribute = 0 @property - def attribute(self) -> int: + def attribute(self) -> int: # type: ignore[no-redef] # HAX return self._attribute @attribute.setter - def attribute(self, value: int) -> None: + def attribute(self, value: int) -> None: # type: ignore[no-redef] # HAX self._attribute = value readwrite_properties: list[str] = [] From a949c69e88d4dfbde7c12b7bea5bcb91024cb5d4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jan 2026 19:28:35 +0000 Subject: [PATCH 115/192] WIP HAX allow parsing without file name --- mypy/parse.py | 69 +++++++++++++++++++++++++++------------------------ 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/mypy/parse.py b/mypy/parse.py index ced4736bef312..14d3e848760ab 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import re from mypy import errorcodes as codes @@ -44,39 +45,43 @@ def parse( The python_version (major, minor) option determines the Python syntax variant. """ if options.native_parser: - import mypy.nativeparse - - ignore_errors = options.ignore_errors or fnam in errors.ignored_files - # If errors are ignored, we can drop many function bodies to speed up type checking. - strip_function_bodies = ignore_errors and not options.preserve_asts - - errors.set_file(fnam, module, options=options) - tree, parse_errors, type_ignores = mypy.nativeparse.native_parse( - fnam, skip_function_bodies=strip_function_bodies - ) - # Convert type ignores list to dict - tree.ignored_lines = dict(type_ignores) - # Set is_stub based on file extension - tree.is_stub = fnam.endswith(".pyi") - # Collect all import nodes from the tree - collector = ImportCollector() - tree.accept(collector) - tree.imports = collector.imports - # Report parse errors - for error in parse_errors: - message = error["message"] - # Standardize error message by capitalizing the first word - message = re.sub(r"^(\s*\w)", lambda m: m.group(1).upper(), message) - errors.report( - error["line"], - error["column"], - message, - blocker=True, - code=codes.SYNTAX, + # Native parser only works with actual files on disk + # Fall back to fastparse for in-memory source or non-existent files + if os.path.exists(fnam): + import mypy.nativeparse + + ignore_errors = options.ignore_errors or fnam in errors.ignored_files + # If errors are ignored, we can drop many function bodies to speed up type checking. + strip_function_bodies = ignore_errors and not options.preserve_asts + + errors.set_file(fnam, module, options=options) + tree, parse_errors, type_ignores = mypy.nativeparse.native_parse( + fnam, skip_function_bodies=strip_function_bodies ) - if raise_on_error and errors.is_errors(): - errors.raise_error() - return tree + # Convert type ignores list to dict + tree.ignored_lines = dict(type_ignores) + # Set is_stub based on file extension + tree.is_stub = fnam.endswith(".pyi") + # Collect all import nodes from the tree + collector = ImportCollector() + tree.accept(collector) + tree.imports = collector.imports + # Report parse errors + for error in parse_errors: + message = error["message"] + # Standardize error message by capitalizing the first word + message = re.sub(r"^(\s*\w)", lambda m: m.group(1).upper(), message) + errors.report( + error["line"], + error["column"], + message, + blocker=True, + code=codes.SYNTAX, + ) + if raise_on_error and errors.is_errors(): + errors.raise_error() + return tree + # Fall through to fastparse for non-existent files if options.transform_source is not None: source = options.transform_source(source) From 69e73accbd2bc5a3bab557a8ccb84857cbffb7c6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jan 2026 20:40:40 +0000 Subject: [PATCH 116/192] TEMPORARY: turn on native parser by default --- mypy/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/options.py b/mypy/options.py index d4f4355569c76..e9d46d943dd77 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -373,7 +373,7 @@ def __init__(self) -> None: # If True, partial types can't span a module top level and a function self.local_partial_types = False # If True, use the native parser (experimental) - self.native_parser = False + self.native_parser = True # Some behaviors are changed when using Bazel (https://bazel.build). self.bazel = False # If True, export inferred types for all expressions as BuildResult.types From 0f569950530ea2dfa10732604702c090aa3ec675 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jan 2026 20:49:08 +0000 Subject: [PATCH 117/192] Fix elif ast structure --- mypy/nativeparse.py | 45 +++++++++++++++++--- test-data/unit/native-parser.test | 68 ++++++++++++++++++++++++++++--- 2 files changed, 103 insertions(+), 10 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 0d0989dc455cd..618a2face69e4 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -250,18 +250,53 @@ def read_statement(data: ReadBuffer) -> Statement: expect_end_tag(data) return stmt elif tag == nodes.IF_STMT: - expr = [read_expression(data)] - body = [read_block(data)] + # Read the main if condition and body + expr = read_expression(data) + body = read_block(data) + + # Read elif clauses num_elif = read_int(data) + elif_exprs = [] + elif_bodies = [] for i in range(num_elif): - expr.append(read_expression(data)) - body.append(read_block(data)) + elif_exprs.append(read_expression(data)) + elif_bodies.append(read_block(data)) + + # Read else clause has_else = read_bool(data) if has_else: else_body = read_block(data) else: else_body = None - if_stmt = IfStmt(expr, body, else_body) + + # Normalize elif into nested if/else statements + # Build from the bottom up, starting with the final else body + current_else = else_body + + # Process elif clauses in reverse order + for i in range(len(elif_exprs) - 1, -1, -1): + # Create an IfStmt for this elif + elif_stmt = IfStmt([elif_exprs[i]], [elif_bodies[i]], current_else) + # Set location from the elif expression + elif_stmt.line = elif_exprs[i].line + elif_stmt.column = elif_exprs[i].column + # Set end location based on what follows + if current_else is not None: + elif_stmt.end_line = current_else.end_line + elif_stmt.end_column = current_else.end_column + else: + elif_stmt.end_line = elif_bodies[i].end_line + elif_stmt.end_column = elif_bodies[i].end_column + + # Wrap in a Block to become the else clause for the outer if + current_else = Block([elif_stmt]) + current_else.line = elif_stmt.line + current_else.column = elif_stmt.column + current_else.end_line = elif_stmt.end_line + current_else.end_column = elif_stmt.end_column + + # Create the main if statement + if_stmt = IfStmt([expr], [body], current_else) read_loc(data, if_stmt) expect_end_tag(data) return if_stmt diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 40ddbbea9f056..9fc0b8c94ce68 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -129,14 +129,72 @@ MypyFile:1( Then( ExpressionStmt:2( NameExpr(b))) + Else( + IfStmt:3( + If( + NameExpr(c)) + Then( + ExpressionStmt:4( + NameExpr(d))) + Else( + ExpressionStmt:6( + NameExpr(e))))))) + +[case testIfStmtMultipleElif] +if 1: + 2 +elif 3: + 4 +elif 5: + 6 +else: + 7 +[out] +MypyFile:1( + IfStmt:1( If( - NameExpr(c)) + IntExpr(1)) Then( - ExpressionStmt:4( - NameExpr(d))) + ExpressionStmt:2( + IntExpr(2))) Else( - ExpressionStmt:6( - NameExpr(e))))) + IfStmt:3( + If( + IntExpr(3)) + Then( + ExpressionStmt:4( + IntExpr(4))) + Else( + IfStmt:5( + If( + IntExpr(5)) + Then( + ExpressionStmt:6( + IntExpr(6))) + Else( + ExpressionStmt:8( + IntExpr(7))))))))) + +[case testIfStmtElifNoElse] +if a: + b +elif c: + d +[out] +MypyFile:1( + IfStmt:1( + If( + NameExpr(a)) + Then( + ExpressionStmt:2( + NameExpr(b))) + Else( + IfStmt:3( + If( + NameExpr(c)) + Then( + ExpressionStmt:4( + NameExpr(d))))))) [case testWhileStmt] while a: From 8cdcc61ff63d688b94c50e5dcf6d5cc207c35e73 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jan 2026 21:00:33 +0000 Subject: [PATCH 118/192] WIP Ignore --- mypy/plugins/attrs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/plugins/attrs.py b/mypy/plugins/attrs.py index 619315dc68256..cc3f13ce591ad 100644 --- a/mypy/plugins/attrs.py +++ b/mypy/plugins/attrs.py @@ -1107,7 +1107,7 @@ def _meet_fields(types: list[Mapping[str, Type]]) -> Mapping[str, Type]: return { name: ( - get_proper_type(reduce(meet_types, f_types)) # type: ignore[call-arg, misc] # HAX + get_proper_type(reduce(meet_types, f_types)) # type: ignore[call-arg, misc, unused-ignore] # HAX if len(f_types) == len(types) else UninhabitedType() ) From fcfecce07a6d2b157c1bd4c14970b72329e98cf7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 24 Jan 2026 10:32:31 +0000 Subject: [PATCH 119/192] Revert "TEMPORARY: turn on native parser by default" This reverts commit 69e73accbd2bc5a3bab557a8ccb84857cbffb7c6. --- mypy/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/options.py b/mypy/options.py index e9d46d943dd77..d4f4355569c76 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -373,7 +373,7 @@ def __init__(self) -> None: # If True, partial types can't span a module top level and a function self.local_partial_types = False # If True, use the native parser (experimental) - self.native_parser = True + self.native_parser = False # Some behaviors are changed when using Bazel (https://bazel.build). self.bazel = False # If True, export inferred types for all expressions as BuildResult.types From 3dabbd0c2f47b5079a47947b35da54d78993ccab Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 24 Jan 2026 11:29:07 +0000 Subject: [PATCH 120/192] Fix decorated functions --- mypy/nativeparse.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 618a2face69e4..87e8bae3175c0 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -197,6 +197,7 @@ def read_statement(data: ReadBuffer) -> Statement: column = read_int(data) fdef = read_statement(data) assert isinstance(fdef, FuncDef) + fdef.is_decorated = True var = Var(fdef.name) var.line = fdef.line var.is_ready = False From 43fc602c6fe83135b119f657a10758128378c671 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 24 Jan 2026 11:32:27 +0000 Subject: [PATCH 121/192] Revert some ignores --- mypy/test/teststubgen.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index f8b0cc3185743..605409f995232 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -1153,7 +1153,7 @@ def test(self, arg0: str) -> None: def test_generate_c_type_classmethod(self) -> None: class TestClass: @classmethod - def test(cls, arg0: str) -> None: # type: ignore[no-redef] # HAX + def test(cls, arg0: str) -> None: pass output: list[str] = [] @@ -1171,7 +1171,7 @@ def test(cls, arg0: str) -> None: # type: ignore[no-redef] # HAX def test_generate_c_type_classmethod_with_overloads(self) -> None: class TestClass: @classmethod - def test(cls, arg0: str) -> None: # type: ignore[no-redef] # HAX + def test(cls, arg0: str) -> None: """ test(cls, arg0: str) test(cls, arg0: int) @@ -1370,11 +1370,11 @@ def __init__(self) -> None: self._attribute = 0 @property - def attribute(self) -> int: # type: ignore[no-redef] # HAX + def attribute(self) -> int: return self._attribute @attribute.setter - def attribute(self, value: int) -> None: # type: ignore[no-redef] # HAX + def attribute(self, value: int) -> None: self._attribute = value readwrite_properties: list[str] = [] From 1e0983217558094d2c727a4a7e3a5ff710963a9c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 24 Jan 2026 15:16:21 +0000 Subject: [PATCH 122/192] Lint --- mypy/nativeparse.py | 84 +++++++++++++++++++++-------------- mypy/parse.py | 8 +--- mypy/test/test_nativeparse.py | 1 - 3 files changed, 52 insertions(+), 41 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 87e8bae3175c0..e5601d7a06e49 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -19,9 +19,14 @@ from __future__ import annotations import os -from typing import Final, cast, Any +from typing import Any, Final, cast import ast_serialize # type: ignore[import-untyped] +from librt.internal import ( + read_float as read_float_bare, + read_int as read_int_bare, + read_str as read_str_bare, +) from mypy import nodes, types from mypy.cache import ( @@ -35,35 +40,30 @@ LOCATION, ReadBuffer, Tag, + read_bool, read_int, read_str, read_tag, - read_bool, ) -from librt.internal import read_str as read_str_bare, read_float as read_float_bare, read_int as read_int_bare from mypy.nodes import ( - ARG_POS, - ARG_OPT, - ARG_STAR, - ARG_NAMED, - ARG_STAR2, - ARG_NAMED_OPT, ARG_KINDS, + ARG_POS, + MISSING_FALLBACK, Argument, AssertStmt, AssignmentExpr, - AwaitExpr, AssignmentStmt, + AwaitExpr, Block, BreakStmt, BytesExpr, CallExpr, ClassDef, ComparisonExpr, - ConditionalExpr, - ContinueStmt, ComplexExpr, + ConditionalExpr, Context, + ContinueStmt, Decorator, DelStmt, DictExpr, @@ -80,23 +80,22 @@ Import, ImportAll, ImportFrom, - ListComprehension, - SetComprehension, IndexExpr, IntExpr, LambdaExpr, + ListComprehension, ListExpr, MemberExpr, MypyFile, NameExpr, - Node, NonlocalDecl, - OpExpr, OperatorAssignmentStmt, + OpExpr, OverloadedFuncDef, PassStmt, RaiseStmt, ReturnStmt, + SetComprehension, SetExpr, SliceExpr, StarExpr, @@ -112,10 +111,21 @@ WithStmt, YieldExpr, YieldFromExpr, - MISSING_FALLBACK, ) -from mypy.types import CallableType, UnboundType, NoneType, UnionType, AnyType, TypeOfAny, Instance, Type, TypeList, EllipsisType, RawExpressionType, UnpackType, CallableArgument - +from mypy.types import ( + AnyType, + CallableArgument, + CallableType, + EllipsisType, + Instance, + RawExpressionType, + Type, + TypeList, + TypeOfAny, + UnboundType, + UnionType, + UnpackType, +) TypeIgnores = list[tuple[int, list[str]]] @@ -562,13 +572,15 @@ def read_func_def(data: ReadBuffer) -> FuncDef: if has_ann: typ = CallableType( - [arg.type_annotation if arg.type_annotation else AnyType(TypeOfAny.unannotated) - for arg in arguments], + [ + arg.type_annotation if arg.type_annotation else AnyType(TypeOfAny.unannotated) + for arg in arguments + ], [arg.kind for arg in arguments], [None if arg.pos_only else arg.variable.name for arg in arguments], return_type if return_type else AnyType(TypeOfAny.unannotated), - _dummy_fallback - ) + _dummy_fallback, + ) else: typ = None @@ -618,7 +630,7 @@ def read_class_def(data: ReadBuffer) -> ClassDef: body, base_type_exprs=base_type_exprs if base_type_exprs else None, metaclass=metaclass, - keywords=filtered_keywords + keywords=filtered_keywords, ) class_def.decorators = decorators read_loc(data, class_def) @@ -703,8 +715,12 @@ def read_type(data: ReadBuffer) -> Type: original_str_fallback = read_str_bare(data) else: assert False, f"Unexpected tag for original_str_fallback: {t}" - unbound = UnboundType(name, args, original_str_expr=original_str_expr, - original_str_fallback=original_str_fallback) + unbound = UnboundType( + name, + args, + original_str_expr=original_str_expr, + original_str_fallback=original_str_fallback, + ) read_loc(data, unbound) expect_end_tag(data) return unbound @@ -864,11 +880,7 @@ def read_expression(data: ReadBuffer) -> Expression: attr = read_str(data) m = MemberExpr(e, attr) # Check if this is a super() call - if so, convert to SuperExpr - if ( - isinstance(e, CallExpr) - and isinstance(e.callee, NameExpr) - and e.callee.name == "super" - ): + if isinstance(e, CallExpr) and isinstance(e.callee, NameExpr) and e.callee.name == "super": result: Expression = SuperExpr(attr, e) else: result = m @@ -1124,8 +1136,10 @@ def read_expression(data: ReadBuffer) -> Expression: # Create lambda expression if has_ann: typ = CallableType( - [arg.type_annotation if arg.type_annotation else AnyType(TypeOfAny.unannotated) - for arg in arguments], + [ + arg.type_annotation if arg.type_annotation else AnyType(TypeOfAny.unannotated) + for arg in arguments + ], [arg.kind for arg in arguments], [None if arg.pos_only else arg.variable.name for arg in arguments], AnyType(TypeOfAny.unannotated), @@ -1148,7 +1162,9 @@ def read_expression(data: ReadBuffer) -> Expression: if not isinstance(target, NameExpr): # In case target is not a NameExpr, we need to handle this # For now, we'll assert since the grammar should ensure it's a NameExpr - assert isinstance(target, NameExpr), f"Expected NameExpr for target, got {type(target)}" + assert isinstance( + target, NameExpr + ), f"Expected NameExpr for target, got {type(target)}" expr = AssignmentExpr(target, value) read_loc(data, expr) expect_end_tag(data) diff --git a/mypy/parse.py b/mypy/parse.py index 14d3e848760ab..27daef9b91f43 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -5,7 +5,7 @@ from mypy import errorcodes as codes from mypy.errors import Errors -from mypy.nodes import MypyFile, ImportBase, Import, ImportFrom, ImportAll +from mypy.nodes import Import, ImportAll, ImportBase, ImportFrom, MypyFile from mypy.options import Options from mypy.traverser import TraverserVisitor @@ -72,11 +72,7 @@ def parse( # Standardize error message by capitalizing the first word message = re.sub(r"^(\s*\w)", lambda m: m.group(1).upper(), message) errors.report( - error["line"], - error["column"], - message, - blocker=True, - code=codes.SYNTAX, + error["line"], error["column"], message, blocker=True, code=codes.SYNTAX ) if raise_on_error and errors.is_errors(): errors.raise_error() diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index 12ccdbacb4add..e98ace89c9f55 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -8,7 +8,6 @@ import tempfile import unittest from collections.abc import Iterator - from typing import Any from mypy import defaults, nodes From 85c335f41bb9aca16eabfba30df89992b18b87c1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 24 Jan 2026 15:46:09 +0000 Subject: [PATCH 123/192] Add additional syntax error tests --- test-data/unit/native-parser.test | 86 +++++++++++++++++++++++++------ 1 file changed, 71 insertions(+), 15 deletions(-) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 9fc0b8c94ce68..1e4e6240ff248 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1794,21 +1794,6 @@ MypyFile:1( Args( NameExpr(x)))))))) -[case testSyntaxError] -x = [1 2] -y = 2 -[out] -1:8: error: Expected ',', found int -MypyFile:1( - AssignmentStmt:1( - NameExpr(x) - ListExpr:1( - IntExpr(1) - IntExpr(2))) - AssignmentStmt:2( - NameExpr(y) - IntExpr(2))) - [case testFunctionOverload] @overload def f() -> x: pass @@ -2475,3 +2460,74 @@ MypyFile:1( ListExpr:1( NameExpr(b2) NameExpr(c2))))) + +[case testSyntaxError1] +x = [1 2] +y = 2 +[out] +1:8: error: Expected ',', found int +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + ListExpr:1( + IntExpr(1) + IntExpr(2))) + AssignmentStmt:2( + NameExpr(y) + IntExpr(2))) + +[case testSyntaxError2] +f(a.) +[out] +1:5: error: Expected an identifier +MypyFile:1( + ExpressionStmt:1( + CallExpr:1( + NameExpr(f) + Args( + MemberExpr:1( + NameExpr(a) + ))))) + +[case testSyntaxError3] +f(a[]) +[out] +1:5: error: Expected index or slice expression +MypyFile:1( + ExpressionStmt:1( + CallExpr:1( + NameExpr(f) + Args( + IndexExpr:1( + NameExpr(a) + NameExpr()))))) + +[case testSyntaxError4] +from m import +1 +[out] +1:14: error: Expected one or more symbol names after import +MypyFile:1( + ImportFrom:1(m, []) + ExpressionStmt:2( + IntExpr(1))) + +[case testSyntaxError5] +from m +1 +[out] +1:7: error: Expected 'import', found newline +MypyFile:1( + ImportFrom:1(m, []) + ExpressionStmt:2( + IntExpr(1))) + +[case testSyntaxError6] +from m +1 +[out] +1:7: error: Expected 'import', found newline +MypyFile:1( + ImportFrom:1(m, []) + ExpressionStmt:2( + IntExpr(1))) From 89d6def92f2b9023f90326b54baabf0279110c16 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 24 Jan 2026 17:32:48 +0000 Subject: [PATCH 124/192] Adjust type ignore --- mypy/nativeparse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index e5601d7a06e49..8a769d9f02083 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -21,7 +21,7 @@ import os from typing import Any, Final, cast -import ast_serialize # type: ignore[import-untyped] +import ast_serialize # type: ignore[import-untyped, import-not-found, unused-ignore] from librt.internal import ( read_float as read_float_bare, read_int as read_int_bare, From 010b4435485f340fd796b7042c8cd4a5f4ad5a51 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 25 Jan 2026 14:36:43 +0000 Subject: [PATCH 125/192] Add test for error conditions --- test-data/unit/native-parser.test | 47 +++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 1e4e6240ff248..a446d7daa27f8 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -2531,3 +2531,50 @@ MypyFile:1( ImportFrom:1(m, []) ExpressionStmt:2( IntExpr(1))) + +[case testSyntaxError7] +def f(): + g( + h() + +def g(): ... +[out] +3:8: error: Expected ')', found newline +MypyFile:1( + FuncDef:1( + f + Block:2( + ExpressionStmt:2( + CallExpr:2( + NameExpr(g) + Args( + CallExpr:3( + NameExpr(h) + Args())))))) + FuncDef:5( + g + Block:5( + ExpressionStmt:5( + Ellipsis)))) + +[case testSyntaxError8] +def f( + x, + y + +def g(): ... +[out] +3:6: error: Expected ')', found newline +5:1: error: Expected an indented block after function definition +MypyFile:1( + FuncDef:1( + f + Args( + Var(x) + Var(y)) + Block:1()) + FuncDef:5( + g + Block:5( + ExpressionStmt:5( + Ellipsis)))) From 4a93c7f4226c6cf23174954c359fec41c993e281 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 25 Jan 2026 14:44:04 +0000 Subject: [PATCH 126/192] Propagate state param through parse functions --- mypy/nativeparse.py | 251 ++++++++++++++++++++++---------------------- 1 file changed, 128 insertions(+), 123 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 8a769d9f02083..2488fdf1ed9c6 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -135,6 +135,10 @@ _dummy_fallback: Final = Instance(MISSING_FALLBACK, [], -1) +class State: + pass + + def expect_end_tag(data: ReadBuffer) -> None: assert read_tag(data) == END_TAG @@ -156,18 +160,19 @@ def native_parse( b, errors, ignores = parse_to_binary_ast(filename, skip_function_bodies) data = ReadBuffer(b) n = read_int(data) - defs = read_statements(data, n) + state = State() + defs = read_statements(state, data, n) node = MypyFile(defs, []) node.path = filename return node, errors, ignores -def read_statements(data: ReadBuffer, n: int) -> list[Statement]: +def read_statements(state: State, data: ReadBuffer, n: int) -> list[Statement]: defs: list[Statement] = [] prev_func = False prev_name = "" for _ in range(n): - stmt = read_statement(data) + stmt = read_statement(state, data) if isinstance(stmt, (FuncDef, Decorator)): if prev_func and stmt.name == prev_name: # Merge into overloaded function definition @@ -194,18 +199,18 @@ def parse_to_binary_ast( return ast_serialize.parse(filename, skip_function_bodies) # type: ignore[no-any-return] -def read_statement(data: ReadBuffer) -> Statement: +def read_statement(state: State, data: ReadBuffer) -> Statement: tag = read_tag(data) stmt: Statement if tag == nodes.FUNC_DEF_STMT: - return read_func_def(data) + return read_func_def(state, data) elif tag == nodes.DECORATOR: expect_tag(data, LIST_GEN) n_decorators = read_int_bare(data) - decorators = [read_expression(data) for i in range(n_decorators)] + decorators = [read_expression(state, data) for i in range(n_decorators)] line = read_int(data) column = read_int(data) - fdef = read_statement(data) + fdef = read_statement(state, data) assert isinstance(fdef, FuncDef) fdef.is_decorated = True var = Var(fdef.name) @@ -221,7 +226,7 @@ def read_statement(data: ReadBuffer) -> Statement: expect_end_tag(data) return stmt elif tag == nodes.EXPR_STMT: - es = ExpressionStmt(read_expression(data)) + es = ExpressionStmt(read_expression(state, data)) es.line = es.expr.line es.column = es.expr.column es.end_line = es.expr.end_line @@ -229,12 +234,12 @@ def read_statement(data: ReadBuffer) -> Statement: expect_end_tag(data) return es elif tag == nodes.ASSIGNMENT_STMT: - lvalues = read_expression_list(data) - rvalue = read_expression(data) + lvalues = read_expression_list(state, data) + rvalue = read_expression(state, data) # Read type annotation has_type = read_bool(data) if has_type: - type_annotation = read_type(data) + type_annotation = read_type(state, data) else: type_annotation = None # Read new_syntax flag @@ -253,30 +258,30 @@ def read_statement(data: ReadBuffer) -> Statement: # Read operator string op = read_str(data) # Read lvalue (target) - lvalue = read_expression(data) + lvalue = read_expression(state, data) # Read rvalue (value) - rvalue = read_expression(data) + rvalue = read_expression(state, data) stmt = OperatorAssignmentStmt(op, lvalue, rvalue) read_loc(data, stmt) expect_end_tag(data) return stmt elif tag == nodes.IF_STMT: # Read the main if condition and body - expr = read_expression(data) - body = read_block(data) + expr = read_expression(state, data) + body = read_block(state, data) # Read elif clauses num_elif = read_int(data) elif_exprs = [] elif_bodies = [] for i in range(num_elif): - elif_exprs.append(read_expression(data)) - elif_bodies.append(read_block(data)) + elif_exprs.append(read_expression(state, data)) + elif_bodies.append(read_block(state, data)) # Read else clause has_else = read_bool(data) if has_else: - else_body = read_block(data) + else_body = read_block(state, data) else: else_body = None @@ -314,7 +319,7 @@ def read_statement(data: ReadBuffer) -> Statement: elif tag == nodes.RETURN_STMT: has_value = read_bool(data) if has_value: - value = read_expression(data) + value = read_expression(state, data) else: value = None stmt = ReturnStmt(value) @@ -325,13 +330,13 @@ def read_statement(data: ReadBuffer) -> Statement: # Read exception expression (optional) has_exc = read_bool(data) if has_exc: - exc = read_expression(data) + exc = read_expression(state, data) else: exc = None # Read from expression (optional) has_from = read_bool(data) if has_from: - from_expr = read_expression(data) + from_expr = read_expression(state, data) else: from_expr = None stmt = RaiseStmt(exc, from_expr) @@ -340,11 +345,11 @@ def read_statement(data: ReadBuffer) -> Statement: return stmt elif tag == nodes.ASSERT_STMT: # Read test expression - test = read_expression(data) + test = read_expression(state, data) # Read optional message expression has_msg = read_bool(data) if has_msg: - msg = read_expression(data) + msg = read_expression(state, data) else: msg = None stmt = AssertStmt(test, msg) @@ -352,22 +357,22 @@ def read_statement(data: ReadBuffer) -> Statement: expect_end_tag(data) return stmt elif tag == nodes.WHILE_STMT: - expr = read_expression(data) - body = read_block(data) - else_body = read_optional_block(data) + expr = read_expression(state, data) + body = read_block(state, data) + else_body = read_optional_block(state, data) stmt = WhileStmt(expr, body, else_body) read_loc(data, stmt) expect_end_tag(data) return stmt elif tag == nodes.FOR_STMT: # Read index (target) - index = read_expression(data) + index = read_expression(state, data) # Read iterator expression - expr = read_expression(data) + expr = read_expression(state, data) # Read body - body = read_block(data) + body = read_block(state, data) # Read else clause - else_body = read_optional_block(data) + else_body = read_optional_block(state, data) # Read is_async flag is_async = read_bool(data) stmt = ForStmt(index, expr, body, else_body) @@ -383,17 +388,17 @@ def read_statement(data: ReadBuffer) -> Statement: # Read each item for _ in range(n): # Read context expression - context_expr = read_expression(data) + context_expr = read_expression(state, data) expr_list.append(context_expr) # Read optional target has_target = read_bool(data) if has_target: - target = read_expression(data) + target = read_expression(state, data) target_list.append(target) else: target_list.append(None) # Read body - body = read_block(data) + body = read_block(state, data) # Read is_async flag is_async = read_bool(data) stmt = WithStmt(expr_list, target_list, body) @@ -471,12 +476,12 @@ def read_statement(data: ReadBuffer) -> Statement: expect_end_tag(data) return stmt elif tag == nodes.CLASS_DEF: - return read_class_def(data) + return read_class_def(state, data) elif tag == nodes.TRY_STMT: - return read_try_stmt(data) + return read_try_stmt(state, data) elif tag == nodes.DEL_STMT: # Read the target expression - expr = read_expression(data) + expr = read_expression(state, data) stmt = DelStmt(expr) read_loc(data, stmt) expect_end_tag(data) @@ -505,7 +510,7 @@ def read_statement(data: ReadBuffer) -> Statement: assert False, tag -def read_parameters(data: ReadBuffer) -> tuple[list[Argument], bool]: +def read_parameters(state: State, data: ReadBuffer) -> tuple[list[Argument], bool]: """Read function/lambda parameters from the buffer. Returns: @@ -523,14 +528,14 @@ def read_parameters(data: ReadBuffer) -> tuple[list[Argument], bool]: # Read type annotation has_type = read_bool(data) if has_type: - ann = read_type(data) + ann = read_type(state, data) has_ann = True else: ann = None # Read default value has_default = read_bool(data) if has_default: - default = read_expression(data) + default = read_expression(state, data) else: default = None pos_only = read_bool(data) @@ -547,14 +552,14 @@ def read_parameters(data: ReadBuffer) -> tuple[list[Argument], bool]: return arguments, has_ann -def read_func_def(data: ReadBuffer) -> FuncDef: +def read_func_def(state: State, data: ReadBuffer) -> FuncDef: # Function name name = read_str(data) # Parameters - arguments, has_ann = read_parameters(data) + arguments, has_ann = read_parameters(state, data) - body = read_block(data) + body = read_block(state, data) is_async = read_bool(data) @@ -565,7 +570,7 @@ def read_func_def(data: ReadBuffer) -> FuncDef: # TODO: Return type annotation has_return_type = read_bool(data) if has_return_type: - return_type = read_type(data) + return_type = read_type(state, data) has_ann = True else: return_type = None @@ -592,20 +597,20 @@ def read_func_def(data: ReadBuffer) -> FuncDef: return func_def -def read_class_def(data: ReadBuffer) -> ClassDef: +def read_class_def(state: State, data: ReadBuffer) -> ClassDef: # Class name name = read_str(data) # Body - body = read_block(data) + body = read_block(state, data) # Base classes - base_type_exprs = read_expression_list(data) + base_type_exprs = read_expression_list(state, data) # Decorators expect_tag(data, LIST_GEN) n_decorators = read_int_bare(data) - decorators = [read_expression(data) for _ in range(n_decorators)] + decorators = [read_expression(state, data) for _ in range(n_decorators)] # TODO: Type parameters (skip for now) has_type_params = read_bool(data) @@ -617,7 +622,7 @@ def read_class_def(data: ReadBuffer) -> ClassDef: keywords = [] for _ in range(n_keywords): key = read_str(data) - value = read_expression(data) + value = read_expression(state, data) keywords.append((key, value)) # Extract metaclass from keywords if present @@ -638,9 +643,9 @@ def read_class_def(data: ReadBuffer) -> ClassDef: return class_def -def read_try_stmt(data: ReadBuffer) -> TryStmt: +def read_try_stmt(state: State, data: ReadBuffer) -> TryStmt: # Read try body - body = read_block(data) + body = read_block(state, data) # Read number of except handlers num_handlers = read_int(data) @@ -650,7 +655,7 @@ def read_try_stmt(data: ReadBuffer) -> TryStmt: for _ in range(num_handlers): has_type = read_bool(data) if has_type: - exc_type = read_expression(data) + exc_type = read_expression(state, data) types_list.append(exc_type) else: types_list.append(None) @@ -669,20 +674,20 @@ def read_try_stmt(data: ReadBuffer) -> TryStmt: # Read handler bodies handlers = [] for _ in range(num_handlers): - handler_body = read_block(data) + handler_body = read_block(state, data) handlers.append(handler_body) # Read else body (optional) has_else = read_bool(data) if has_else: - else_body = read_block(data) + else_body = read_block(state, data) else: else_body = None # Read finally body (optional) has_finally = read_bool(data) if has_finally: - finally_body = read_block(data) + finally_body = read_block(state, data) else: finally_body = None @@ -692,13 +697,13 @@ def read_try_stmt(data: ReadBuffer) -> TryStmt: return stmt -def read_type(data: ReadBuffer) -> Type: +def read_type(state: State, data: ReadBuffer) -> Type: tag = read_tag(data) if tag == types.UNBOUND_TYPE: name = read_str(data) expect_tag(data, LIST_GEN) n = read_int_bare(data) - args = tuple(read_type(data) for i in range(n)) + args = tuple(read_type(state, data) for i in range(n)) # Read optional original_str_expr t = read_tag(data) if t == LITERAL_NONE: @@ -728,7 +733,7 @@ def read_type(data: ReadBuffer) -> Type: # Read items list expect_tag(data, LIST_GEN) n = read_int_bare(data) - items = [read_type(data) for i in range(n)] + items = [read_type(state, data) for i in range(n)] # Read uses_pep604_syntax flag uses_pep604_syntax = read_bool(data) # Read optional original_str_expr @@ -757,7 +762,7 @@ def read_type(data: ReadBuffer) -> Type: # Read items list expect_tag(data, LIST_GEN) n = read_int_bare(data) - items = [read_type(data) for i in range(n)] + items = [read_type(state, data) for i in range(n)] type_list = TypeList(items) read_loc(data, type_list) expect_end_tag(data) @@ -787,18 +792,18 @@ def read_type(data: ReadBuffer) -> Type: expect_end_tag(data) return raw_type elif tag == types.UNPACK_TYPE: - inner_type = read_type(data) + inner_type = read_type(state, data) unpack = UnpackType(inner_type) read_loc(data, unpack) expect_end_tag(data) return unpack elif tag == types.CALL_TYPE: - return read_call_type(data) + return read_call_type(state, data) else: assert False, tag -def read_block(data: ReadBuffer) -> Block: +def read_block(state: State, data: ReadBuffer) -> Block: expect_tag(data, nodes.BLOCK) expect_tag(data, LIST_GEN) n = read_int_bare(data) @@ -810,7 +815,7 @@ def read_block(data: ReadBuffer) -> Block: return b else: # Non-empty block - read statements and set location from them - a = read_statements(data, n) + a = read_statements(state, data, n) expect_end_tag(data) b = Block(a) b.line = a[0].line @@ -820,14 +825,14 @@ def read_block(data: ReadBuffer) -> Block: return b -def read_optional_block(data: ReadBuffer) -> Block | None: +def read_optional_block(state: State, data: ReadBuffer) -> Block | None: expect_tag(data, nodes.BLOCK) expect_tag(data, LIST_GEN) n = read_int_bare(data) if n == 0: b = None else: - a = [read_statement(data) for i in range(n)] + a = [read_statement(state, data) for i in range(n)] b = Block(a) b.line = a[0].line b.column = a[0].column @@ -843,12 +848,12 @@ def read_optional_block(data: ReadBuffer) -> Block | None: unary_ops: Final = ["~", "not", "+", "-"] -def read_expression(data: ReadBuffer) -> Expression: +def read_expression(state: State, data: ReadBuffer) -> Expression: tag = read_tag(data) expr: Expression if tag == nodes.CALL_EXPR: - callee = read_expression(data) - args = read_expression_list(data) + callee = read_expression(state, data) + args = read_expression_list(state, data) # Read argument kinds expect_tag(data, LIST_INT) n_kinds = read_int_bare(data) @@ -876,7 +881,7 @@ def read_expression(data: ReadBuffer) -> Expression: expect_end_tag(data) return ne elif tag == nodes.MEMBER_EXPR: - e = read_expression(data) + e = read_expression(state, data) attr = read_str(data) m = MemberExpr(e, attr) # Check if this is a super() call - if so, convert to SuperExpr @@ -905,30 +910,30 @@ def read_expression(data: ReadBuffer) -> Expression: expect_end_tag(data) return fe elif tag == nodes.LIST_EXPR: - items = read_expression_list(data) + items = read_expression_list(state, data) expr = ListExpr(items) read_loc(data, expr) expect_end_tag(data) return expr elif tag == nodes.TUPLE_EXPR: - items = read_expression_list(data) + items = read_expression_list(state, data) t = TupleExpr(items) read_loc(data, t) expect_end_tag(data) return t elif tag == nodes.SET_EXPR: - items = read_expression_list(data) + items = read_expression_list(state, data) expr = SetExpr(items) read_loc(data, expr) expect_end_tag(data) return expr elif tag == nodes.GENERATOR_EXPR: - expr = read_generator_expr(data) + expr = read_generator_expr(state, data) read_loc(data, expr) expect_end_tag(data) return expr elif tag == nodes.LIST_COMPREHENSION: - generator = read_generator_expr(data) + generator = read_generator_expr(state, data) expr = ListComprehension(generator) read_loc(data, expr) # Also copy location to the inner generator @@ -939,7 +944,7 @@ def read_expression(data: ReadBuffer) -> Expression: expect_end_tag(data) return expr elif tag == nodes.SET_COMPREHENSION: - generator = read_generator_expr(data) + generator = read_generator_expr(state, data) expr = SetComprehension(generator) read_loc(data, expr) # Also copy location to the inner generator @@ -951,17 +956,17 @@ def read_expression(data: ReadBuffer) -> Expression: return expr elif tag == nodes.DICT_COMPREHENSION: # Read key expression - key = read_expression(data) + key = read_expression(state, data) # Read value expression - value = read_expression(data) + value = read_expression(state, data) # Read number of generators n_generators = read_int(data) # Read all indices (targets) - indices = [read_expression(data) for _ in range(n_generators)] + indices = [read_expression(state, data) for _ in range(n_generators)] # Read all sequences (iters) - sequences = [read_expression(data) for _ in range(n_generators)] + sequences = [read_expression(state, data) for _ in range(n_generators)] # Read all condlists (ifs for each generator) - condlists = [read_expression_list(data) for _ in range(n_generators)] + condlists = [read_expression_list(state, data) for _ in range(n_generators)] # Read all is_async flags is_async = [read_bool(data) for _ in range(n_generators)] expr = DictionaryComprehension(key, value, indices, sequences, condlists, is_async) @@ -972,7 +977,7 @@ def read_expression(data: ReadBuffer) -> Expression: # Read optional value expression has_value = read_bool(data) if has_value: - value = read_expression(data) + value = read_expression(state, data) else: value = None expr = YieldExpr(value) @@ -981,15 +986,15 @@ def read_expression(data: ReadBuffer) -> Expression: return expr elif tag == nodes.YIELD_FROM_EXPR: # Read value expression (required for yield from) - value = read_expression(data) + value = read_expression(state, data) expr = YieldFromExpr(value) read_loc(data, expr) expect_end_tag(data) return expr elif tag == nodes.OP_EXPR: op = bin_ops[read_int(data)] - left = read_expression(data) - right = read_expression(data) + left = read_expression(state, data) + right = read_expression(state, data) o = OpExpr(op, left, right) # TODO: Store these explicitly? o.line = left.line @@ -999,15 +1004,15 @@ def read_expression(data: ReadBuffer) -> Expression: expect_end_tag(data) return o elif tag == nodes.INDEX_EXPR: - base = read_expression(data) - index = read_expression(data) + base = read_expression(state, data) + index = read_expression(state, data) expr = IndexExpr(base, index) read_loc(data, expr) expect_end_tag(data) return expr elif tag == nodes.BOOL_OP_EXPR: op = bool_ops[read_int(data)] - values = read_expression_list(data) + values = read_expression_list(state, data) # Convert list of values to nested OpExpr nodes # E.g., [a, b, c] with "and" becomes OpExpr("and", OpExpr("and", a, b), c) assert len(values) >= 2 @@ -1022,13 +1027,13 @@ def read_expression(data: ReadBuffer) -> Expression: expect_end_tag(data) return result elif tag == nodes.COMPARISON_EXPR: - left = read_expression(data) + left = read_expression(state, data) # Read operators list expect_tag(data, LIST_INT) n_ops = read_int_bare(data) ops = [cmp_ops[read_int_bare(data)] for _ in range(n_ops)] # Read comparators list - comparators = read_expression_list(data) + comparators = read_expression_list(state, data) assert len(ops) == len(comparators) expr = ComparisonExpr(ops, [left] + comparators) read_loc(data, expr) @@ -1036,7 +1041,7 @@ def read_expression(data: ReadBuffer) -> Expression: return expr elif tag == nodes.UNARY_EXPR: op = unary_ops[read_int(data)] - operand = read_expression(data) + operand = read_expression(state, data) expr = UnaryExpr(op, operand) read_loc(data, expr) expect_end_tag(data) @@ -1049,11 +1054,11 @@ def read_expression(data: ReadBuffer) -> Expression: for _ in range(n_keys): has_key = read_bool(data) if has_key: - keys.append(read_expression(data)) + keys.append(read_expression(state, data)) else: keys.append(None) # Read values - values = read_expression_list(data) + values = read_expression_list(state, data) # Zip keys and values into items items = list(zip(keys, values)) expr = DictExpr(items) @@ -1076,13 +1081,13 @@ def read_expression(data: ReadBuffer) -> Expression: elif tag == nodes.SLICE_EXPR: # Read begin_index (lower in Ruff) has_begin = read_bool(data) - begin_index = read_expression(data) if has_begin else None + begin_index = read_expression(state, data) if has_begin else None # Read end_index (upper in Ruff) has_end = read_bool(data) - end_index = read_expression(data) if has_end else None + end_index = read_expression(state, data) if has_end else None # Read stride (step in Ruff) has_stride = read_bool(data) - stride = read_expression(data) if has_stride else None + stride = read_expression(state, data) if has_stride else None expr = SliceExpr(begin_index, end_index, stride) read_loc(data, expr) expect_end_tag(data) @@ -1099,11 +1104,11 @@ def read_expression(data: ReadBuffer) -> Expression: return expr elif tag == nodes.CONDITIONAL_EXPR: # Read if_expr (value when condition is true) - if_expr = read_expression(data) + if_expr = read_expression(state, data) # Read cond (the condition) - cond = read_expression(data) + cond = read_expression(state, data) # Read else_expr (value when condition is false) - else_expr = read_expression(data) + else_expr = read_expression(state, data) expr = ConditionalExpr(cond, if_expr, else_expr) read_loc(data, expr) expect_end_tag(data) @@ -1118,20 +1123,20 @@ def read_expression(data: ReadBuffer) -> Expression: if b: n = read_int(data) for i in range(n): - fitems.append(read_fstring_item(data)) + fitems.append(read_fstring_item(state, data)) else: s = StrExpr(read_str(data)) read_loc(data, s) fitems.append(s) - expr = build_fstring_join(data, fitems) + expr = build_fstring_join(state, data, fitems) expect_end_tag(data) return expr elif tag == nodes.LAMBDA_EXPR: # Read parameters - arguments, has_ann = read_parameters(data) + arguments, has_ann = read_parameters(state, data) # Read body block - body = read_block(data) + body = read_block(state, data) # Create lambda expression if has_ann: @@ -1155,9 +1160,9 @@ def read_expression(data: ReadBuffer) -> Expression: return expr elif tag == nodes.NAMED_EXPR: # Read target expression - target = read_expression(data) + target = read_expression(state, data) # Read value expression - value = read_expression(data) + value = read_expression(state, data) # AssignmentExpr expects target to be a NameExpr if not isinstance(target, NameExpr): # In case target is not a NameExpr, we need to handle this @@ -1171,7 +1176,7 @@ def read_expression(data: ReadBuffer) -> Expression: return expr elif tag == nodes.STAR_EXPR: # Read the wrapped expression - wrapped_expr = read_expression(data) + wrapped_expr = read_expression(state, data) expr = StarExpr(wrapped_expr) read_loc(data, expr) expect_end_tag(data) @@ -1185,7 +1190,7 @@ def read_expression(data: ReadBuffer) -> Expression: return expr elif tag == nodes.AWAIT_EXPR: # Read awaited expression - value = read_expression(data) + value = read_expression(state, data) expr = AwaitExpr(value) read_loc(data, expr) expect_end_tag(data) @@ -1200,14 +1205,14 @@ def read_expression(data: ReadBuffer) -> Expression: assert False, tag -def read_fstring_items(data: ReadBuffer) -> Expression: +def read_fstring_items(state: State, data: ReadBuffer) -> Expression: items = [] n = read_int(data) - items = [read_fstring_item(data) for i in range(n)] - return build_fstring_join(data, items) + items = [read_fstring_item(state, data) for i in range(n)] + return build_fstring_join(state, data, items) -def build_fstring_join(data: ReadBuffer, items: list[Expression]) -> Expression: +def build_fstring_join(state: State, data: ReadBuffer, items: list[Expression]) -> Expression: if len(items) == 1: expr = items[0] read_loc(data, expr) @@ -1228,14 +1233,14 @@ def build_fstring_join(data: ReadBuffer, items: list[Expression]) -> Expression: return call -def read_fstring_item(data: ReadBuffer) -> Expression: +def read_fstring_item(state: State, data: ReadBuffer) -> Expression: t = read_tag(data) if t == LITERAL_STR: str_expr = StrExpr(read_str_bare(data)) read_loc(data, str_expr) return str_expr elif t == nodes.FSTRING_INTERPOLATION: - expr = read_expression(data) + expr = read_expression(state, data) # Read conversion flag such as !r has_conv = read_bool(data) @@ -1248,7 +1253,7 @@ def read_fstring_item(data: ReadBuffer) -> Expression: # Read format spec such as <30 (which may have nested {...}) has_spec = read_bool(data) if has_spec: - spec = read_fstring_items(data) + spec = read_fstring_items(state, data) else: spec = StrExpr("") @@ -1267,24 +1272,24 @@ def set_line_column(target: Context, src: Context) -> None: target.column = src.column -def read_expression_list(data: ReadBuffer) -> list[Expression]: +def read_expression_list(state: State, data: ReadBuffer) -> list[Expression]: expect_tag(data, LIST_GEN) n = read_int_bare(data) - return [read_expression(data) for i in range(n)] + return [read_expression(state, data) for i in range(n)] -def read_generator_expr(data: ReadBuffer) -> GeneratorExpr: +def read_generator_expr(state: State, data: ReadBuffer) -> GeneratorExpr: """Helper function to read comprehension data (shared by Generator, ListComp, SetComp)""" # Read element expression - left_expr = read_expression(data) + left_expr = read_expression(state, data) # Read number of generators n_generators = read_int(data) # Read all indices (targets) - indices = [read_expression(data) for _ in range(n_generators)] + indices = [read_expression(state, data) for _ in range(n_generators)] # Read all sequences (iters) - sequences = [read_expression(data) for _ in range(n_generators)] + sequences = [read_expression(state, data) for _ in range(n_generators)] # Read all condlists (ifs for each generator) - condlists = [read_expression_list(data) for _ in range(n_generators)] + condlists = [read_expression_list(state, data) for _ in range(n_generators)] # Read all is_async flags is_async = [read_bool(data) for _ in range(n_generators)] return GeneratorExpr(left_expr, indices, sequences, condlists, is_async) @@ -1321,7 +1326,7 @@ def extract_arg_name(typ: Type) -> str | None: return None # Invalid, but let validation handle it -def read_call_type(data: ReadBuffer) -> Type: +def read_call_type(state: State, data: ReadBuffer) -> Type: """Read Call in type context - check if it's an Arg/DefaultArg/VarArg/KwArg constructor. TODO: This deserialization is lenient and defers error handling to type analysis. @@ -1334,12 +1339,12 @@ def read_call_type(data: ReadBuffer) -> Type: - Invalid argument name validation: When name is not a string literal or None """ # Read callee - callee_type = read_type(data) + callee_type = read_type(state, data) # Read positional arguments expect_tag(data, LIST_GEN) n_args = read_int_bare(data) - args = [read_type(data) for _ in range(n_args)] + args = [read_type(state, data) for _ in range(n_args)] # Read keyword arguments expect_tag(data, LIST_GEN) @@ -1353,7 +1358,7 @@ def read_call_type(data: ReadBuffer) -> Type: kw_name = read_str_bare(data) else: assert False, f"Unexpected tag for keyword name: {tag_kw}" - kw_value = read_type(data) + kw_value = read_type(state, data) kwargs.append((kw_name, kw_value)) # Try to detect Arg/DefaultArg/VarArg/KwArg pattern From e7fb3e32b77e20513c3d1586ef01a29b8540c07b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 25 Jan 2026 14:53:09 +0000 Subject: [PATCH 127/192] Add syntax error test case --- test-data/unit/native-parser.test | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index a446d7daa27f8..8d914eb190289 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -2578,3 +2578,18 @@ MypyFile:1( Block:5( ExpressionStmt:5( Ellipsis)))) + +[case testSyntaxError9] +class A + +def f(): pass +[out] +1:8: error: Expected ':', found newline +3:1: error: Expected an indented block after `class` definition +MypyFile:1( + ClassDef:1( + A) + FuncDef:3( + f + Block:3( + PassStmt:3()))) \ No newline at end of file From 89e1ec26de7cee4b18812959b12716b10248aff9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 25 Jan 2026 15:10:52 +0000 Subject: [PATCH 128/192] Check callable expression types for errors --- mypy/nativeparse.py | 125 ++++++++++++++++++------------ test-data/unit/native-parser.test | 111 +++++++++++++++++++++++++- 2 files changed, 186 insertions(+), 50 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 2488fdf1ed9c6..33acb5d46f6c6 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -28,7 +28,7 @@ read_str as read_str_bare, ) -from mypy import nodes, types +from mypy import message_registry, nodes, types from mypy.cache import ( DICT_STR_GEN, END_TAG, @@ -136,7 +136,12 @@ class State: - pass + def __init__(self) -> None: + self.errors: list[dict[str, Any]] = [] + + def add_error(self, message: str, line: int, column: int) -> None: + """Report an error at a specific location.""" + self.errors.append({"line": line, "column": column, "message": message}) def expect_end_tag(data: ReadBuffer) -> None: @@ -164,7 +169,9 @@ def native_parse( defs = read_statements(state, data, n) node = MypyFile(defs, []) node.path = filename - return node, errors, ignores + # Merge deserialization errors with parsing errors + all_errors = errors + state.errors + return node, all_errors, ignores def read_statements(state: State, data: ReadBuffer, n: int) -> list[Statement]: @@ -1329,14 +1336,7 @@ def extract_arg_name(typ: Type) -> str | None: def read_call_type(state: State, data: ReadBuffer) -> Type: """Read Call in type context - check if it's an Arg/DefaultArg/VarArg/KwArg constructor. - TODO: This deserialization is lenient and defers error handling to type analysis. - The following validations are missing (see mypy/fastparse.py:1948-1999 for reference): - - ARG_CONSTRUCTOR_NAME_EXPECTED: When constructor name cannot be extracted - - ARG_CONSTRUCTOR_TOO_MANY_ARGS: When more than 2 positional arguments - - MULTIPLE_VALUES_FOR_NAME_KWARG: When 'name' is specified multiple times - - MULTIPLE_VALUES_FOR_TYPE_KWARG: When 'type' is specified multiple times - - ARG_CONSTRUCTOR_UNEXPECTED_ARG: When keyword arg is not 'name' or 'type' - - Invalid argument name validation: When name is not a string literal or None + This performs validation and error reporting similar to mypy/fastparse.py. """ # Read callee callee_type = read_type(state, data) @@ -1364,44 +1364,71 @@ def read_call_type(state: State, data: ReadBuffer) -> Type: # Try to detect Arg/DefaultArg/VarArg/KwArg pattern constructor = stringify_type_name(callee_type) - if constructor: - # Extract type and name from arguments - name: str | None = None - default_type = AnyType(TypeOfAny.special_form) - typ: Type = default_type - - # Process positional arguments - for i, arg in enumerate(args): - if i == 0: - typ = arg - elif i == 1: - name = extract_arg_name(arg) - # TODO: Emit error for more than 2 positional args (ARG_CONSTRUCTOR_TOO_MANY_ARGS) - - # Process keyword arguments - for kw_name, kw_value in kwargs: - if kw_name == "name": - # TODO: Emit error if name already set (MULTIPLE_VALUES_FOR_NAME_KWARG) - if name is not None: - pass # Duplicate, but defer to type analysis - name = extract_arg_name(kw_value) - elif kw_name == "type": - # TODO: Emit error if type already set (MULTIPLE_VALUES_FOR_TYPE_KWARG) - if typ is not default_type: - pass # Duplicate, but defer to type analysis - typ = kw_value - # TODO: Emit error for unexpected keyword arg (ARG_CONSTRUCTOR_UNEXPECTED_ARG) - - # Create CallableArgument - call_arg = CallableArgument(typ, name, constructor) - read_loc(data, call_arg) - expect_end_tag(data) - return call_arg - - # TODO: Emit error when constructor name is None (ARG_CONSTRUCTOR_NAME_EXPECTED) - # If not a valid constructor, fall back to creating an invalid type - # (validation will report error later) + # We'll read location before processing errors so we can report them correctly invalid = AnyType(TypeOfAny.from_error) read_loc(data, invalid) expect_end_tag(data) - return invalid + + if not constructor: + # ARG_CONSTRUCTOR_NAME_EXPECTED + state.add_error( + message_registry.ARG_CONSTRUCTOR_NAME_EXPECTED.value, invalid.line, invalid.column + ) + return invalid + + # Extract type and name from arguments + name: str | None = None + name_set_from_positional = False + default_type = AnyType(TypeOfAny.special_form) + typ: Type = default_type + typ_set_from_positional = False + + # Process positional arguments + for i, arg in enumerate(args): + if i == 0: + typ = arg + typ_set_from_positional = True + elif i == 1: + name = extract_arg_name(arg) + name_set_from_positional = True + else: + # ARG_CONSTRUCTOR_TOO_MANY_ARGS + state.add_error( + message_registry.ARG_CONSTRUCTOR_TOO_MANY_ARGS.value, invalid.line, invalid.column + ) + + # Process keyword arguments + for kw_name, kw_value in kwargs: + if kw_name == "name": + # MULTIPLE_VALUES_FOR_NAME_KWARG + if name is not None and name_set_from_positional: + state.add_error( + message_registry.MULTIPLE_VALUES_FOR_NAME_KWARG.format(constructor).value, + invalid.line, + invalid.column, + ) + name = extract_arg_name(kw_value) + elif kw_name == "type": + # MULTIPLE_VALUES_FOR_TYPE_KWARG + if typ is not default_type and typ_set_from_positional: + state.add_error( + message_registry.MULTIPLE_VALUES_FOR_TYPE_KWARG.format(constructor).value, + invalid.line, + invalid.column, + ) + typ = kw_value + else: + # ARG_CONSTRUCTOR_UNEXPECTED_ARG + state.add_error( + message_registry.ARG_CONSTRUCTOR_UNEXPECTED_ARG.format(kw_name).value, + invalid.line, + invalid.column, + ) + + # Create CallableArgument + call_arg = CallableArgument(typ, name, constructor) + call_arg.line = invalid.line + call_arg.column = invalid.column + call_arg.end_line = invalid.end_line + call_arg.end_column = invalid.end_column + return call_arg diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 8d914eb190289..1176b6ac35420 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -2592,4 +2592,113 @@ MypyFile:1( FuncDef:3( f Block:3( - PassStmt:3()))) \ No newline at end of file + PassStmt:3()))) + +[case testArgConstructorTooManyArgs] +from typing import Callable +from mypy_extensions import Arg +def f(x: Callable[[Arg(int, 'x', 'extra')], bool]) -> None: + pass +[out] +3:19: error: Too many arguments for argument constructor +MypyFile:1( + ImportFrom:1(typing, [Callable]) + ImportFrom:2(mypy_extensions, [Arg]) + FuncDef:3( + f + Args( + Var(x)) + def (x: Callable?[, bool?]) -> None? + Block:4( + PassStmt:4()))) + +[case testArgConstructorMultipleValuesForName] +from typing import Callable +from mypy_extensions import Arg +def f(x: Callable[[Arg(int, 'x', name='y')], bool]) -> None: + pass +[out] +3:19: error: "Arg" gets multiple values for keyword argument "name" +MypyFile:1( + ImportFrom:1(typing, [Callable]) + ImportFrom:2(mypy_extensions, [Arg]) + FuncDef:3( + f + Args( + Var(x)) + def (x: Callable?[, bool?]) -> None? + Block:4( + PassStmt:4()))) + +[case testArgConstructorMultipleValuesForType] +from typing import Callable +from mypy_extensions import Arg +def f(x: Callable[[Arg(int, type=str)], bool]) -> None: + pass +[out] +3:19: error: "Arg" gets multiple values for keyword argument "type" +MypyFile:1( + ImportFrom:1(typing, [Callable]) + ImportFrom:2(mypy_extensions, [Arg]) + FuncDef:3( + f + Args( + Var(x)) + def (x: Callable?[, bool?]) -> None? + Block:4( + PassStmt:4()))) + +[case testArgConstructorUnexpectedArg] +from typing import Callable +from mypy_extensions import Arg +def f(x: Callable[[Arg(int, foo='bar')], bool]) -> None: + pass +[out] +3:19: error: Unexpected argument "foo" for argument constructor +MypyFile:1( + ImportFrom:1(typing, [Callable]) + ImportFrom:2(mypy_extensions, [Arg]) + FuncDef:3( + f + Args( + Var(x)) + def (x: Callable?[, bool?]) -> None? + Block:4( + PassStmt:4()))) + +[case testArgConstructorMultipleErrors] +from typing import Callable +from mypy_extensions import Arg +def f(x: Callable[[Arg(int, 'x', 'extra', foo='bar', name='y')], bool]) -> None: + pass +[out] +3:19: error: Too many arguments for argument constructor +3:19: error: Unexpected argument "foo" for argument constructor +3:19: error: "Arg" gets multiple values for keyword argument "name" +MypyFile:1( + ImportFrom:1(typing, [Callable]) + ImportFrom:2(mypy_extensions, [Arg]) + FuncDef:3( + f + Args( + Var(x)) + def (x: Callable?[, bool?]) -> None? + Block:4( + PassStmt:4()))) + +[case testArgConstructorValidSyntax] +from typing import Callable +from mypy_extensions import Arg, DefaultArg, VarArg, KwArg +def f(x: Callable[[Arg(int, 'x'), DefaultArg(str, 'y'), VarArg(bool), KwArg(float)], None]) -> None: + pass +[out] +MypyFile:1( + ImportFrom:1(typing, [Callable]) + ImportFrom:2(mypy_extensions, [Arg, DefaultArg, VarArg, KwArg]) + FuncDef:3( + f + Args( + Var(x)) + def (x: Callable?[, None?]) -> None? + Block:4( + PassStmt:4()))) \ No newline at end of file From 0b56ba74afde031209db1f9dcc34ab13dc1fe01e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 25 Jan 2026 15:57:59 +0000 Subject: [PATCH 129/192] Support invalid type annotations --- mypy/nativeparse.py | 7 +- test-data/unit/native-parser.test | 121 +++++++++++++++++++++++++++++- 2 files changed, 126 insertions(+), 2 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 33acb5d46f6c6..82c237fbf3f9e 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -782,7 +782,7 @@ def read_type(state: State, data: ReadBuffer) -> Type: return ellipsis_type elif tag == types.RAW_EXPRESSION_TYPE: type_name = read_str(data) - value: types.LiteralValue | str + value: types.LiteralValue | str | None if type_name == "builtins.bool": value = read_bool(data) elif type_name == "builtins.int": @@ -792,6 +792,11 @@ def read_type(state: State, data: ReadBuffer) -> Type: elif type_name == "builtins.bytes": # Bytes literals are serialized as escaped strings value = read_str(data) + elif type_name == "typing.Any": + # Invalid type - read None value + tag = read_tag(data) + assert tag == types.LITERAL_NONE, f"Expected LITERAL_NONE for invalid type, got {tag}" + value = None else: assert False, f"Unsupported RawExpressionType: {type_name}" raw_type = RawExpressionType(value, type_name) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 1176b6ac35420..4fad9544f443a 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -2701,4 +2701,123 @@ MypyFile:1( Var(x)) def (x: Callable?[, None?]) -> None? Block:4( - PassStmt:4()))) \ No newline at end of file + PassStmt:4()))) +[case testInvalidTypeFloatLiteral] +x: 3.14 +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(x) + TempNode:1( + Any) + None)) + +[case testInvalidTypeComplexLiteral] +y: 1j +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(y) + TempNode:1( + Any) + None)) + +[case testInvalidTypeBinaryOperatorAdd] +z: int + str +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(z) + TempNode:1( + Any) + None)) + +[case testInvalidTypeBinaryOperatorMul] +a: int * str +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(a) + TempNode:1( + Any) + None)) + +[case testInvalidTypeUnaryOperatorNot] +b: not int +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(b) + TempNode:1( + Any) + None)) + +[case testInvalidTypeUnaryOperatorInvert] +c: ~int +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(c) + TempNode:1( + Any) + None)) + +[case testInvalidTypeSetExpression] +f: {1, 2, 3} +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(f) + TempNode:1( + Any) + None)) + +[case testInvalidTypeListComprehension] +g: [x for x in range(10)] +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(g) + TempNode:1( + Any) + None)) + +[case testInvalidTypeLambda] +h: lambda x: x +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(h) + TempNode:1( + Any) + None)) + +[case testInvalidTypeIfExpression] +i: int if True else str +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(i) + TempNode:1( + Any) + None)) + +[case testInvalidTypeComparisonExpression] +j: int < str +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(j) + TempNode:1( + Any) + None)) + +[case testInvalidTypePositiveUnaryOp] +k: +5 +[out] +MypyFile:1( + AssignmentStmt:1( + NameExpr(k) + TempNode:1( + Any) + 5)) From d57da8db292ff4fcbae71babe0fc6330293e0ea7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 25 Jan 2026 16:04:03 +0000 Subject: [PATCH 130/192] Remove temporary test case --- test-data/unit/parse.test | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test index 4bd3707a8a128..a77a3c61c876e 100644 --- a/test-data/unit/parse.test +++ b/test-data/unit/parse.test @@ -3878,25 +3878,3 @@ MypyFile:1( Args( Var(self)) Block:7())))) - -[case testXXX] -from functools import total_ordering -from typing import Any, Callable, ClassVar - -@total_ordering -class Ord: - __eq__: Callable[[Any, object], bool] = lambda self, other: False - __lt__: Callable[[Any, "Ord"], bool] = lambda self, other: False - -reveal_type(Ord() < Ord()) # N: Revealed type is "builtins.bool" -reveal_type(Ord() <= Ord()) # N: Revealed type is "builtins.bool" -reveal_type(Ord() == Ord()) # N: Revealed type is "builtins.bool" -reveal_type(Ord() > Ord()) # N: Revealed type is "builtins.bool" -reveal_type(Ord() >= Ord()) # N: Revealed type is "builtins.bool" - -Ord() < 1 # E: Argument 1 has incompatible type "int"; expected "Ord" -Ord() <= 1 # E: Unsupported operand types for <= ("Ord" and "int") -Ord() == 1 -Ord() > 1 # E: Unsupported operand types for > ("Ord" and "int") -Ord() >= 1 # E: Unsupported operand types for >= ("Ord" and "int") -[out] \ No newline at end of file From b7a9b761c7fb483e6fd87d3ae16eb700cfb99f4b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 25 Jan 2026 16:55:52 +0000 Subject: [PATCH 131/192] Update function param Var node compatibility --- mypy/nativeparse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 82c237fbf3f9e..f6d8a4417e1af 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -547,7 +547,8 @@ def read_parameters(state: State, data: ReadBuffer) -> tuple[list[Argument], boo default = None pos_only = read_bool(data) - var = Var(arg_name) + var = Var(arg_name, ann) + var.is_inferred = False arg = Argument(var, ann, default, arg_kind, pos_only) read_loc(data, arg) var.line = arg.line From 05716ea480f8c84c05f10a83f52481ffaaa3eb61 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 25 Jan 2026 16:56:10 +0000 Subject: [PATCH 132/192] Set more function type annotation attributes --- mypy/nativeparse.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index f6d8a4417e1af..3ac25406351e4 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -598,6 +598,12 @@ def read_func_def(state: State, data: ReadBuffer) -> FuncDef: typ = None func_def = FuncDef(name, arguments, body, typ=typ) + if typ: + # TODO: This seems wasteful, can we avoid it? + func_def.unanalyzed_type = typ.copy_modified() + + typ.definition = func_def + typ.line = func_def.line if is_async: func_def.is_coroutine = True read_loc(data, func_def) From 05f51f3bf02780c098fd16586880f0c3adf6a3fc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 26 Jan 2026 20:42:58 +0000 Subject: [PATCH 133/192] Make special dunder params positional only --- mypy/nativeparse.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 3ac25406351e4..58d19aad29753 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -29,6 +29,7 @@ ) from mypy import message_registry, nodes, types +from mypy.sharedparse import special_function_elide_names from mypy.cache import ( DICT_STR_GEN, END_TAG, @@ -567,6 +568,10 @@ def read_func_def(state: State, data: ReadBuffer) -> FuncDef: # Parameters arguments, has_ann = read_parameters(state, data) + if special_function_elide_names(name): + for arg in arguments: + arg.pos_only = True + body = read_block(state, data) is_async = read_bool(data) From ab52bae648e0897191e10d0e2aa3292bccd9445d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Jan 2026 15:28:53 +0000 Subject: [PATCH 134/192] FIXME enable --native-parser in testcheck.py --- mypy/test/testcheck.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index fb36f67ca1191..494d0f0f7e052 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -136,6 +136,7 @@ def run_case_once( options = parse_options(original_program_text, testcase, incremental_step) options.use_builtins_fixtures = True options.show_traceback = True + options.native_parser = True # XXX remove if options.num_workers: options.fixed_format_cache = True From e0cba2d83671ba4cdc333c632c8c7745b8ff08d9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 31 Jan 2026 13:30:11 +0000 Subject: [PATCH 135/192] [old parser] Add some conditional overload tests --- test-data/unit/parse.test | 139 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test index a77a3c61c876e..dfe352fada89d 100644 --- a/test-data/unit/parse.test +++ b/test-data/unit/parse.test @@ -2683,6 +2683,145 @@ MypyFile:1( Block:5( PassStmt:5()))))))) +[case testConditionalOverload1] +@overload +def foo(x: int): ... +if sys.version_info[0] > 2: + @overload + def foo(x: str): ... +def foo(x): pass +[out] +MypyFile:1( + IfStmt:3( + If( + ComparisonExpr:3( + > + IndexExpr:3( + MemberExpr:3( + NameExpr(sys) + version_info) + IntExpr(0)) + IntExpr(2))) + Then() + Else()) + OverloadedFuncDef:1( + Decorator:1( + Var(foo) + NameExpr(overload) + FuncDef:2( + foo + Args( + Var(x)) + def (x: int?) -> Any + Block:2( + ExpressionStmt:2( + Ellipsis)))) + Decorator:4( + Var(foo) + NameExpr(overload) + FuncDef:5( + foo + Args( + Var(x)) + def (x: str?) -> Any + Block:5( + ExpressionStmt:5( + Ellipsis)))) + FuncDef:6( + foo + Args( + Var(x)) + Block:6( + PassStmt:6())))) + +[case testConditionalOverload2] +@overload +def foo(x: int): ... +if sys.platform == "foo": + @overload + def foo(x: str): ... +def foo(x): pass +[out] +MypyFile:1( + IfStmt:3( + If( + ComparisonExpr:3( + == + MemberExpr:3( + NameExpr(sys) + platform) + StrExpr(foo))) + Then()) + OverloadedFuncDef:1( + Decorator:1( + Var(foo) + NameExpr(overload) + FuncDef:2( + foo + Args( + Var(x)) + def (x: int?) -> Any + Block:2( + ExpressionStmt:2( + Ellipsis)))) + FuncDef:6( + foo + Args( + Var(x)) + Block:6( + PassStmt:6())))) + +[case testConditionalOverload3] +if sys.version_info[0] > 2: + @overload + def foo(x: str): ... +@overload +def foo(x: int): ... +def foo(x): pass +[out] +MypyFile:1( + IfStmt:1( + If( + ComparisonExpr:1( + > + IndexExpr:1( + MemberExpr:1( + NameExpr(sys) + version_info) + IntExpr(0)) + IntExpr(2))) + Then() + Else()) + OverloadedFuncDef:2( + Decorator:2( + Var(foo) + NameExpr(overload) + FuncDef:3( + foo + Args( + Var(x)) + def (x: str?) -> Any + Block:3( + ExpressionStmt:3( + Ellipsis)))) + Decorator:4( + Var(foo) + NameExpr(overload) + FuncDef:5( + foo + Args( + Var(x)) + def (x: int?) -> Any + Block:5( + ExpressionStmt:5( + Ellipsis)))) + FuncDef:6( + foo + Args( + Var(x)) + Block:6( + PassStmt:6())))) + [case testCommentFunctionAnnotation] def f(): # type: () -> A pass From 74d5877ebc443d09ef27891a28e7d9c74c7ce608 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 31 Jan 2026 13:52:51 +0000 Subject: [PATCH 136/192] Add helpers --- mypy/nativeparse.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 58d19aad29753..904eb43b36f28 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -1449,3 +1449,38 @@ def read_call_type(state: State, data: ReadBuffer) -> Type: call_arg.end_line = invalid.end_line call_arg.end_column = invalid.end_column return call_arg + + +def strip_contents_from_if_stmt(stmt: IfStmt) -> None: + """Remove contents from IfStmt. + + Needed to still be able to check the conditions after the contents + have been merged with the surrounding function overloads. + """ + if len(stmt.body) == 1: + stmt.body[0].body = [] + if stmt.else_body and len(stmt.else_body.body) == 1: + if isinstance(stmt.else_body.body[0], IfStmt): + strip_contents_from_if_stmt(stmt.else_body.body[0]) + else: + stmt.else_body.body = [] + + +def is_stripped_if_stmt(stmt: Statement) -> bool: + """Check stmt to make sure it is a stripped IfStmt. + + See also: strip_contents_from_if_stmt + """ + if not isinstance(stmt, IfStmt): + return False + + if not (len(stmt.body) == 1 and len(stmt.body[0].body) == 0): + # Body not empty + return False + + if not stmt.else_body or len(stmt.else_body.body) == 0: + # No or empty else_body + return True + + # For elif, IfStmt are stored recursively in else_body + return is_stripped_if_stmt(stmt.else_body.body[0]) From 35ee1d330bb7bc99e710c9e93fcfb804c8c8c011 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 31 Jan 2026 13:56:16 +0000 Subject: [PATCH 137/192] Update native_parse to accept Options --- mypy/nativeparse.py | 8 +++++--- mypy/parse.py | 2 +- mypy/test/test_nativeparse.py | 11 ++++++----- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 904eb43b36f28..a1afb38d2e35a 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -29,6 +29,7 @@ ) from mypy import message_registry, nodes, types +from mypy.options import Options from mypy.sharedparse import special_function_elide_names from mypy.cache import ( DICT_STR_GEN, @@ -137,7 +138,8 @@ class State: - def __init__(self) -> None: + def __init__(self, options: Options) -> None: + self.options = options self.errors: list[dict[str, Any]] = [] def add_error(self, message: str, line: int, column: int) -> None: @@ -154,7 +156,7 @@ def expect_tag(data: ReadBuffer, tag: Tag) -> None: def native_parse( - filename: str, skip_function_bodies: bool = False + filename: str, options: Options, skip_function_bodies: bool = False ) -> tuple[MypyFile, list[dict[str, Any]], TypeIgnores]: # If the path is a directory, return empty AST (matching fastparse behavior) # This can happen for packages that only contain .pyc files without source @@ -166,7 +168,7 @@ def native_parse( b, errors, ignores = parse_to_binary_ast(filename, skip_function_bodies) data = ReadBuffer(b) n = read_int(data) - state = State() + state = State(options) defs = read_statements(state, data, n) node = MypyFile(defs, []) node.path = filename diff --git a/mypy/parse.py b/mypy/parse.py index 27daef9b91f43..470d13afd3d90 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -56,7 +56,7 @@ def parse( errors.set_file(fnam, module, options=options) tree, parse_errors, type_ignores = mypy.nativeparse.native_parse( - fnam, skip_function_bodies=strip_function_bodies + fnam, options, skip_function_bodies=strip_function_bodies ) # Convert type ignores list to dict tree.ignored_lines = dict(type_ignores) diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index e98ace89c9f55..ca768a83608e9 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -65,7 +65,7 @@ def test_parser(testcase: DataDrivenTestCase) -> None: try: with temp_source(source) as fnam: try: - node, errors, type_ignores = native_parse(fnam, skip_function_bodies) + node, errors, type_ignores = native_parse(fnam, options, skip_function_bodies) except ValueError as e: print(f"Parse failed: {e}") assert False @@ -125,12 +125,12 @@ def locs(start_line: int, start_column: int, end_line: int, end_column: int) -> def test_deserialize_hello(self) -> None: with temp_source("print('hello')") as fnam: - node = native_parse(fnam) + node = native_parse(fnam, Options()) assert isinstance(node, MypyFile) def test_deserialize_member_expr(self) -> None: with temp_source("foo_bar.xyz2") as fnam: - node, _, _ = native_parse(fnam) + node, _, _ = native_parse(fnam, Options()) assert isinstance(node, MypyFile) assert isinstance(node.defs[0], ExpressionStmt) assert isinstance(node.defs[0].expr, MemberExpr) @@ -139,11 +139,12 @@ def test_deserialize_bench(self) -> None: with temp_source("print('hello')\n" * 4000) as fnam: import time + options = Options() for i in range(10): - native_parse(fnam) + native_parse(fnam, options) t0 = time.time() for i in range(25): - node, _, _ = native_parse(fnam) + node, _, _ = native_parse(fnam, options) assert isinstance(node, MypyFile) print(len(node.defs)) print((time.time() - t0) * 1000) From 9558665ab69f2af31748315f588622ddd30e7e10 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 31 Jan 2026 14:03:35 +0000 Subject: [PATCH 138/192] Add helper from fastparse.py --- mypy/nativeparse.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index a1afb38d2e35a..770e8684d5a8a 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -30,6 +30,7 @@ from mypy import message_registry, nodes, types from mypy.options import Options +from mypy.reachability import infer_reachability_of_if_statement from mypy.sharedparse import special_function_elide_names from mypy.cache import ( DICT_STR_GEN, @@ -1486,3 +1487,36 @@ def is_stripped_if_stmt(stmt: Statement) -> bool: # For elif, IfStmt are stored recursively in else_body return is_stripped_if_stmt(stmt.else_body.body[0]) + + +def get_executable_if_block_with_overloads( + stmt: IfStmt, options: Options +) -> tuple[Block | None, IfStmt | None]: + """Return block from IfStmt that will get executed. + + Return + 0 -> A block if sure that alternative blocks are unreachable. + 1 -> An IfStmt if the reachability of it can't be inferred, + i.e. the truth value is unknown. + """ + infer_reachability_of_if_statement(stmt, options) + if stmt.else_body is None and stmt.body[0].is_unreachable is True: + # always False condition with no else + return None, None + if ( + stmt.else_body is None + or stmt.body[0].is_unreachable is False + and stmt.else_body.is_unreachable is False + ): + # The truth value is unknown, thus not conclusive + return None, stmt + if stmt.else_body.is_unreachable is True: + # else_body will be set unreachable if condition is always True + return stmt.body[0], None + if stmt.body[0].is_unreachable is True: + # body will be set unreachable if condition is always False + # else_body can contain an IfStmt itself (for elif) -> do a recursive check + if isinstance(stmt.else_body.body[0], IfStmt): + return get_executable_if_block_with_overloads(stmt.else_body.body[0], options) + return stmt.else_body, None + return None, stmt From 103ecc42f0438ff83bf527c3817766a1fcbd3fd3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 31 Jan 2026 14:19:04 +0000 Subject: [PATCH 139/192] Add helper functions --- mypy/nativeparse.py | 51 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 770e8684d5a8a..4cf9eb4eb13d3 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -1489,6 +1489,57 @@ def is_stripped_if_stmt(stmt: Statement) -> bool: return is_stripped_if_stmt(stmt.else_body.body[0]) +def fail_merge_overload(state: State, node: IfStmt) -> None: + """Report an error when overloads cannot be merged due to unknown condition.""" + state.add_error( + message_registry.FAILED_TO_MERGE_OVERLOADS.value, + node.line, + node.column, + ) + + +def check_ifstmt_for_overloads( + stmt: IfStmt, current_overload_name: str | None = None +) -> str | None: + """Check if IfStmt contains only overloads with the same name. + Return overload_name if found, None otherwise. + """ + # Check that block only contains a single Decorator, FuncDef, or OverloadedFuncDef. + # Multiple overloads have already been merged as OverloadedFuncDef. + if not ( + len(stmt.body[0].body) == 1 + and ( + isinstance(stmt.body[0].body[0], (Decorator, OverloadedFuncDef)) + or current_overload_name is not None + and isinstance(stmt.body[0].body[0], FuncDef) + ) + or len(stmt.body[0].body) > 1 + and isinstance(stmt.body[0].body[-1], OverloadedFuncDef) + and all(is_stripped_if_stmt(if_stmt) for if_stmt in stmt.body[0].body[:-1]) + ): + return None + + overload_name = cast(Decorator | FuncDef | OverloadedFuncDef, stmt.body[0].body[-1]).name + if stmt.else_body is None: + return overload_name + + if len(stmt.else_body.body) == 1: + # For elif: else_body contains an IfStmt itself -> do a recursive check. + if ( + isinstance(stmt.else_body.body[0], (Decorator, FuncDef, OverloadedFuncDef)) + and stmt.else_body.body[0].name == overload_name + ): + return overload_name + if ( + isinstance(stmt.else_body.body[0], IfStmt) + and check_ifstmt_for_overloads(stmt.else_body.body[0], current_overload_name) + == overload_name + ): + return overload_name + + return None + + def get_executable_if_block_with_overloads( stmt: IfStmt, options: Options ) -> tuple[Block | None, IfStmt | None]: From c0d728930ed88b8c131bad93a5df0690c6845f53 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 31 Jan 2026 14:21:16 +0000 Subject: [PATCH 140/192] Add fix_function_overloads from fastparse.py --- mypy/nativeparse.py | 160 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 4cf9eb4eb13d3..e54212d7d5b22 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -32,6 +32,7 @@ from mypy.options import Options from mypy.reachability import infer_reachability_of_if_statement from mypy.sharedparse import special_function_elide_names +from mypy.util import unnamed_function from mypy.cache import ( DICT_STR_GEN, END_TAG, @@ -95,6 +96,7 @@ OperatorAssignmentStmt, OpExpr, OverloadedFuncDef, + OverloadPart, PassStmt, RaiseStmt, ReturnStmt, @@ -1571,3 +1573,161 @@ def get_executable_if_block_with_overloads( return get_executable_if_block_with_overloads(stmt.else_body.body[0], options) return stmt.else_body, None return None, stmt + + +def fix_function_overloads( + state: State, options: Options, stmts: list[Statement] +) -> list[Statement]: + """Merge consecutive function overloads into OverloadedFuncDef nodes. + + This function processes a list of statements and combines function overloads + (marked with @overload decorator) that have the same name into a single + OverloadedFuncDef node. It also handles conditional overloads (overloads + inside if statements) when the condition can be evaluated. + """ + ret: list[Statement] = [] + current_overload: list[OverloadPart] = [] + current_overload_name: str | None = None + last_unconditional_func_def: str | None = None + last_if_stmt: IfStmt | None = None + last_if_overload: Decorator | FuncDef | OverloadedFuncDef | None = None + last_if_stmt_overload_name: str | None = None + last_if_unknown_truth_value: IfStmt | None = None + skipped_if_stmts: list[IfStmt] = [] + for stmt in stmts: + if_overload_name: str | None = None + if_block_with_overload: Block | None = None + if_unknown_truth_value: IfStmt | None = None + if isinstance(stmt, IfStmt): + # Check IfStmt block to determine if function overloads can be merged + if_overload_name = check_ifstmt_for_overloads(stmt, current_overload_name) + if if_overload_name is not None: + (if_block_with_overload, if_unknown_truth_value) = ( + get_executable_if_block_with_overloads(stmt, options) + ) + + if ( + current_overload_name is not None + and isinstance(stmt, (Decorator, FuncDef)) + and stmt.name == current_overload_name + ): + if last_if_stmt is not None: + skipped_if_stmts.append(last_if_stmt) + if last_if_overload is not None: + # Last stmt was an IfStmt with same overload name + # Add overloads to current_overload + if isinstance(last_if_overload, OverloadedFuncDef): + current_overload.extend(last_if_overload.items) + else: + current_overload.append(last_if_overload) + last_if_stmt, last_if_overload = None, None + if last_if_unknown_truth_value: + fail_merge_overload(state, last_if_unknown_truth_value) + last_if_unknown_truth_value = None + current_overload.append(stmt) + if isinstance(stmt, FuncDef): + # This is, strictly speaking, wrong: there might be a decorated + # implementation. However, it only affects the error message we show: + # ideally it's "already defined", but "implementation must come last" + # is also reasonable. + # TODO: can we get rid of this completely and just always emit + # "implementation must come last" instead? + last_unconditional_func_def = stmt.name + elif ( + current_overload_name is not None + and isinstance(stmt, IfStmt) + and if_overload_name == current_overload_name + and last_unconditional_func_def != current_overload_name + ): + # IfStmt only contains stmts relevant to current_overload. + # Check if stmts are reachable and add them to current_overload, + # otherwise skip IfStmt to allow subsequent overload + # or function definitions. + skipped_if_stmts.append(stmt) + if if_block_with_overload is None: + if if_unknown_truth_value is not None: + fail_merge_overload(state, if_unknown_truth_value) + continue + if last_if_overload is not None: + # Last stmt was an IfStmt with same overload name + # Add overloads to current_overload + if isinstance(last_if_overload, OverloadedFuncDef): + current_overload.extend(last_if_overload.items) + else: + current_overload.append(last_if_overload) + last_if_stmt, last_if_overload = None, None + if isinstance(if_block_with_overload.body[-1], OverloadedFuncDef): + skipped_if_stmts.extend(cast(list[IfStmt], if_block_with_overload.body[:-1])) + current_overload.extend(if_block_with_overload.body[-1].items) + else: + current_overload.append( + cast(Decorator | FuncDef, if_block_with_overload.body[0]) + ) + else: + if last_if_stmt is not None: + ret.append(last_if_stmt) + last_if_stmt_overload_name = current_overload_name + last_if_stmt, last_if_overload = None, None + last_if_unknown_truth_value = None + + if current_overload and current_overload_name == last_if_stmt_overload_name: + # Remove last stmt (IfStmt) from ret if the overload names matched + # Only happens if no executable block had been found in IfStmt + popped = ret.pop() + assert isinstance(popped, IfStmt) + skipped_if_stmts.append(popped) + if current_overload and skipped_if_stmts: + # Add bare IfStmt (without overloads) to ret + # Required for mypy to be able to still check conditions + for if_stmt in skipped_if_stmts: + strip_contents_from_if_stmt(if_stmt) + ret.append(if_stmt) + skipped_if_stmts = [] + if len(current_overload) == 1: + ret.append(current_overload[0]) + elif len(current_overload) > 1: + ret.append(OverloadedFuncDef(current_overload)) + + # If we have multiple decorated functions named "_" next to each, we want to treat + # them as a series of regular FuncDefs instead of one OverloadedFuncDef because + # most of mypy/mypyc assumes that all the functions in an OverloadedFuncDef are + # related, but multiple underscore functions next to each other aren't necessarily + # related + last_unconditional_func_def = None + if isinstance(stmt, Decorator) and not unnamed_function(stmt.name): + current_overload = [stmt] + current_overload_name = stmt.name + elif isinstance(stmt, IfStmt) and if_overload_name is not None: + current_overload = [] + current_overload_name = if_overload_name + last_if_stmt = stmt + last_if_stmt_overload_name = None + if if_block_with_overload is not None: + skipped_if_stmts.extend( + cast(list[IfStmt], if_block_with_overload.body[:-1]) + ) + last_if_overload = cast( + Decorator | FuncDef | OverloadedFuncDef, + if_block_with_overload.body[-1], + ) + last_if_unknown_truth_value = if_unknown_truth_value + else: + current_overload = [] + current_overload_name = None + ret.append(stmt) + + if current_overload and skipped_if_stmts: + # Add bare IfStmt (without overloads) to ret + # Required for mypy to be able to still check conditions + for if_stmt in skipped_if_stmts: + strip_contents_from_if_stmt(if_stmt) + ret.append(if_stmt) + if len(current_overload) == 1: + ret.append(current_overload[0]) + elif len(current_overload) > 1: + ret.append(OverloadedFuncDef(current_overload)) + elif last_if_overload is not None: + ret.append(last_if_overload) + elif last_if_stmt is not None: + ret.append(last_if_stmt) + return ret From e89ddf781fe4cc7cf77ff02d95392b0a3508a2c7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 31 Jan 2026 14:31:39 +0000 Subject: [PATCH 141/192] Fix conditional overload items --- mypy/nativeparse.py | 31 +++---- test-data/unit/native-parser.test | 139 ++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+), 21 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index e54212d7d5b22..b9022841e3433 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -144,6 +144,7 @@ class State: def __init__(self, options: Options) -> None: self.options = options self.errors: list[dict[str, Any]] = [] + self.num_funcs = 0 def add_error(self, message: str, line: int, column: int) -> None: """Report an error at a specific location.""" @@ -182,27 +183,13 @@ def native_parse( def read_statements(state: State, data: ReadBuffer, n: int) -> list[Statement]: defs: list[Statement] = [] - prev_func = False - prev_name = "" + old_num_funcs = state.num_funcs for _ in range(n): stmt = read_statement(state, data) - if isinstance(stmt, (FuncDef, Decorator)): - if prev_func and stmt.name == prev_name: - # Merge into overloaded function definition - prev = defs[-1] - if isinstance(prev, OverloadedFuncDef): - prev.items.append(stmt) - prev.unanalyzed_items.append(stmt) - else: - assert isinstance(prev, (FuncDef, Decorator)) - defs[-1] = OverloadedFuncDef([prev, stmt]) - else: - defs.append(stmt) - prev_name = stmt.name - prev_func = True - else: - defs.append(stmt) - prev_func = False + defs.append(stmt) + if state.num_funcs > old_num_funcs + 1: + # There were at least two functions, so we may need to merge overloads. + defs = fix_function_overloads(state, defs) return defs @@ -567,6 +554,8 @@ def read_parameters(state: State, data: ReadBuffer) -> tuple[list[Argument], boo def read_func_def(state: State, data: ReadBuffer) -> FuncDef: + state.num_funcs += 1 + # Function name name = read_str(data) @@ -1576,7 +1565,7 @@ def get_executable_if_block_with_overloads( def fix_function_overloads( - state: State, options: Options, stmts: list[Statement] + state: State, stmts: list[Statement] ) -> list[Statement]: """Merge consecutive function overloads into OverloadedFuncDef nodes. @@ -1603,7 +1592,7 @@ def fix_function_overloads( if_overload_name = check_ifstmt_for_overloads(stmt, current_overload_name) if if_overload_name is not None: (if_block_with_overload, if_unknown_truth_value) = ( - get_executable_if_block_with_overloads(stmt, options) + get_executable_if_block_with_overloads(stmt, state.options) ) if ( diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 4fad9544f443a..3f6e530de3b18 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -2821,3 +2821,142 @@ MypyFile:1( TempNode:1( Any) 5)) + +[case testConditionalOverload1] +@overload +def foo(x: int): ... +if sys.version_info[0] > 2: + @overload + def foo(x: str): ... +def foo(x): pass +[out] +MypyFile:1( + IfStmt:3( + If( + ComparisonExpr:3( + > + IndexExpr:3( + MemberExpr:3( + NameExpr(sys) + version_info) + IntExpr(0)) + IntExpr(2))) + Then() + Else()) + OverloadedFuncDef:1( + Decorator:1( + Var(foo) + NameExpr(overload) + FuncDef:2( + foo + Args( + Var(x)) + def (x: int?) -> Any + Block:2( + ExpressionStmt:2( + Ellipsis)))) + Decorator:4( + Var(foo) + NameExpr(overload) + FuncDef:5( + foo + Args( + Var(x)) + def (x: str?) -> Any + Block:5( + ExpressionStmt:5( + Ellipsis)))) + FuncDef:6( + foo + Args( + Var(x)) + Block:6( + PassStmt:6())))) + +[case testConditionalOverload2] +@overload +def foo(x: int): ... +if sys.platform == "foo": + @overload + def foo(x: str): ... +def foo(x): pass +[out] +MypyFile:1( + IfStmt:3( + If( + ComparisonExpr:3( + == + MemberExpr:3( + NameExpr(sys) + platform) + StrExpr(foo))) + Then()) + OverloadedFuncDef:1( + Decorator:1( + Var(foo) + NameExpr(overload) + FuncDef:2( + foo + Args( + Var(x)) + def (x: int?) -> Any + Block:2( + ExpressionStmt:2( + Ellipsis)))) + FuncDef:6( + foo + Args( + Var(x)) + Block:6( + PassStmt:6())))) + +[case testConditionalOverload3] +if sys.version_info[0] > 2: + @overload + def foo(x: str): ... +@overload +def foo(x: int): ... +def foo(x): pass +[out] +MypyFile:1( + IfStmt:1( + If( + ComparisonExpr:1( + > + IndexExpr:1( + MemberExpr:1( + NameExpr(sys) + version_info) + IntExpr(0)) + IntExpr(2))) + Then() + Else()) + OverloadedFuncDef:2( + Decorator:2( + Var(foo) + NameExpr(overload) + FuncDef:3( + foo + Args( + Var(x)) + def (x: str?) -> Any + Block:3( + ExpressionStmt:3( + Ellipsis)))) + Decorator:4( + Var(foo) + NameExpr(overload) + FuncDef:5( + foo + Args( + Var(x)) + def (x: int?) -> Any + Block:5( + ExpressionStmt:5( + Ellipsis)))) + FuncDef:6( + foo + Args( + Var(x)) + Block:6( + PassStmt:6())))) From 425152dfe0ea68ea266b14b0fb1f4b8c5d92e833 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 31 Jan 2026 14:55:30 +0000 Subject: [PATCH 142/192] Make some errors non-blocking --- mypy/nativeparse.py | 48 ++++++++++++++++++++++++++++++++++++++++----- mypy/parse.py | 7 ++++++- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index b9022841e3433..9d5786b4ca01a 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -146,9 +146,31 @@ def __init__(self, options: Options) -> None: self.errors: list[dict[str, Any]] = [] self.num_funcs = 0 - def add_error(self, message: str, line: int, column: int) -> None: - """Report an error at a specific location.""" - self.errors.append({"line": line, "column": column, "message": message}) + def add_error( + self, + message: str, + line: int, + column: int, + *, + blocker: bool = False, + code: str | None = None, + ) -> None: + """Report an error at a specific location. + + Args: + message: Error message to display + line: Line number where error occurred + column: Column number where error occurred + blocker: If True, this error blocks further analysis + code: Error code for categorization + """ + self.errors.append({ + "line": line, + "column": column, + "message": message, + "blocker": blocker, + "code": code, + }) def expect_end_tag(data: ReadBuffer) -> None: @@ -1383,7 +1405,11 @@ def read_call_type(state: State, data: ReadBuffer) -> Type: if not constructor: # ARG_CONSTRUCTOR_NAME_EXPECTED state.add_error( - message_registry.ARG_CONSTRUCTOR_NAME_EXPECTED.value, invalid.line, invalid.column + message_registry.ARG_CONSTRUCTOR_NAME_EXPECTED.value, + invalid.line, + invalid.column, + blocker=True, + code="misc", ) return invalid @@ -1405,7 +1431,11 @@ def read_call_type(state: State, data: ReadBuffer) -> Type: else: # ARG_CONSTRUCTOR_TOO_MANY_ARGS state.add_error( - message_registry.ARG_CONSTRUCTOR_TOO_MANY_ARGS.value, invalid.line, invalid.column + message_registry.ARG_CONSTRUCTOR_TOO_MANY_ARGS.value, + invalid.line, + invalid.column, + blocker=True, + code="misc", ) # Process keyword arguments @@ -1417,6 +1447,8 @@ def read_call_type(state: State, data: ReadBuffer) -> Type: message_registry.MULTIPLE_VALUES_FOR_NAME_KWARG.format(constructor).value, invalid.line, invalid.column, + blocker=True, + code="misc", ) name = extract_arg_name(kw_value) elif kw_name == "type": @@ -1426,6 +1458,8 @@ def read_call_type(state: State, data: ReadBuffer) -> Type: message_registry.MULTIPLE_VALUES_FOR_TYPE_KWARG.format(constructor).value, invalid.line, invalid.column, + blocker=True, + code="misc", ) typ = kw_value else: @@ -1434,6 +1468,8 @@ def read_call_type(state: State, data: ReadBuffer) -> Type: message_registry.ARG_CONSTRUCTOR_UNEXPECTED_ARG.format(kw_name).value, invalid.line, invalid.column, + blocker=True, + code="misc", ) # Create CallableArgument @@ -1486,6 +1522,8 @@ def fail_merge_overload(state: State, node: IfStmt) -> None: message_registry.FAILED_TO_MERGE_OVERLOADS.value, node.line, node.column, + blocker=False, + code="misc", ) diff --git a/mypy/parse.py b/mypy/parse.py index 470d13afd3d90..4c7aa68820ceb 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -71,8 +71,13 @@ def parse( message = error["message"] # Standardize error message by capitalizing the first word message = re.sub(r"^(\s*\w)", lambda m: m.group(1).upper(), message) + # Respect blocker status from error, default to True for syntax errors + is_blocker = error.get("blocker", True) + error_code = error.get("code") + if error_code is None: + error_code = codes.SYNTAX errors.report( - error["line"], error["column"], message, blocker=True, code=codes.SYNTAX + error["line"], error["column"], message, blocker=is_blocker, code=error_code ) if raise_on_error and errors.is_errors(): errors.raise_error() From e725ca6c9eb7426a98cac800f2191222a683e6bf Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 31 Jan 2026 15:05:36 +0000 Subject: [PATCH 143/192] Support --implicit-optional --- mypy/nativeparse.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 9d5786b4ca01a..e85b12f45b594 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -562,6 +562,12 @@ def read_parameters(state: State, data: ReadBuffer) -> tuple[list[Argument], boo default = None pos_only = read_bool(data) + # Apply implicit_optional if enabled and default is None + if state.options.implicit_optional and ann is not None: + optional = isinstance(default, NameExpr) and default.name == "None" + if isinstance(ann, UnboundType): + ann.optional = optional + var = Var(arg_name, ann) var.is_inferred = False arg = Argument(var, ann, default, arg_kind, pos_only) From 7dc2f8d23e92e6f5d433773ad70f70881d15edd6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 31 Jan 2026 15:51:43 +0000 Subject: [PATCH 144/192] Update legacy parse tests to pass by adding Keywords --- test-data/unit/parse.test | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test index dfe352fada89d..c8ebd9dced6ed 100644 --- a/test-data/unit/parse.test +++ b/test-data/unit/parse.test @@ -2476,6 +2476,7 @@ class Foo(metaclass=Bar): pass MypyFile:1( ClassDef:1( Foo + Keywords(metaclass=NameExpr(Bar)) Metaclass(NameExpr(Bar)) PassStmt:1())) @@ -2485,6 +2486,9 @@ class Foo(metaclass=foo.Bar): pass MypyFile:1( ClassDef:1( Foo + Keywords(metaclass=MemberExpr:1( + NameExpr(foo) + Bar)) Metaclass(MemberExpr:1( NameExpr(foo) Bar)) @@ -2496,6 +2500,7 @@ class Foo(foo.bar[x], metaclass=Bar): pass MypyFile:1( ClassDef:1( Foo + Keywords(metaclass=NameExpr(Bar)) Metaclass(NameExpr(Bar)) BaseTypeExpr( IndexExpr:1( @@ -2511,6 +2516,7 @@ class Foo(_root=None): pass MypyFile:1( ClassDef:1( Foo + Keywords(_root=NameExpr(None)) PassStmt:1())) [case testClassKeywordArgsBeforeMeta] @@ -2519,6 +2525,7 @@ class Foo(_root=None, metaclass=Bar): pass MypyFile:1( ClassDef:1( Foo + Keywords(_root=NameExpr(None), metaclass=NameExpr(Bar)) Metaclass(NameExpr(Bar)) PassStmt:1())) @@ -2528,6 +2535,7 @@ class Foo(metaclass=Bar, _root=None): pass MypyFile:1( ClassDef:1( Foo + Keywords(metaclass=NameExpr(Bar), _root=NameExpr(None)) Metaclass(NameExpr(Bar)) PassStmt:1())) From caf515916de507a11e74abec5bfc5abaaaa4c944 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 31 Jan 2026 16:00:22 +0000 Subject: [PATCH 145/192] Add legacy parser match statement tests --- test-data/unit/parse.test | 137 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test index c8ebd9dced6ed..6448ea47cc545 100644 --- a/test-data/unit/parse.test +++ b/test-data/unit/parse.test @@ -4025,3 +4025,140 @@ MypyFile:1( Args( Var(self)) Block:7())))) + +[case testMatchStmt] +match x: + case foo(y): + pass + case _: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(x) + Pattern( + ClassPattern:2( + NameExpr(foo) + Positionals( + AsPattern:2( + NameExpr(y))))) + Body( + PassStmt:3()) + Pattern( + AsPattern:4()) + Body( + PassStmt:5()))) + +[case testMatchLiteralPatterns] +match x: + case 1: + pass + case "hello": + pass + case True: + pass + case None: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(x) + Pattern( + ValuePattern:2( + IntExpr(1))) + Body( + PassStmt:3()) + Pattern( + ValuePattern:4( + StrExpr(hello))) + Body( + PassStmt:5()) + Pattern( + SingletonPattern:6( + True)) + Body( + PassStmt:7()) + Pattern( + SingletonPattern:8()) + Body( + PassStmt:9()))) + +[case testMatchOrPattern] +match x: + case 1 | 2 | 3: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(x) + Pattern( + OrPattern:2( + ValuePattern:2( + IntExpr(1)) + ValuePattern:2( + IntExpr(2)) + ValuePattern:2( + IntExpr(3)))) + Body( + PassStmt:3()))) + +[case testMatchSequencePattern] +match x: + case [a, *rest, b]: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(x) + Pattern( + SequencePattern:2( + AsPattern:2( + NameExpr(a)) + StarredPattern:2( + NameExpr(rest)) + AsPattern:2( + NameExpr(b)))) + Body( + PassStmt:3()))) + +[case testMatchMappingPattern] +match x: + case {"key": value, "x": y}: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(x) + Pattern( + MappingPattern:2( + Key( + StrExpr(key)) + Value( + AsPattern:2( + NameExpr(value))) + Key( + StrExpr(x)) + Value( + AsPattern:2( + NameExpr(y))))) + Body( + PassStmt:3()))) + +[case testMatchGuard] +match x: + case y if y > 0: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(x) + Pattern( + AsPattern:2( + NameExpr(y))) + Guard( + ComparisonExpr:2( + > + NameExpr(y) + IntExpr(0))) + Body( + PassStmt:3()))) From 305b6d303a5e56d12a3de4b355c64b25c3b07d22 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 31 Jan 2026 16:10:04 +0000 Subject: [PATCH 146/192] Deserialize match statements --- mypy/nativeparse.py | 153 ++++++++++++++++++++++++++++++ mypy/nodes.py | 9 ++ test-data/unit/native-parser.test | 137 ++++++++++++++++++++++++++ 3 files changed, 299 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index e85b12f45b594..aebaaedf90df3 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -89,6 +89,7 @@ LambdaExpr, ListComprehension, ListExpr, + MatchStmt, MemberExpr, MypyFile, NameExpr, @@ -117,6 +118,17 @@ YieldExpr, YieldFromExpr, ) +from mypy.patterns import ( + AsPattern, + ClassPattern, + MappingPattern, + OrPattern, + Pattern, + SequencePattern, + SingletonPattern, + StarredPattern, + ValuePattern, +) from mypy.types import ( AnyType, CallableArgument, @@ -528,6 +540,32 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: read_loc(data, stmt) expect_end_tag(data) return stmt + elif tag == nodes.MATCH_STMT: + # Read subject expression + subject = read_expression(state, data) + # Read number of cases + n_cases = read_int(data) + patterns = [] + guards = [] + bodies = [] + for _ in range(n_cases): + # Read pattern + pattern = read_pattern(state, data) + patterns.append(pattern) + # Read optional guard + has_guard = read_bool(data) + if has_guard: + guard = read_expression(state, data) + guards.append(guard) + else: + guards.append(None) + # Read body + body = read_block(state, data) + bodies.append(body) + stmt = MatchStmt(subject, patterns, guards, bodies) + read_loc(data, stmt) + expect_end_tag(data) + return stmt else: assert False, tag @@ -849,6 +887,121 @@ def read_type(state: State, data: ReadBuffer) -> Type: assert False, tag +def read_pattern(state: State, data: ReadBuffer) -> Pattern: + """Read a pattern node from the buffer.""" + tag = read_tag(data) + if tag == nodes.AS_PATTERN: + # Read optional pattern + has_pattern = read_bool(data) + if has_pattern: + pattern = read_pattern(state, data) + else: + pattern = None + # Read optional name + has_name = read_bool(data) + if has_name: + name_str = read_str(data) + name = NameExpr(name_str) + read_loc(data, name) + else: + name = None + as_pattern = AsPattern(pattern, name) + read_loc(data, as_pattern) + expect_end_tag(data) + return as_pattern + elif tag == nodes.OR_PATTERN: + # Read number of patterns + n = read_int(data) + patterns = [read_pattern(state, data) for _ in range(n)] + or_pattern = OrPattern(patterns) + read_loc(data, or_pattern) + expect_end_tag(data) + return or_pattern + elif tag == nodes.VALUE_PATTERN: + # Read value expression + expr = read_expression(state, data) + value_pattern = ValuePattern(expr) + read_loc(data, value_pattern) + expect_end_tag(data) + return value_pattern + elif tag == nodes.SINGLETON_PATTERN: + # Read singleton value + singleton_tag = read_tag(data) + if singleton_tag == LITERAL_NONE: + value = None + else: + # It's a boolean + value = singleton_tag == 1 # TAG_LITERAL_TRUE + singleton_pattern = SingletonPattern(value) + read_loc(data, singleton_pattern) + expect_end_tag(data) + return singleton_pattern + elif tag == nodes.SEQUENCE_PATTERN: + # Read number of patterns + n = read_int(data) + patterns = [read_pattern(state, data) for _ in range(n)] + sequence_pattern = SequencePattern(patterns) + read_loc(data, sequence_pattern) + expect_end_tag(data) + return sequence_pattern + elif tag == nodes.STARRED_PATTERN: + # Read optional capture name + has_name = read_bool(data) + if has_name: + name_str = read_str(data) + name = NameExpr(name_str) + read_loc(data, name) + else: + name = None + starred_pattern = StarredPattern(name) + read_loc(data, starred_pattern) + expect_end_tag(data) + return starred_pattern + elif tag == nodes.MAPPING_PATTERN: + # Read number of key-value pairs + n = read_int(data) + keys = [] + values = [] + for _ in range(n): + key = read_expression(state, data) + value = read_pattern(state, data) + keys.append(key) + values.append(value) + # Read optional rest pattern + has_rest = read_bool(data) + if has_rest: + rest_str = read_str(data) + rest = NameExpr(rest_str) + read_loc(data, rest) + else: + rest = None + mapping_pattern = MappingPattern(keys, values, rest) + read_loc(data, mapping_pattern) + expect_end_tag(data) + return mapping_pattern + elif tag == nodes.CLASS_PATTERN: + # Read class reference expression + class_ref = read_expression(state, data) + # Read number of positional patterns + n_positional = read_int(data) + positionals = [read_pattern(state, data) for _ in range(n_positional)] + # Read number of keyword patterns + n_keywords = read_int(data) + keyword_keys = [] + keyword_values = [] + for _ in range(n_keywords): + key = read_str(data) + value = read_pattern(state, data) + keyword_keys.append(key) + keyword_values.append(value) + class_pattern = ClassPattern(class_ref, positionals, keyword_keys, keyword_values) + read_loc(data, class_pattern) + expect_end_tag(data) + return class_pattern + else: + assert False, f"Unknown pattern tag: {tag}" + + def read_block(state: State, data: ReadBuffer) -> Block: expect_tag(data, nodes.BLOCK) expect_tag(data, LIST_GEN) diff --git a/mypy/nodes.py b/mypy/nodes.py index 8556a0af68ae9..331f3c2edb450 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5073,6 +5073,15 @@ def local_definitions( AWAIT_EXPR: Final[Tag] = 213 BIG_INT_EXPR: Final[Tag] = 214 IMPORT_ALL: Final[Tag] = 215 +MATCH_STMT: Final[Tag] = 216 +AS_PATTERN: Final[Tag] = 217 +OR_PATTERN: Final[Tag] = 218 +VALUE_PATTERN: Final[Tag] = 219 +SINGLETON_PATTERN: Final[Tag] = 220 +SEQUENCE_PATTERN: Final[Tag] = 221 +STARRED_PATTERN: Final[Tag] = 222 +MAPPING_PATTERN: Final[Tag] = 223 +CLASS_PATTERN: Final[Tag] = 224 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 3f6e530de3b18..4ddb3f7e88b0e 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -2960,3 +2960,140 @@ MypyFile:1( Var(x)) Block:6( PassStmt:6())))) + +[case testMatchStmt] +match x: + case foo(y): + pass + case _: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(x) + Pattern( + ClassPattern:2( + NameExpr(foo) + Positionals( + AsPattern:2( + NameExpr(y))))) + Body( + PassStmt:3()) + Pattern( + AsPattern:4()) + Body( + PassStmt:5()))) + +[case testMatchLiteralPatterns] +match x: + case 1: + pass + case "hello": + pass + case True: + pass + case None: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(x) + Pattern( + ValuePattern:2( + IntExpr(1))) + Body( + PassStmt:3()) + Pattern( + ValuePattern:4( + StrExpr(hello))) + Body( + PassStmt:5()) + Pattern( + SingletonPattern:6( + True)) + Body( + PassStmt:7()) + Pattern( + SingletonPattern:8()) + Body( + PassStmt:9()))) + +[case testMatchOrPattern] +match x: + case 1 | 2 | 3: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(x) + Pattern( + OrPattern:2( + ValuePattern:2( + IntExpr(1)) + ValuePattern:2( + IntExpr(2)) + ValuePattern:2( + IntExpr(3)))) + Body( + PassStmt:3()))) + +[case testMatchSequencePattern] +match x: + case [a, *rest, b]: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(x) + Pattern( + SequencePattern:2( + AsPattern:2( + NameExpr(a)) + StarredPattern:2( + NameExpr(rest)) + AsPattern:2( + NameExpr(b)))) + Body( + PassStmt:3()))) + +[case testMatchMappingPattern] +match x: + case {"key": value, "x": y}: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(x) + Pattern( + MappingPattern:2( + Key( + StrExpr(key)) + Value( + AsPattern:2( + NameExpr(value))) + Key( + StrExpr(x)) + Value( + AsPattern:2( + NameExpr(y))))) + Body( + PassStmt:3()))) + +[case testMatchGuard] +match x: + case y if y > 0: + pass +[out] +MypyFile:1( + MatchStmt:1( + NameExpr(x) + Pattern( + AsPattern:2( + NameExpr(y))) + Guard( + ComparisonExpr:2( + > + NameExpr(y) + IntExpr(0))) + Body( + PassStmt:3()))) From ceb6c544b672ab0b6b6677d7940dc91621aee5af Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 31 Jan 2026 16:50:11 +0000 Subject: [PATCH 147/192] WIP PEP 695 --- mypy/nativeparse.py | 130 ++++++++++++++++++++++++++++-- mypy/nodes.py | 1 + test-data/unit/native-parser.test | 91 +++++++++++++++++++++ 3 files changed, 216 insertions(+), 6 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index aebaaedf90df3..ab8f814ba72f2 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -53,6 +53,9 @@ ARG_KINDS, ARG_POS, MISSING_FALLBACK, + PARAM_SPEC_KIND, + TYPE_VAR_KIND, + TYPE_VAR_TUPLE_KIND, Argument, AssertStmt, AssignmentExpr, @@ -111,6 +114,8 @@ TempNode, TryStmt, TupleExpr, + TypeAliasStmt, + TypeParam, UnaryExpr, Var, WhileStmt, @@ -511,6 +516,8 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: return stmt elif tag == nodes.CLASS_DEF: return read_class_def(state, data) + elif tag == nodes.TYPE_ALIAS_STMT: + return read_type_alias_stmt(state, data) elif tag == nodes.TRY_STMT: return read_try_stmt(state, data) elif tag == nodes.DEL_STMT: @@ -619,6 +626,41 @@ def read_parameters(state: State, data: ReadBuffer) -> tuple[list[Argument], boo return arguments, has_ann +def read_type_params(state: State, data: ReadBuffer) -> list[TypeParam]: + """Read type parameters (PEP 695 generics).""" + type_params: list[TypeParam] = [] + n = read_int_bare(data) + for _ in range(n): + # Read type param kind + kind = read_int(data) + + # Read name + name = read_str(data) + + # Read upper bound + has_bound = read_bool(data) + if has_bound: + upper_bound = read_type(state, data) + else: + upper_bound = None + + # Read values (for constrained TypeVar) + expect_tag(data, LIST_GEN) + n_values = read_int_bare(data) + values = [read_type(state, data) for _ in range(n_values)] + + # Read default + has_default = read_bool(data) + if has_default: + default = read_type(state, data) + else: + default = None + + type_params.append(TypeParam(name, kind, upper_bound, values, default)) + + return type_params + + def read_func_def(state: State, data: ReadBuffer) -> FuncDef: state.num_funcs += 1 @@ -636,11 +678,14 @@ def read_func_def(state: State, data: ReadBuffer) -> FuncDef: is_async = read_bool(data) - # TODO: type_params + # Type parameters (PEP 695) has_type_params = read_bool(data) - assert not has_type_params, "Type params not yet supported" + if has_type_params: + type_params = read_type_params(state, data) + else: + type_params = None - # TODO: Return type annotation + # Return type annotation has_return_type = read_bool(data) if has_return_type: return_type = read_type(state, data) @@ -662,7 +707,7 @@ def read_func_def(state: State, data: ReadBuffer) -> FuncDef: else: typ = None - func_def = FuncDef(name, arguments, body, typ=typ) + func_def = FuncDef(name, arguments, body, typ=typ, type_args=type_params) if typ: # TODO: This seems wasteful, can we avoid it? func_def.unanalyzed_type = typ.copy_modified() @@ -691,9 +736,12 @@ def read_class_def(state: State, data: ReadBuffer) -> ClassDef: n_decorators = read_int_bare(data) decorators = [read_expression(state, data) for _ in range(n_decorators)] - # TODO: Type parameters (skip for now) + # Type parameters (PEP 695) has_type_params = read_bool(data) - assert not has_type_params, "Type parameters not yet supported" + if has_type_params: + type_params = read_type_params(state, data) + else: + type_params = None # Keywords (all keyword arguments including metaclass) expect_tag(data, DICT_STR_GEN) @@ -715,6 +763,7 @@ def read_class_def(state: State, data: ReadBuffer) -> ClassDef: base_type_exprs=base_type_exprs if base_type_exprs else None, metaclass=metaclass, keywords=filtered_keywords, + type_args=type_params, ) class_def.decorators = decorators read_loc(data, class_def) @@ -722,6 +771,75 @@ def read_class_def(state: State, data: ReadBuffer) -> ClassDef: return class_def +def read_type_alias_stmt(state: State, data: ReadBuffer) -> TypeAliasStmt: + """Read PEP 695 type alias statement.""" + # Read name (NameExpr) + name = read_expression(state, data) + assert isinstance(name, NameExpr), f"Expected NameExpr for type alias name, got {type(name)}" + + # Read type parameters + n_type_params = read_int_bare(data) + if n_type_params > 0: + type_params = [] + for _ in range(n_type_params): + # Read type param kind + kind = read_int(data) + + # Read name + param_name = read_str(data) + + # Read upper bound + has_bound = read_bool(data) + if has_bound: + upper_bound = read_type(state, data) + else: + upper_bound = None + + # Read values (for constrained TypeVar) + expect_tag(data, LIST_GEN) + n_values = read_int_bare(data) + values = [read_type(state, data) for _ in range(n_values)] + + # Read default + has_default = read_bool(data) + if has_default: + default = read_type(state, data) + else: + default = None + + type_params.append(TypeParam(param_name, kind, upper_bound, values, default)) + else: + type_params = [] + + # Read value expression (the type on RHS of type alias) + value_expr = read_expression(state, data) + + # Wrap the value expression in a LambdaExpr as expected by TypeAliasStmt + # The LambdaExpr body is a Block with a single ReturnStmt + return_stmt = ReturnStmt(value_expr) + return_stmt.line = value_expr.line + return_stmt.column = value_expr.column + return_stmt.end_line = value_expr.end_line + return_stmt.end_column = value_expr.end_column + + block = Block([return_stmt]) + block.line = -1 # Synthetic block + block.column = 0 + block.end_line = -1 + block.end_column = 0 + + lambda_expr = LambdaExpr([], block) + lambda_expr.line = value_expr.line + lambda_expr.column = value_expr.column + lambda_expr.end_line = value_expr.end_line + lambda_expr.end_column = value_expr.end_column + + stmt = TypeAliasStmt(name, type_params, lambda_expr) + read_loc(data, stmt) + expect_end_tag(data) + return stmt + + def read_try_stmt(state: State, data: ReadBuffer) -> TryStmt: # Read try body body = read_block(state, data) diff --git a/mypy/nodes.py b/mypy/nodes.py index 331f3c2edb450..29dd3d025c690 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5082,6 +5082,7 @@ def local_definitions( STARRED_PATTERN: Final[Tag] = 222 MAPPING_PATTERN: Final[Tag] = 223 CLASS_PATTERN: Final[Tag] = 224 +TYPE_ALIAS_STMT: Final[Tag] = 225 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index 4ddb3f7e88b0e..faacc6f5c87e0 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -3097,3 +3097,94 @@ MypyFile:1( IntExpr(0))) Body( PassStmt:3()))) + +[case testPEP695TypeAlias] +# comment +type A[T] = C[T] +[out] +MypyFile:1( + TypeAliasStmt:2( + NameExpr(A) + TypeParam( + T) + LambdaExpr:2( + Block:-1( + ReturnStmt:2( + IndexExpr:2( + NameExpr(C) + NameExpr(T))))))) + +[case testPEP695GenericFunction] +# comment + +def f[T](): pass +def g[T: str](): pass +def h[T: (int, str)](): pass +[out] +MypyFile:1( + FuncDef:3( + f + TypeParam( + T) + Block:3( + PassStmt:3())) + FuncDef:4( + g + TypeParam( + T + str?) + Block:4( + PassStmt:4())) + FuncDef:5( + h + TypeParam( + T + Values( + int? + str?)) + Block:5( + PassStmt:5()))) + +[case testPEP695ParamSpec] +# comment + +def f[**P](): pass +class C[T: int, **P]: pass +[out] +MypyFile:1( + FuncDef:3( + f + TypeParam( + **P) + Block:3( + PassStmt:3())) + ClassDef:4( + C + TypeParam( + T + int?) + TypeParam( + **P) + PassStmt:4())) + +[case testPEP695TypeVarTuple] +# comment + +def f[*Ts](): pass +class C[T: int, *Ts]: pass +[out] +MypyFile:1( + FuncDef:3( + f + TypeParam( + *Ts) + Block:3( + PassStmt:3())) + ClassDef:4( + C + TypeParam( + T + int?) + TypeParam( + *Ts) + PassStmt:4())) From 548fb1e9fcf6f68956d929e58ebf2ab3055f50f8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 1 Feb 2026 15:24:51 +0000 Subject: [PATCH 148/192] Support except* --- mypy/nativeparse.py | 4 ++ test-data/unit/native-parser.test | 86 +++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index ab8f814ba72f2..864517c5a1f2b 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -888,7 +888,11 @@ def read_try_stmt(state: State, data: ReadBuffer) -> TryStmt: else: finally_body = None + # Read is_star flag (for except* in Python 3.11+) + is_star = read_bool(data) + stmt = TryStmt(body, vars_list, types_list, handlers, else_body, finally_body) + stmt.is_star = is_star read_loc(data, stmt) expect_end_tag(data) return stmt diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index faacc6f5c87e0..c211021cbde9e 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1293,6 +1293,92 @@ MypyFile:1( ExpressionStmt:6( NameExpr(z))))) +[case testTryExceptStar] +try: + x +except* ValueError: + y +[out] +MypyFile:1( + TryStmt:1( + Block:2( + ExpressionStmt:2( + NameExpr(x))) + * + NameExpr(ValueError) + Block:4( + ExpressionStmt:4( + NameExpr(y))))) + +[case testTryExceptStarWithVar] +try: + x +except* Exception as e: + y +[out] +MypyFile:1( + TryStmt:1( + Block:2( + ExpressionStmt:2( + NameExpr(x))) + * + NameExpr(Exception) + NameExpr(e) + Block:4( + ExpressionStmt:4( + NameExpr(y))))) + +[case testTryMultipleExceptStar] +try: + x +except* ValueError: + y +except* KeyError as e: + z +[out] +MypyFile:1( + TryStmt:1( + Block:2( + ExpressionStmt:2( + NameExpr(x))) + * + NameExpr(ValueError) + Block:4( + ExpressionStmt:4( + NameExpr(y))) + NameExpr(KeyError) + NameExpr(e) + Block:6( + ExpressionStmt:6( + NameExpr(z))))) + +[case testTryExceptStarElseFinally] +try: + x +except* ValueError: + y +else: + z +finally: + w +[out] +MypyFile:1( + TryStmt:1( + Block:2( + ExpressionStmt:2( + NameExpr(x))) + * + NameExpr(ValueError) + Block:4( + ExpressionStmt:4( + NameExpr(y))) + Else( + ExpressionStmt:6( + NameExpr(z))) + Finally( + ExpressionStmt:8( + NameExpr(w))))) + [case testConditionalExprSimple] x if y else z [out] From c68d176939aa0d01eac2269c84dc14f354d9e7ff Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 1 Feb 2026 17:46:30 +0000 Subject: [PATCH 149/192] Fix * syntax --- mypy/nativeparse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 864517c5a1f2b..7d61838ed4916 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -999,7 +999,8 @@ def read_type(state: State, data: ReadBuffer) -> Type: return raw_type elif tag == types.UNPACK_TYPE: inner_type = read_type(state, data) - unpack = UnpackType(inner_type) + from_star_syntax = read_bool(data) + unpack = UnpackType(inner_type, from_star_syntax=from_star_syntax) read_loc(data, unpack) expect_end_tag(data) return unpack From cf786f1fc5a018486fb7deb29d01f76acac27132 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 2 Feb 2026 18:41:00 +0000 Subject: [PATCH 150/192] Revert "Add legacy parser match statement tests" This reverts commit caf515916de507a11e74abec5bfc5abaaaa4c944. --- test-data/unit/parse.test | 137 -------------------------------------- 1 file changed, 137 deletions(-) diff --git a/test-data/unit/parse.test b/test-data/unit/parse.test index 6448ea47cc545..c8ebd9dced6ed 100644 --- a/test-data/unit/parse.test +++ b/test-data/unit/parse.test @@ -4025,140 +4025,3 @@ MypyFile:1( Args( Var(self)) Block:7())))) - -[case testMatchStmt] -match x: - case foo(y): - pass - case _: - pass -[out] -MypyFile:1( - MatchStmt:1( - NameExpr(x) - Pattern( - ClassPattern:2( - NameExpr(foo) - Positionals( - AsPattern:2( - NameExpr(y))))) - Body( - PassStmt:3()) - Pattern( - AsPattern:4()) - Body( - PassStmt:5()))) - -[case testMatchLiteralPatterns] -match x: - case 1: - pass - case "hello": - pass - case True: - pass - case None: - pass -[out] -MypyFile:1( - MatchStmt:1( - NameExpr(x) - Pattern( - ValuePattern:2( - IntExpr(1))) - Body( - PassStmt:3()) - Pattern( - ValuePattern:4( - StrExpr(hello))) - Body( - PassStmt:5()) - Pattern( - SingletonPattern:6( - True)) - Body( - PassStmt:7()) - Pattern( - SingletonPattern:8()) - Body( - PassStmt:9()))) - -[case testMatchOrPattern] -match x: - case 1 | 2 | 3: - pass -[out] -MypyFile:1( - MatchStmt:1( - NameExpr(x) - Pattern( - OrPattern:2( - ValuePattern:2( - IntExpr(1)) - ValuePattern:2( - IntExpr(2)) - ValuePattern:2( - IntExpr(3)))) - Body( - PassStmt:3()))) - -[case testMatchSequencePattern] -match x: - case [a, *rest, b]: - pass -[out] -MypyFile:1( - MatchStmt:1( - NameExpr(x) - Pattern( - SequencePattern:2( - AsPattern:2( - NameExpr(a)) - StarredPattern:2( - NameExpr(rest)) - AsPattern:2( - NameExpr(b)))) - Body( - PassStmt:3()))) - -[case testMatchMappingPattern] -match x: - case {"key": value, "x": y}: - pass -[out] -MypyFile:1( - MatchStmt:1( - NameExpr(x) - Pattern( - MappingPattern:2( - Key( - StrExpr(key)) - Value( - AsPattern:2( - NameExpr(value))) - Key( - StrExpr(x)) - Value( - AsPattern:2( - NameExpr(y))))) - Body( - PassStmt:3()))) - -[case testMatchGuard] -match x: - case y if y > 0: - pass -[out] -MypyFile:1( - MatchStmt:1( - NameExpr(x) - Pattern( - AsPattern:2( - NameExpr(y))) - Guard( - ComparisonExpr:2( - > - NameExpr(y) - IntExpr(0))) - Body( - PassStmt:3()))) From e454453fb30ef84195d81bae1a40117d2d381913 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 7 Feb 2026 11:10:02 +0000 Subject: [PATCH 151/192] Ignore serialized imports --- mypy/nativeparse.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 7d61838ed4916..7e78f67a85d78 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -235,7 +235,9 @@ def read_statements(state: State, data: ReadBuffer, n: int) -> list[Statement]: def parse_to_binary_ast( filename: str, skip_function_bodies: bool = False ) -> tuple[bytes, list[dict[str, Any]], TypeIgnores]: - return ast_serialize.parse(filename, skip_function_bodies) # type: ignore[no-any-return] + ast_bytes, errors, ignores, _import_bytes = ast_serialize.parse(filename, skip_function_bodies) + # TODO: Process import_bytes for dependency tracking + return ast_bytes, errors, ignores def read_statement(state: State, data: ReadBuffer) -> Statement: From 21f27a0572be297cf6088cb3bfef443412a92a09 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 7 Feb 2026 11:30:56 +0000 Subject: [PATCH 152/192] Add function to deserialize out of band imports (currently unused) --- mypy/nativeparse.py | 95 ++++++++++++++++++++++++++++++++++++++++++++- mypy/nodes.py | 2 + 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 7e78f67a85d78..311e3d387dbf4 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -52,6 +52,8 @@ from mypy.nodes import ( ARG_KINDS, ARG_POS, + IMPORT_METADATA, + IMPORTFROM_METADATA, MISSING_FALLBACK, PARAM_SPEC_KIND, TYPE_VAR_KIND, @@ -151,7 +153,6 @@ TypeIgnores = list[tuple[int, list[str]]] - # There is no way to create reasonable fallbacks at this stage, # they must be patched later. _dummy_fallback: Final = Instance(MISSING_FALLBACK, [], -1) @@ -198,6 +199,98 @@ def expect_tag(data: ReadBuffer, tag: Tag) -> None: assert read_tag(data) == tag +def _read_and_set_import_metadata(data: ReadBuffer, stmt: Import | ImportFrom) -> None: + """Read location and metadata flags from buffer and set them on the import statement. + + Args: + data: Buffer containing serialized data + stmt: Import or ImportFrom statement to populate with location and metadata + """ + # Read location + read_loc(data, stmt) + + # Read metadata flags + is_top_level = read_bool(data) + is_unreachable = read_bool(data) + is_mypy_only = read_bool(data) + + # Set metadata on statement + stmt.is_top_level = is_top_level + stmt.is_unreachable = is_unreachable + stmt.is_mypy_only = is_mypy_only + + +def deserialize_imports(import_bytes: bytes) -> list[Import | ImportFrom]: + """Deserialize import metadata from bytes into mypy AST nodes. + + Args: + import_bytes: Serialized import metadata from the Rust parser + + Returns: + List of Import and ImportFrom AST nodes with location and metadata + """ + if not import_bytes: + return [] + + data = ReadBuffer(import_bytes) + + # Read list header + expect_tag(data, LIST_GEN) + n_imports = read_int_bare(data) + + imports: list[Import | ImportFrom] = [] + + for _ in range(n_imports): + tag = read_tag(data) + + if tag == IMPORT_METADATA: + # Read Import fields + name = read_str(data) + relative = read_int(data) + + # Read optional as_name + has_asname = read_bool(data) + if has_asname: + asname = read_str(data) + else: + asname = None + + # Create Import node and read common metadata + # Note: relative imports are handled via ImportFrom, so relative should be 0 here + stmt = Import([(name, asname)]) + _read_and_set_import_metadata(data, stmt) + imports.append(stmt) + + elif tag == IMPORTFROM_METADATA: + # Read ImportFrom fields + module = read_str(data) + relative = read_int(data) + + # Read list of names + expect_tag(data, LIST_GEN) + n_names = read_int_bare(data) + names: list[tuple[str, str | None]] = [] + + for _ in range(n_names): + name = read_str(data) + has_asname = read_bool(data) + if has_asname: + asname = read_str(data) + else: + asname = None + names.append((name, asname)) + + # Create ImportFrom node and read common metadata + stmt = ImportFrom(module, relative, names) + _read_and_set_import_metadata(data, stmt) + imports.append(stmt) + + else: + raise ValueError(f"Unexpected tag in import metadata: {tag}") + + return imports + + def native_parse( filename: str, options: Options, skip_function_bodies: bool = False ) -> tuple[MypyFile, list[dict[str, Any]], TypeIgnores]: diff --git a/mypy/nodes.py b/mypy/nodes.py index 29dd3d025c690..3f3cbd8db051f 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5083,6 +5083,8 @@ def local_definitions( MAPPING_PATTERN: Final[Tag] = 223 CLASS_PATTERN: Final[Tag] = 224 TYPE_ALIAS_STMT: Final[Tag] = 225 +IMPORT_METADATA: Final[Tag] = 226 +IMPORTFROM_METADATA: Final[Tag] = 227 def read_symbol(data: ReadBuffer) -> SymbolNode: From 17455ac5633022a5d2899989b7b1a39abddcf296 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 7 Feb 2026 11:37:11 +0000 Subject: [PATCH 153/192] Use serialized import metadata --- mypy/nativeparse.py | 15 +++++++++------ mypy/parse.py | 7 +++---- mypy/test/test_nativeparse.py | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 311e3d387dbf4..852b285b7f2c0 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -301,12 +301,16 @@ def native_parse( node.path = filename return node, [], [] - b, errors, ignores = parse_to_binary_ast(filename, skip_function_bodies) + b, errors, ignores, import_bytes = parse_to_binary_ast(filename, skip_function_bodies) data = ReadBuffer(b) n = read_int(data) state = State(options) defs = read_statements(state, data, n) - node = MypyFile(defs, []) + + # Deserialize import metadata + imports = deserialize_imports(import_bytes) + + node = MypyFile(defs, imports) node.path = filename # Merge deserialization errors with parsing errors all_errors = errors + state.errors @@ -327,10 +331,9 @@ def read_statements(state: State, data: ReadBuffer, n: int) -> list[Statement]: def parse_to_binary_ast( filename: str, skip_function_bodies: bool = False -) -> tuple[bytes, list[dict[str, Any]], TypeIgnores]: - ast_bytes, errors, ignores, _import_bytes = ast_serialize.parse(filename, skip_function_bodies) - # TODO: Process import_bytes for dependency tracking - return ast_bytes, errors, ignores +) -> tuple[bytes, list[dict[str, Any]], TypeIgnores, bytes]: + ast_bytes, errors, ignores, import_bytes = ast_serialize.parse(filename, skip_function_bodies) + return ast_bytes, errors, ignores, import_bytes def read_statement(state: State, data: ReadBuffer) -> Statement: diff --git a/mypy/parse.py b/mypy/parse.py index 4c7aa68820ceb..5629847130376 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -62,10 +62,9 @@ def parse( tree.ignored_lines = dict(type_ignores) # Set is_stub based on file extension tree.is_stub = fnam.endswith(".pyi") - # Collect all import nodes from the tree - collector = ImportCollector() - tree.accept(collector) - tree.imports = collector.imports + # Note: tree.imports is populated directly by native_parse with deserialized + # import metadata, so we don't need to collect imports via AST traversal + # Report parse errors for error in parse_errors: message = error["message"] diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index ca768a83608e9..f5c1c36812df2 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -107,7 +107,7 @@ def locs(start_line: int, start_column: int, end_line: int, end_column: int) -> ] with temp_source("print('hello')") as fnam: - b, _, _ = parse_to_binary_ast(fnam) + b, _, _, _ = parse_to_binary_ast(fnam) assert list(b) == ( [LITERAL_INT, 22, nodes.EXPR_STMT, nodes.CALL_EXPR] + [nodes.NAME_EXPR, LITERAL_STR] From 82c448cce817f1b0319a0219813483a46d06e63e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 7 Feb 2026 11:40:22 +0000 Subject: [PATCH 154/192] Remove now unused ImportCollector --- mypy/parse.py | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/mypy/parse.py b/mypy/parse.py index 5629847130376..b58c847f5d569 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -5,28 +5,8 @@ from mypy import errorcodes as codes from mypy.errors import Errors -from mypy.nodes import Import, ImportAll, ImportBase, ImportFrom, MypyFile +from mypy.nodes import MypyFile from mypy.options import Options -from mypy.traverser import TraverserVisitor - - -class ImportCollector(TraverserVisitor): - """Visitor that collects all import nodes from an AST.""" - - def __init__(self) -> None: - self.imports: list[ImportBase] = [] - - def visit_import(self, node: Import) -> None: - self.imports.append(node) - super().visit_import(node) - - def visit_import_from(self, node: ImportFrom) -> None: - self.imports.append(node) - super().visit_import_from(node) - - def visit_import_all(self, node: ImportAll) -> None: - self.imports.append(node) - super().visit_import_all(node) def parse( From cee5597fdbcfc06c316233631f5cb076ddeb7885 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 7 Feb 2026 12:04:51 +0000 Subject: [PATCH 155/192] Fix self check --- mypy/nativeparse.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 852b285b7f2c0..1cfcb7e9793bb 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -88,6 +88,7 @@ IfStmt, Import, ImportAll, + ImportBase, ImportFrom, IndexExpr, IntExpr, @@ -105,6 +106,7 @@ OverloadPart, PassStmt, RaiseStmt, + RefExpr, ReturnStmt, SetComprehension, SetExpr, @@ -220,7 +222,7 @@ def _read_and_set_import_metadata(data: ReadBuffer, stmt: Import | ImportFrom) - stmt.is_mypy_only = is_mypy_only -def deserialize_imports(import_bytes: bytes) -> list[Import | ImportFrom]: +def deserialize_imports(import_bytes: bytes) -> list[ImportBase]: """Deserialize import metadata from bytes into mypy AST nodes. Args: @@ -238,7 +240,7 @@ def deserialize_imports(import_bytes: bytes) -> list[Import | ImportFrom]: expect_tag(data, LIST_GEN) n_imports = read_int_bare(data) - imports: list[Import | ImportFrom] = [] + imports: list[ImportBase] = [] for _ in range(n_imports): tag = read_tag(data) @@ -651,7 +653,7 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: # Read number of cases n_cases = read_int(data) patterns = [] - guards = [] + guards: list[Expression | None] = [] bodies = [] for _ in range(n_cases): # Read pattern @@ -1087,7 +1089,7 @@ def read_type(state: State, data: ReadBuffer) -> Type: elif type_name == "typing.Any": # Invalid type - read None value tag = read_tag(data) - assert tag == types.LITERAL_NONE, f"Expected LITERAL_NONE for invalid type, got {tag}" + assert tag == LITERAL_NONE, f"Expected LITERAL_NONE for invalid type, got {tag}" value = None else: assert False, f"Unsupported RawExpressionType: {type_name}" @@ -1202,7 +1204,7 @@ def read_pattern(state: State, data: ReadBuffer) -> Pattern: return mapping_pattern elif tag == nodes.CLASS_PATTERN: # Read class reference expression - class_ref = read_expression(state, data) + class_ref = cast(RefExpr, read_expression(state, data)) # Read number of positional patterns n_positional = read_int(data) positionals = [read_pattern(state, data) for _ in range(n_positional)] From 17bcce5cb586925e8fb331651a9e9f24e2a7e076 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 7 Feb 2026 12:26:17 +0000 Subject: [PATCH 156/192] Test import serialization/deserialization --- mypy/test/test_nativeparse.py | 97 +++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index f5c1c36812df2..33d1e52f7800e 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -33,6 +33,15 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: test_parser(testcase) +class NativeParserImportsSuite(DataSuite): + required_out_section = True + base_path = "." + files = ["native-parser-imports.test"] + + def run_case(self, testcase: DataDrivenTestCase) -> None: + test_parser_imports(testcase) + + def test_parser(testcase: DataDrivenTestCase) -> None: """Perform a single native parser test case. @@ -92,6 +101,94 @@ def format_ignore(ignore: tuple[int, list[str]]) -> str: return f"ignore: {line} [{', '.join(codes)}]" +def test_parser_imports(testcase: DataDrivenTestCase) -> None: + """Perform a single native parser imports test case. + + The argument contains the description of the test case. + This test outputs only reachable import information. + """ + options = Options() + options.hide_error_codes = True + options.python_version = (3, 10) + + source = "\n".join(testcase.input) + + try: + with temp_source(source) as fnam: + try: + node, errors, type_ignores = native_parse(fnam, options) + except ValueError as e: + print(f"Parse failed: {e}") + assert False + + # Extract and format reachable imports + a = format_reachable_imports(node) + a = [format_error(err) for err in errors] + a + except CompileError as e: + a = e.messages + + assert_string_arrays_equal( + testcase.output, a, f"Invalid parser output ({testcase.file}, line {testcase.line})" + ) + + +def format_reachable_imports(node: MypyFile) -> list[str]: + """Format reachable imports from a MypyFile node. + + Returns a list of strings representing reachable imports with line numbers and flags. + """ + from mypy.nodes import Import, ImportFrom + + output: list[str] = [] + + # Filter for reachable imports (is_unreachable == False) + reachable_imports = [imp for imp in node.imports if not imp.is_unreachable] + + for imp in reachable_imports: + line_num = imp.line + + # Collect flags (only show when flag is False/not set) + flags = [] + if not imp.is_top_level: + flags.append("not top_level") + if imp.is_mypy_only: + flags.append("mypy_only") + + flags_str = " [" + ", ".join(flags) + "]" if flags else "" + + if isinstance(imp, Import): + # Format: line: import foo [as bar] [flags] + for module_id, as_id in imp.ids: + if as_id: + output.append(f"{line_num}: import {module_id} as {as_id}{flags_str}") + else: + output.append(f"{line_num}: import {module_id}{flags_str}") + elif isinstance(imp, ImportFrom): + # Format: line: from foo import bar, baz [as b] [flags] + # Handle relative imports + if imp.relative > 0: + prefix = "." * imp.relative + if imp.id: + module = f"{prefix}{imp.id}" + else: + module = prefix + else: + module = imp.id + + # Group all names together + name_parts = [] + for name, as_name in imp.names: + if as_name: + name_parts.append(f"{name} as {as_name}") + else: + name_parts.append(name) + + names_str = ", ".join(name_parts) + output.append(f"{line_num}: from {module} import {names_str}{flags_str}") + + return output + + class TestNativeParse(unittest.TestCase): def test_trivial_binary_data(self) -> None: def int_enc(n: int) -> int: From 0f37b175b21d110e5dcd1a2f9f21a17405cccd8f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 7 Feb 2026 13:40:06 +0000 Subject: [PATCH 157/192] Deserialize block reachability --- mypy/nativeparse.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 1cfcb7e9793bb..31c5c95453797 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -1229,9 +1229,10 @@ def read_block(state: State, data: ReadBuffer) -> Block: expect_tag(data, nodes.BLOCK) expect_tag(data, LIST_GEN) n = read_int_bare(data) + is_unreachable = read_bool(data) if n == 0: # Empty block - read explicit location - b = Block([]) + b = Block([], is_unreachable=is_unreachable) read_loc(data, b) expect_end_tag(data) return b @@ -1239,7 +1240,7 @@ def read_block(state: State, data: ReadBuffer) -> Block: # Non-empty block - read statements and set location from them a = read_statements(state, data, n) expect_end_tag(data) - b = Block(a) + b = Block(a, is_unreachable=is_unreachable) b.line = a[0].line b.column = a[0].column b.end_line = a[-1].end_line @@ -1251,11 +1252,12 @@ def read_optional_block(state: State, data: ReadBuffer) -> Block | None: expect_tag(data, nodes.BLOCK) expect_tag(data, LIST_GEN) n = read_int_bare(data) + is_unreachable = read_bool(data) if n == 0: b = None else: a = [read_statement(state, data) for i in range(n)] - b = Block(a) + b = Block(a, is_unreachable=is_unreachable) b.line = a[0].line b.column = a[0].column b.end_line = a[-1].end_line From 76f6a32499725d2cad7caad4c8fc73a2f0add0e8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 7 Feb 2026 13:47:17 +0000 Subject: [PATCH 158/192] Add reachability tests --- test-data/unit/native-parser-imports.test | 176 ++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 test-data/unit/native-parser-imports.test diff --git a/test-data/unit/native-parser-imports.test b/test-data/unit/native-parser-imports.test new file mode 100644 index 0000000000000..1440307287630 --- /dev/null +++ b/test-data/unit/native-parser-imports.test @@ -0,0 +1,176 @@ +-- Test cases for import returned by the native parser using a side channel +-- In particular, test import flags related to reachability and priority when ordering SCCs + +[case testNoImport] +# No imports +print() +[out] + +[case testSimpleImport] +import foo +[out] +1: import foo + +[case testMultipleImports] +import foo +import bar +import baz +[out] +1: import foo +2: import bar +3: import baz + +[case testImportAs] +import foo as f +import bar as b +[out] +1: import foo as f +2: import bar as b + +[case testFromImport] +from foo import bar +[out] +1: from foo import bar + +[case testFromImportMultiple] +from foo import bar, baz, qux +[out] +1: from foo import bar, baz, qux + +[case testFromImportAs] +from foo import bar as b, baz as z +[out] +1: from foo import bar as b, baz as z + +[case testRelativeImport] +from . import foo +from .. import bar +from ...baz import qux +[out] +1: from . import foo +2: from .. import bar +3: from ...baz import qux + +[case testNonTopLevelImport] +def f(): + import foo + from bar import baz +[out] +2: import foo [not top_level] +3: from bar import baz [not top_level] + +[case testMixedTopLevel] +import toplevel +def f(): + import inside_func +import another_toplevel +[out] +1: import toplevel +3: import inside_func [not top_level] +4: import another_toplevel + +[case testImportInClass] +class Foo: + import bar +[out] +2: import bar +[case testUnreachableImportPY2] +# Imports in unreachable blocks should not appear +if PY2: + import unreachable_module +import reachable_module +[out] +4: import reachable_module + +[case testUnreachableFromImportPY2] +# From imports in unreachable blocks should not appear +if PY2: + from unreachable import foo +from reachable import bar +[out] +4: from reachable import bar + +[case testReachableImportPY3] +# Imports in reachable if PY3 blocks should appear +if PY3: + import reachable_module +[out] +3: import reachable_module + +[case testUnreachableElseBlock] +# Imports in unreachable else block should not appear +if PY3: + import reachable +else: + import unreachable +[out] +3: import reachable + +[case testMixedReachableUnreachable] +# Mix of reachable and unreachable imports +import before +if PY2: + import unreachable_in_if +else: + import reachable_in_else +import after +[out] +2: import before +6: import reachable_in_else +7: import after + +[case testNestedUnreachable] +# Nested unreachable blocks +if PY2: + import outer_unreachable + if True: + import inner_unreachable +import reachable +[out] +6: import reachable + +[case testUnreachableWithTopLevel] +# Unreachable imports are still marked as top_level correctly +def f(): + if PY2: + import unreachable_func + import reachable_func +if PY2: + import unreachable_top +import reachable_top +[out] +5: import reachable_func [not top_level] +8: import reachable_top + +[case testVersionCheckUnreachable] +# sys.version_info check makes else unreachable +import sys +if sys.version_info >= (3, 8): + import reachable_version_check +else: + import unreachable_old_version +[out] +2: import sys +4: import reachable_version_check + +[case testElifUnreachable] +# Elif after always-true condition is unreachable +if PY3: + import reachable_if +elif True: + import unreachable_elif +else: + import unreachable_else +[out] +3: import reachable_if + +[case testAllBranchesUnreachable] +# When all branches are unreachable, no imports appear +if PY2: + import unreachable_if +elif PY2: + import unreachable_elif +else: + import reachable_else +[out] +7: import reachable_else From 0deb566f63dc8c9a8538b5a3dcffd46287f18e5a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 7 Feb 2026 14:58:53 +0000 Subject: [PATCH 159/192] Add reachability tests --- test-data/unit/native-parser-imports.test | 188 ++++++++++++++++++++++ 1 file changed, 188 insertions(+) diff --git a/test-data/unit/native-parser-imports.test b/test-data/unit/native-parser-imports.test index 1440307287630..0dc55cd14dbd9 100644 --- a/test-data/unit/native-parser-imports.test +++ b/test-data/unit/native-parser-imports.test @@ -174,3 +174,191 @@ else: import reachable_else [out] 7: import reachable_else + +[case testMypyOnlyImport] +# Imports in TYPE_CHECKING blocks are mypy_only +from typing import TYPE_CHECKING +if TYPE_CHECKING: + import mypy_only_module +import regular_module +[out] +2: from typing import TYPE_CHECKING +4: import mypy_only_module [mypy_only] +5: import regular_module + +[case testMypyOnlyFromImport] +# From imports in TYPE_CHECKING blocks are mypy_only +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from mypy_types import MyType +from regular import RegularType +[out] +2: from typing import TYPE_CHECKING +4: from mypy_types import MyType [mypy_only] +5: from regular import RegularType + +[case testMypyNameAlias] +# MYPY is also recognized as mypy_only +if MYPY: + import only_for_mypy +import for_everyone +[out] +3: import only_for_mypy [mypy_only] +4: import for_everyone + +[case testMypyOnlyElseBranch] +# Else branch of TYPE_CHECKING is unreachable (runtime-only, not analyzed by mypy) +from typing import TYPE_CHECKING +if TYPE_CHECKING: + import for_mypy +else: + import for_runtime +[out] +2: from typing import TYPE_CHECKING +4: import for_mypy [mypy_only] + +[case testNestedMypyOnly] +# Nested blocks inside TYPE_CHECKING are also mypy_only +from typing import TYPE_CHECKING +if TYPE_CHECKING: + if True: + import deeply_nested + import also_mypy_only +[out] +2: from typing import TYPE_CHECKING +5: import deeply_nested [mypy_only] +6: import also_mypy_only [mypy_only] + +[case testMypyOnlyInFunction] +# Mypy-only imports in functions +from typing import TYPE_CHECKING +def f(): + if TYPE_CHECKING: + import mypy_func_import + import regular_func_import +[out] +2: from typing import TYPE_CHECKING +5: import mypy_func_import [not top_level, mypy_only] +6: import regular_func_import [not top_level] + +[case testMypyOnlyElifBranch] +# Elif after TYPE_CHECKING is unreachable (runtime-only) +from typing import TYPE_CHECKING +if TYPE_CHECKING: + import for_mypy +elif True: + import for_runtime_elif +[out] +2: from typing import TYPE_CHECKING +4: import for_mypy [mypy_only] + +[case testNotMypyFalseBranch] +# Not TYPE_CHECKING makes body unreachable, else is mypy_only +from typing import TYPE_CHECKING +if not TYPE_CHECKING: + import runtime_only +else: + import mypy_only +[out] +2: from typing import TYPE_CHECKING +6: import mypy_only [mypy_only] + +[case testMixedMypyOnlyAndUnreachable] +# Combination of mypy_only and unreachable +from typing import TYPE_CHECKING +if PY2: + import unreachable_py2 +if TYPE_CHECKING: + import mypy_only_import +if PY3: + import always_reachable +[out] +2: from typing import TYPE_CHECKING +6: import mypy_only_import [mypy_only] +8: import always_reachable + +[case testTypingExtensionsTypeChecking] +# typing_extensions.TYPE_CHECKING is also recognized +from typing_extensions import TYPE_CHECKING +if TYPE_CHECKING: + import mypy_import +[out] +2: from typing_extensions import TYPE_CHECKING +4: import mypy_import [mypy_only] + +[case testUnknownCondition] +# Unknown conditions - all branches reachable +if x: + import in_if +else: + import in_else +[out] +3: import in_if +5: import in_else + +[case testUnknownConditionWithElif] +# Unknown conditions with elif +if unknown_var: + import first +elif another_var: + import second +else: + import third +[out] +3: import first +5: import second +7: import third + +[case testMixedKnownUnknown] +# Mix of known and unknown conditions +if TYPE_CHECKING: + import mypy_only +if some_var: + import maybe_reachable +if PY2: + import unreachable +[out] +3: import mypy_only [mypy_only] +5: import maybe_reachable + +[case testNestedUnknownInMypyOnly] +# Unknown condition nested in mypy-only block +from typing import TYPE_CHECKING +if TYPE_CHECKING: + if condition: + import nested_mypy + import also_mypy +[out] +2: from typing import TYPE_CHECKING +5: import nested_mypy [mypy_only] +6: import also_mypy [mypy_only] + +[case testUnknownAfterMypyTrue] +# Unknown elif after TYPE_CHECKING is unreachable +from typing import TYPE_CHECKING +if TYPE_CHECKING: + import for_mypy +elif x: + import unreachable_elif +[out] +2: from typing import TYPE_CHECKING +4: import for_mypy [mypy_only] + +[case testComplexCondition] +# Complex boolean conditions that can't be evaluated +if x and y: + import complex_if +elif z or w: + import complex_elif +[out] +3: import complex_if +5: import complex_elif + +[case testUnknownWithFunction] +# Unknown condition with function call +if func(): + import from_func_call +import regular +[out] +3: import from_func_call +4: import regular From e09c3c2b83ef9da836bfcc45ab7a3bee6cbb5882 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 8 Feb 2026 15:14:46 +0000 Subject: [PATCH 160/192] Fixes to import serialization --- mypy/nativeparse.py | 32 ++++++++++++++------ mypy/nodes.py | 1 + mypy/test/test_nativeparse.py | 15 ++++++++- test-data/unit/native-parser-imports.test | 37 +++++++++++++++++++++++ 4 files changed, 74 insertions(+), 11 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 31c5c95453797..0c5418cc8187b 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -54,6 +54,7 @@ ARG_POS, IMPORT_METADATA, IMPORTFROM_METADATA, + IMPORTALL_METADATA, MISSING_FALLBACK, PARAM_SPEC_KIND, TYPE_VAR_KIND, @@ -201,25 +202,26 @@ def expect_tag(data: ReadBuffer, tag: Tag) -> None: assert read_tag(data) == tag -def _read_and_set_import_metadata(data: ReadBuffer, stmt: Import | ImportFrom) -> None: +def _read_and_set_import_metadata(data: ReadBuffer, stmt: Import | ImportFrom | ImportAll) -> None: """Read location and metadata flags from buffer and set them on the import statement. Args: data: Buffer containing serialized data - stmt: Import or ImportFrom statement to populate with location and metadata + stmt: Import, ImportFrom, or ImportAll statement to populate with location and metadata """ # Read location read_loc(data, stmt) - # Read metadata flags - is_top_level = read_bool(data) - is_unreachable = read_bool(data) - is_mypy_only = read_bool(data) + # Read metadata flags as a single integer bitfield + flags = read_int(data) - # Set metadata on statement - stmt.is_top_level = is_top_level - stmt.is_unreachable = is_unreachable - stmt.is_mypy_only = is_mypy_only + # Extract individual flags using bitwise operations + # Bit 0: is_top_level + # Bit 1: is_unreachable + # Bit 2: is_mypy_only + stmt.is_top_level = (flags & 0x01) != 0 + stmt.is_unreachable = (flags & 0x02) != 0 + stmt.is_mypy_only = (flags & 0x04) != 0 def deserialize_imports(import_bytes: bytes) -> list[ImportBase]: @@ -287,6 +289,16 @@ def deserialize_imports(import_bytes: bytes) -> list[ImportBase]: _read_and_set_import_metadata(data, stmt) imports.append(stmt) + elif tag == IMPORTALL_METADATA: + # Read ImportAll fields + module = read_str(data) + relative = read_int(data) + + # Create ImportAll node and read common metadata + stmt = ImportAll(module, relative) + _read_and_set_import_metadata(data, stmt) + imports.append(stmt) + else: raise ValueError(f"Unexpected tag in import metadata: {tag}") diff --git a/mypy/nodes.py b/mypy/nodes.py index 3f3cbd8db051f..bcc598745e885 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -5085,6 +5085,7 @@ def local_definitions( TYPE_ALIAS_STMT: Final[Tag] = 225 IMPORT_METADATA: Final[Tag] = 226 IMPORTFROM_METADATA: Final[Tag] = 227 +IMPORTALL_METADATA: Final[Tag] = 228 def read_symbol(data: ReadBuffer) -> SymbolNode: diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index 33d1e52f7800e..9fcb13bb3e378 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -137,7 +137,7 @@ def format_reachable_imports(node: MypyFile) -> list[str]: Returns a list of strings representing reachable imports with line numbers and flags. """ - from mypy.nodes import Import, ImportFrom + from mypy.nodes import Import, ImportAll, ImportFrom output: list[str] = [] @@ -185,6 +185,19 @@ def format_reachable_imports(node: MypyFile) -> list[str]: names_str = ", ".join(name_parts) output.append(f"{line_num}: from {module} import {names_str}{flags_str}") + elif isinstance(imp, ImportAll): + # Format: line: from foo import * [flags] + # Handle relative imports + if imp.relative > 0: + prefix = "." * imp.relative + if imp.id: + module = f"{prefix}{imp.id}" + else: + module = prefix + else: + module = imp.id + + output.append(f"{line_num}: from {module} import *{flags_str}") return output diff --git a/test-data/unit/native-parser-imports.test b/test-data/unit/native-parser-imports.test index 0dc55cd14dbd9..1a80c4c640d8c 100644 --- a/test-data/unit/native-parser-imports.test +++ b/test-data/unit/native-parser-imports.test @@ -362,3 +362,40 @@ import regular [out] 3: import from_func_call 4: import regular + +[case testStarImport] +from foo import * +[out] +1: from foo import * + +[case testRelativeStarImport] +from . import * +from ..bar import * +from ...baz import * +[out] +1: from . import * +2: from ..bar import * +3: from ...baz import * + +[case testStarImportInFunction] +def f(): + from foo import * +[out] +2: from foo import * [not top_level] + +[case testMypyOnlyStarImport] +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from mypy_types import * +from regular import * +[out] +1: from typing import TYPE_CHECKING +3: from mypy_types import * [mypy_only] +4: from regular import * + +[case testUnreachableStarImport] +if PY2: + from unreachable import * +from reachable import * +[out] +3: from reachable import * From 363a70aab836c39c63d1d27c1f7299eda8762448 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 8 Feb 2026 15:29:15 +0000 Subject: [PATCH 161/192] Fix conditional overloads --- mypy/nativeparse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 0c5418cc8187b..64e9e04ca2e3b 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -1945,7 +1945,7 @@ def check_ifstmt_for_overloads( return None overload_name = cast(Decorator | FuncDef | OverloadedFuncDef, stmt.body[0].body[-1]).name - if stmt.else_body is None: + if stmt.else_body is None or stmt.else_body.is_unreachable: return overload_name if len(stmt.else_body.body) == 1: @@ -1986,7 +1986,7 @@ def get_executable_if_block_with_overloads( ): # The truth value is unknown, thus not conclusive return None, stmt - if stmt.else_body.is_unreachable is True: + if stmt.else_body.is_unreachable: # else_body will be set unreachable if condition is always True return stmt.body[0], None if stmt.body[0].is_unreachable is True: From 3945f7d16f12197111e28a25696d576294a63ab2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 8 Feb 2026 15:31:23 +0000 Subject: [PATCH 162/192] Lint --- mypy/nativeparse.py | 38 ++++++++++++-------------------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 64e9e04ca2e3b..5759ea91dd500 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -29,10 +29,6 @@ ) from mypy import message_registry, nodes, types -from mypy.options import Options -from mypy.reachability import infer_reachability_of_if_statement -from mypy.sharedparse import special_function_elide_names -from mypy.util import unnamed_function from mypy.cache import ( DICT_STR_GEN, END_TAG, @@ -53,12 +49,9 @@ ARG_KINDS, ARG_POS, IMPORT_METADATA, - IMPORTFROM_METADATA, IMPORTALL_METADATA, + IMPORTFROM_METADATA, MISSING_FALLBACK, - PARAM_SPEC_KIND, - TYPE_VAR_KIND, - TYPE_VAR_TUPLE_KIND, Argument, AssertStmt, AssignmentExpr, @@ -128,6 +121,7 @@ YieldExpr, YieldFromExpr, ) +from mypy.options import Options from mypy.patterns import ( AsPattern, ClassPattern, @@ -139,6 +133,8 @@ StarredPattern, ValuePattern, ) +from mypy.reachability import infer_reachability_of_if_statement +from mypy.sharedparse import special_function_elide_names from mypy.types import ( AnyType, CallableArgument, @@ -153,6 +149,7 @@ UnionType, UnpackType, ) +from mypy.util import unnamed_function TypeIgnores = list[tuple[int, list[str]]] @@ -185,13 +182,9 @@ def add_error( blocker: If True, this error blocks further analysis code: Error code for categorization """ - self.errors.append({ - "line": line, - "column": column, - "message": message, - "blocker": blocker, - "code": code, - }) + self.errors.append( + {"line": line, "column": column, "message": message, "blocker": blocker, "code": code} + ) def expect_end_tag(data: ReadBuffer) -> None: @@ -1998,9 +1991,7 @@ def get_executable_if_block_with_overloads( return None, stmt -def fix_function_overloads( - state: State, stmts: list[Statement] -) -> list[Statement]: +def fix_function_overloads(state: State, stmts: list[Statement]) -> list[Statement]: """Merge consecutive function overloads into OverloadedFuncDef nodes. This function processes a list of statements and combines function overloads @@ -2083,9 +2074,7 @@ def fix_function_overloads( skipped_if_stmts.extend(cast(list[IfStmt], if_block_with_overload.body[:-1])) current_overload.extend(if_block_with_overload.body[-1].items) else: - current_overload.append( - cast(Decorator | FuncDef, if_block_with_overload.body[0]) - ) + current_overload.append(cast(Decorator | FuncDef, if_block_with_overload.body[0])) else: if last_if_stmt is not None: ret.append(last_if_stmt) @@ -2126,12 +2115,9 @@ def fix_function_overloads( last_if_stmt = stmt last_if_stmt_overload_name = None if if_block_with_overload is not None: - skipped_if_stmts.extend( - cast(list[IfStmt], if_block_with_overload.body[:-1]) - ) + skipped_if_stmts.extend(cast(list[IfStmt], if_block_with_overload.body[:-1])) last_if_overload = cast( - Decorator | FuncDef | OverloadedFuncDef, - if_block_with_overload.body[-1], + Decorator | FuncDef | OverloadedFuncDef, if_block_with_overload.body[-1] ) last_if_unknown_truth_value = if_unknown_truth_value else: From c6bc1af00eb45d605ea31605f3508ce4981bc2bb Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 8 Feb 2026 15:34:57 +0000 Subject: [PATCH 163/192] Simplify redundant tests --- test-data/unit/native-parser-imports.test | 68 ----------------------- 1 file changed, 68 deletions(-) diff --git a/test-data/unit/native-parser-imports.test b/test-data/unit/native-parser-imports.test index 1a80c4c640d8c..a5c908b9f6a9c 100644 --- a/test-data/unit/native-parser-imports.test +++ b/test-data/unit/native-parser-imports.test @@ -59,16 +59,6 @@ def f(): 2: import foo [not top_level] 3: from bar import baz [not top_level] -[case testMixedTopLevel] -import toplevel -def f(): - import inside_func -import another_toplevel -[out] -1: import toplevel -3: import inside_func [not top_level] -4: import another_toplevel - [case testImportInClass] class Foo: import bar @@ -82,14 +72,6 @@ import reachable_module [out] 4: import reachable_module -[case testUnreachableFromImportPY2] -# From imports in unreachable blocks should not appear -if PY2: - from unreachable import foo -from reachable import bar -[out] -4: from reachable import bar - [case testReachableImportPY3] # Imports in reachable if PY3 blocks should appear if PY3: @@ -153,17 +135,6 @@ else: 2: import sys 4: import reachable_version_check -[case testElifUnreachable] -# Elif after always-true condition is unreachable -if PY3: - import reachable_if -elif True: - import unreachable_elif -else: - import unreachable_else -[out] -3: import reachable_if - [case testAllBranchesUnreachable] # When all branches are unreachable, no imports appear if PY2: @@ -186,17 +157,6 @@ import regular_module 4: import mypy_only_module [mypy_only] 5: import regular_module -[case testMypyOnlyFromImport] -# From imports in TYPE_CHECKING blocks are mypy_only -from typing import TYPE_CHECKING -if TYPE_CHECKING: - from mypy_types import MyType -from regular import RegularType -[out] -2: from typing import TYPE_CHECKING -4: from mypy_types import MyType [mypy_only] -5: from regular import RegularType - [case testMypyNameAlias] # MYPY is also recognized as mypy_only if MYPY: @@ -344,39 +304,11 @@ elif x: 2: from typing import TYPE_CHECKING 4: import for_mypy [mypy_only] -[case testComplexCondition] -# Complex boolean conditions that can't be evaluated -if x and y: - import complex_if -elif z or w: - import complex_elif -[out] -3: import complex_if -5: import complex_elif - -[case testUnknownWithFunction] -# Unknown condition with function call -if func(): - import from_func_call -import regular -[out] -3: import from_func_call -4: import regular - [case testStarImport] from foo import * [out] 1: from foo import * -[case testRelativeStarImport] -from . import * -from ..bar import * -from ...baz import * -[out] -1: from . import * -2: from ..bar import * -3: from ...baz import * - [case testStarImportInFunction] def f(): from foo import * From eb72dbf8f13c6e62ed421db31994bd521979a96a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 10 Feb 2026 19:32:20 +0000 Subject: [PATCH 164/192] Pass Python version to ast_serialize (fixes self check) --- mypy/nativeparse.py | 8 +++++--- mypy/test/test_nativeparse.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 5759ea91dd500..b02b42f6fd8c3 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -308,7 +308,7 @@ def native_parse( node.path = filename return node, [], [] - b, errors, ignores, import_bytes = parse_to_binary_ast(filename, skip_function_bodies) + b, errors, ignores, import_bytes = parse_to_binary_ast(filename, options, skip_function_bodies) data = ReadBuffer(b) n = read_int(data) state = State(options) @@ -337,9 +337,11 @@ def read_statements(state: State, data: ReadBuffer, n: int) -> list[Statement]: def parse_to_binary_ast( - filename: str, skip_function_bodies: bool = False + filename: str, options: Options, skip_function_bodies: bool = False ) -> tuple[bytes, list[dict[str, Any]], TypeIgnores, bytes]: - ast_bytes, errors, ignores, import_bytes = ast_serialize.parse(filename, skip_function_bodies) + ast_bytes, errors, ignores, import_bytes = ast_serialize.parse( + filename, skip_function_bodies, python_version=options.python_version + ) return ast_bytes, errors, ignores, import_bytes diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index 9fcb13bb3e378..141f5fe9137e1 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -217,7 +217,7 @@ def locs(start_line: int, start_column: int, end_line: int, end_column: int) -> ] with temp_source("print('hello')") as fnam: - b, _, _, _ = parse_to_binary_ast(fnam) + b, _, _, _ = parse_to_binary_ast(fnam, Options()) assert list(b) == ( [LITERAL_INT, 22, nodes.EXPR_STMT, nodes.CALL_EXPR] + [nodes.NAME_EXPR, LITERAL_STR] From ae963f0152276378022e884ce810e02b7b0c4ac2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 15 Feb 2026 14:06:53 +0000 Subject: [PATCH 165/192] Skip native parser tests if ast_serialize is not installed --- mypy/nativeparse.py | 2 +- mypy/test/test_nativeparse.py | 21 +++++++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index b02b42f6fd8c3..234a0a8ff1117 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -2018,7 +2018,7 @@ def fix_function_overloads(state: State, stmts: list[Statement]) -> list[Stateme # Check IfStmt block to determine if function overloads can be merged if_overload_name = check_ifstmt_for_overloads(stmt, current_overload_name) if if_overload_name is not None: - (if_block_with_overload, if_unknown_truth_value) = ( + if_block_with_overload, if_unknown_truth_value = ( get_executable_if_block_with_overloads(stmt, state.options) ) diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index 141f5fe9137e1..6f1be89d86a15 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -1,4 +1,8 @@ -"""Tests for the native mypy parser.""" +"""Tests for the experimental native mypy parser. + +To run these, you will need to manually install ast_serialize from +https://github.com/mypyc/ast_serialize first (see the README for the details). +""" from __future__ import annotations @@ -14,20 +18,28 @@ from mypy.cache import END_TAG, LIST_GEN, LITERAL_INT, LITERAL_STR, LOCATION from mypy.config_parser import parse_mypy_comments from mypy.errors import CompileError -from mypy.nativeparse import native_parse, parse_to_binary_ast from mypy.nodes import ExpressionStmt, MemberExpr, MypyFile from mypy.options import Options from mypy.test.data import DataDrivenTestCase, DataSuite from mypy.test.helpers import assert_string_arrays_equal from mypy.util import get_mypy_comments +# If the experimental ast_serialize module isn't installed, the following import will fail +# and we won't run any native parser tests. +try: + from mypy.nativeparse import native_parse, parse_to_binary_ast + + has_nativeparse = True +except ImportError: + has_nativeparse = False + gc.set_threshold(200 * 1000, 30, 30) class NativeParserSuite(DataSuite): required_out_section = True base_path = "." - files = ["native-parser.test"] + files = ["native-parser.test"] if has_nativeparse else [] def run_case(self, testcase: DataDrivenTestCase) -> None: test_parser(testcase) @@ -36,7 +48,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: class NativeParserImportsSuite(DataSuite): required_out_section = True base_path = "." - files = ["native-parser-imports.test"] + files = ["native-parser-imports.test"] if has_nativeparse else [] def run_case(self, testcase: DataDrivenTestCase) -> None: test_parser_imports(testcase) @@ -202,6 +214,7 @@ def format_reachable_imports(node: MypyFile) -> list[str]: return output +@unittest.skipUnless(has_nativeparse, "nativeparse not available") class TestNativeParse(unittest.TestCase): def test_trivial_binary_data(self) -> None: def int_enc(n: int) -> int: From 46f810d327fec5df59cdca49fcfd10dbe9a4cac6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 15 Feb 2026 14:08:27 +0000 Subject: [PATCH 166/192] Cleanup: Remove temporary benchmark tests --- mypy/test/test_nativeparse.py | 40 ----------------------------------- 1 file changed, 40 deletions(-) diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index 6f1be89d86a15..3b2543d5304dd 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -33,8 +33,6 @@ except ImportError: has_nativeparse = False -gc.set_threshold(200 * 1000, 30, 30) - class NativeParserSuite(DataSuite): required_out_section = True @@ -258,44 +256,6 @@ def test_deserialize_member_expr(self) -> None: assert isinstance(node.defs[0], ExpressionStmt) assert isinstance(node.defs[0].expr, MemberExpr) - def test_deserialize_bench(self) -> None: - with temp_source("print('hello')\n" * 4000) as fnam: - import time - - options = Options() - for i in range(10): - native_parse(fnam, options) - t0 = time.time() - for i in range(25): - node, _, _ = native_parse(fnam, options) - assert isinstance(node, MypyFile) - print(len(node.defs)) - print((time.time() - t0) * 1000) - assert False, 1 / ((time.time() - t0) / 100000) - - def test_parse_bench(self) -> None: - with temp_source("print('hello')\n" * 4000) as fnam: - import time - - from mypy.errors import Errors - from mypy.options import Options - from mypy.parse import parse - - o = Options() - - for i in range(10): - with open(fnam, "rb") as f: - data = f.read() - node = parse(data, fnam, "__main__", Errors(o), o) - - t0 = time.time() - for i in range(25): - with open(fnam, "rb") as f: - data = f.read() - node = parse(data, fnam, "__main__", Errors(o), o) - assert isinstance(node, MypyFile) - assert False, 1 / ((time.time() - t0) / 100000) - @contextlib.contextmanager def temp_source(text: str) -> Iterator[str]: From 83914fd71a05169b8a64ea6b3ddb9ddd1885ce64 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 15 Feb 2026 14:08:48 +0000 Subject: [PATCH 167/192] Fix failing native parser test --- mypy/test/test_nativeparse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index 3b2543d5304dd..a9a9c7958ee91 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -246,7 +246,7 @@ def locs(start_line: int, start_column: int, end_line: int, end_column: int) -> def test_deserialize_hello(self) -> None: with temp_source("print('hello')") as fnam: - node = native_parse(fnam, Options()) + node, _, _ = native_parse(fnam, Options()) assert isinstance(node, MypyFile) def test_deserialize_member_expr(self) -> None: From 9e1f65d778b0e9377cc26da7d859af03078451ab Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 15 Feb 2026 14:14:18 +0000 Subject: [PATCH 168/192] Fix test case --- mypy/test/test_nativeparse.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index a9a9c7958ee91..c13cd65861b44 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -7,7 +7,6 @@ from __future__ import annotations import contextlib -import gc import os import tempfile import unittest @@ -15,7 +14,15 @@ from typing import Any from mypy import defaults, nodes -from mypy.cache import END_TAG, LIST_GEN, LITERAL_INT, LITERAL_STR, LOCATION +from mypy.cache import ( + END_TAG, + LIST_GEN, + LIST_INT, + LITERAL_INT, + LITERAL_NONE, + LITERAL_STR, + LOCATION, +) from mypy.config_parser import parse_mypy_comments from mypy.errors import CompileError from mypy.nodes import ExpressionStmt, MemberExpr, MypyFile @@ -234,13 +241,17 @@ def locs(start_line: int, start_column: int, end_line: int, end_column: int) -> + [nodes.NAME_EXPR, LITERAL_STR] + [int_enc(5)] + list(b"print") - + locs(1, 1, 1, 6) + + locs(1, 0, 1, 5) + [END_TAG, LIST_GEN, 22, nodes.STR_EXPR] + [LITERAL_STR, int_enc(5)] + list(b"hello") - + locs(1, 7, 1, 14) + + locs(1, 6, 1, 13) + [END_TAG] - + locs(1, 1, 1, 15) + # arg_kinds: [ARG_POS] + + [LIST_INT, 22, int_enc(0)] + # arg_names: [None] + + [LIST_GEN, 22, LITERAL_NONE] + + locs(1, 0, 1, 14) + [END_TAG, END_TAG] ) From 95319e317b04a537a3f5c608afc37106affecc0b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 15 Feb 2026 15:23:12 +0000 Subject: [PATCH 169/192] Refactor line/column range set --- mypy/nativeparse.py | 52 ++++++++++++++------------------------------- 1 file changed, 16 insertions(+), 36 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 234a0a8ff1117..2431145e54e6d 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -373,10 +373,7 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: return stmt elif tag == nodes.EXPR_STMT: es = ExpressionStmt(read_expression(state, data)) - es.line = es.expr.line - es.column = es.expr.column - es.end_line = es.expr.end_line - es.end_column = es.expr.end_column + set_line_column_range(es, es.expr) expect_end_tag(data) return es elif tag == nodes.ASSIGNMENT_STMT: @@ -394,10 +391,7 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: read_loc(data, a) # If rvalue is TempNode, copy location from AssignmentStmt if isinstance(rvalue, TempNode): - rvalue.line = a.line - rvalue.column = a.column - rvalue.end_line = a.end_line - rvalue.end_column = a.end_column + set_line_column_range(rvalue, a) expect_end_tag(data) return a elif tag == nodes.OPERATOR_ASSIGNMENT_STMT: @@ -452,10 +446,7 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: # Wrap in a Block to become the else clause for the outer if current_else = Block([elif_stmt]) - current_else.line = elif_stmt.line - current_else.column = elif_stmt.column - current_else.end_line = elif_stmt.end_line - current_else.end_column = elif_stmt.end_column + set_line_column_range(current_else, elif_stmt) # Create the main if statement if_stmt = IfStmt([expr], [body], current_else) @@ -724,10 +715,7 @@ def read_parameters(state: State, data: ReadBuffer) -> tuple[list[Argument], boo var.is_inferred = False arg = Argument(var, ann, default, arg_kind, pos_only) read_loc(data, arg) - var.line = arg.line - var.column = arg.column - var.end_line = arg.end_line - var.end_column = arg.end_column + set_line_column_range(var, arg) arguments.append(arg) return arguments, has_ann @@ -924,10 +912,7 @@ def read_type_alias_stmt(state: State, data: ReadBuffer) -> TypeAliasStmt: # Wrap the value expression in a LambdaExpr as expected by TypeAliasStmt # The LambdaExpr body is a Block with a single ReturnStmt return_stmt = ReturnStmt(value_expr) - return_stmt.line = value_expr.line - return_stmt.column = value_expr.column - return_stmt.end_line = value_expr.end_line - return_stmt.end_column = value_expr.end_column + set_line_column_range(return_stmt, value_expr) block = Block([return_stmt]) block.line = -1 # Synthetic block @@ -936,10 +921,7 @@ def read_type_alias_stmt(state: State, data: ReadBuffer) -> TypeAliasStmt: block.end_column = 0 lambda_expr = LambdaExpr([], block) - lambda_expr.line = value_expr.line - lambda_expr.column = value_expr.column - lambda_expr.end_line = value_expr.end_line - lambda_expr.end_column = value_expr.end_column + set_line_column_range(lambda_expr, value_expr) stmt = TypeAliasStmt(name, type_params, lambda_expr) read_loc(data, stmt) @@ -1368,10 +1350,7 @@ def read_expression(state: State, data: ReadBuffer) -> Expression: expr = ListComprehension(generator) read_loc(data, expr) # Also copy location to the inner generator - generator.line = expr.line - generator.column = expr.column - generator.end_line = expr.end_line - generator.end_column = expr.end_column + set_line_column_range(generator, expr) expect_end_tag(data) return expr elif tag == nodes.SET_COMPREHENSION: @@ -1379,10 +1358,7 @@ def read_expression(state: State, data: ReadBuffer) -> Expression: expr = SetComprehension(generator) read_loc(data, expr) # Also copy location to the inner generator - generator.line = expr.line - generator.column = expr.column - generator.end_line = expr.end_line - generator.end_column = expr.end_column + set_line_column_range(generator, expr) expect_end_tag(data) return expr elif tag == nodes.DICT_COMPREHENSION: @@ -1703,6 +1679,13 @@ def set_line_column(target: Context, src: Context) -> None: target.column = src.column +def set_line_column_range(target: Context, src: Context) -> None: + target.line = src.line + target.column = src.column + target.end_line = src.end_line + target.end_column = src.end_column + + def read_expression_list(state: State, data: ReadBuffer) -> list[Expression]: expect_tag(data, LIST_GEN) n = read_int_bare(data) @@ -1865,10 +1848,7 @@ def read_call_type(state: State, data: ReadBuffer) -> Type: # Create CallableArgument call_arg = CallableArgument(typ, name, constructor) - call_arg.line = invalid.line - call_arg.column = invalid.column - call_arg.end_line = invalid.end_line - call_arg.end_column = invalid.end_column + set_line_column_range(call_arg, invalid) return call_arg From 60f733c04914c5b0af192086ff37530d69af1ceb Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 15 Feb 2026 15:25:38 +0000 Subject: [PATCH 170/192] Add note about ast_serialize repo --- mypy/nativeparse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 2431145e54e6d..b53e62b961ded 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -4,7 +4,8 @@ Use a Rust extension to generate a serialized AST, and deserialize the AST directly to a mypy AST. -NOTE: This is heavily work in progress. +NOTE: This is heavily work in progress. To use this, you need to manually build the + ast_serialize Rust extension, see instructions in https://github.com/mypyc/ast_serialize. Expected benefits over mypy.fastparse: * No intermediate non-mypyc AST created, to improve performance From 7ccf14f41da7dea4306bec22e4d5d904b4b2f385 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 15 Feb 2026 15:39:20 +0000 Subject: [PATCH 171/192] Remove some low-value comments from mypy.nativeparse --- mypy/nativeparse.py | 145 +------------------------------------------- 1 file changed, 3 insertions(+), 142 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index b53e62b961ded..598614abc4914 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -203,10 +203,9 @@ def _read_and_set_import_metadata(data: ReadBuffer, stmt: Import | ImportFrom | data: Buffer containing serialized data stmt: Import, ImportFrom, or ImportAll statement to populate with location and metadata """ - # Read location read_loc(data, stmt) - # Read metadata flags as a single integer bitfield + # Metadata flags as a single integer bitfield flags = read_int(data) # Extract individual flags using bitwise operations @@ -232,7 +231,6 @@ def deserialize_imports(import_bytes: bytes) -> list[ImportBase]: data = ReadBuffer(import_bytes) - # Read list header expect_tag(data, LIST_GEN) n_imports = read_int_bare(data) @@ -242,29 +240,24 @@ def deserialize_imports(import_bytes: bytes) -> list[ImportBase]: tag = read_tag(data) if tag == IMPORT_METADATA: - # Read Import fields name = read_str(data) relative = read_int(data) - # Read optional as_name has_asname = read_bool(data) if has_asname: asname = read_str(data) else: asname = None - # Create Import node and read common metadata # Note: relative imports are handled via ImportFrom, so relative should be 0 here stmt = Import([(name, asname)]) _read_and_set_import_metadata(data, stmt) imports.append(stmt) elif tag == IMPORTFROM_METADATA: - # Read ImportFrom fields module = read_str(data) relative = read_int(data) - # Read list of names expect_tag(data, LIST_GEN) n_names = read_int_bare(data) names: list[tuple[str, str | None]] = [] @@ -278,17 +271,14 @@ def deserialize_imports(import_bytes: bytes) -> list[ImportBase]: asname = None names.append((name, asname)) - # Create ImportFrom node and read common metadata stmt = ImportFrom(module, relative, names) _read_and_set_import_metadata(data, stmt) imports.append(stmt) elif tag == IMPORTALL_METADATA: - # Read ImportAll fields module = read_str(data) relative = read_int(data) - # Create ImportAll node and read common metadata stmt = ImportAll(module, relative) _read_and_set_import_metadata(data, stmt) imports.append(stmt) @@ -315,7 +305,6 @@ def native_parse( state = State(options) defs = read_statements(state, data, n) - # Deserialize import metadata imports = deserialize_imports(import_bytes) node = MypyFile(defs, imports) @@ -363,7 +352,6 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: var = Var(fdef.name) var.line = fdef.line var.is_ready = False - # Create Decorator wrapping the FuncDef stmt = Decorator(fdef, decorators, var) stmt.line = line stmt.column = column @@ -380,13 +368,11 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: elif tag == nodes.ASSIGNMENT_STMT: lvalues = read_expression_list(state, data) rvalue = read_expression(state, data) - # Read type annotation has_type = read_bool(data) if has_type: type_annotation = read_type(state, data) else: type_annotation = None - # Read new_syntax flag new_syntax = read_bool(data) a = AssignmentStmt(lvalues, rvalue, type=type_annotation, new_syntax=new_syntax) read_loc(data, a) @@ -419,7 +405,6 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: elif_exprs.append(read_expression(state, data)) elif_bodies.append(read_block(state, data)) - # Read else clause has_else = read_bool(data) if has_else: else_body = read_block(state, data) @@ -432,7 +417,6 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: # Process elif clauses in reverse order for i in range(len(elif_exprs) - 1, -1, -1): - # Create an IfStmt for this elif elif_stmt = IfStmt([elif_exprs[i]], [elif_bodies[i]], current_else) # Set location from the elif expression elif_stmt.line = elif_exprs[i].line @@ -449,7 +433,6 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: current_else = Block([elif_stmt]) set_line_column_range(current_else, elif_stmt) - # Create the main if statement if_stmt = IfStmt([expr], [body], current_else) read_loc(data, if_stmt) expect_end_tag(data) @@ -465,13 +448,11 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: expect_end_tag(data) return stmt elif tag == nodes.RAISE_STMT: - # Read exception expression (optional) has_exc = read_bool(data) if has_exc: exc = read_expression(state, data) else: exc = None - # Read from expression (optional) has_from = read_bool(data) if has_from: from_expr = read_expression(state, data) @@ -482,9 +463,7 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: expect_end_tag(data) return stmt elif tag == nodes.ASSERT_STMT: - # Read test expression test = read_expression(state, data) - # Read optional message expression has_msg = read_bool(data) if has_msg: msg = read_expression(state, data) @@ -503,15 +482,10 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: expect_end_tag(data) return stmt elif tag == nodes.FOR_STMT: - # Read index (target) index = read_expression(state, data) - # Read iterator expression expr = read_expression(state, data) - # Read body body = read_block(state, data) - # Read else clause else_body = read_optional_block(state, data) - # Read is_async flag is_async = read_bool(data) stmt = ForStmt(index, expr, body, else_body) stmt.is_async = is_async @@ -519,25 +493,19 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: expect_end_tag(data) return stmt elif tag == nodes.WITH_STMT: - # Read number of items n = read_int(data) expr_list = [] target_list: list[Expression | None] = [] - # Read each item for _ in range(n): - # Read context expression context_expr = read_expression(state, data) expr_list.append(context_expr) - # Read optional target has_target = read_bool(data) if has_target: target = read_expression(state, data) target_list.append(target) else: target_list.append(None) - # Read body body = read_block(state, data) - # Read is_async flag is_async = read_bool(data) stmt = WithStmt(expr_list, target_list, body) stmt.is_async = is_async @@ -560,13 +528,10 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: expect_end_tag(data) return stmt elif tag == nodes.IMPORT: - # Read number of imports n = read_int(data) ids = [] for _ in range(n): - # Read import name name = read_str(data) - # Read as_name (optional) has_asname = read_bool(data) if has_asname: asname = read_str(data) @@ -578,19 +543,12 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: expect_end_tag(data) return stmt elif tag == nodes.IMPORT_FROM: - # Read relative import level relative = read_int(data) - - # Read module name (empty string for "from . import x") - module_id = read_str(data) - - # Read number of imported names + module_id = read_str(data) # Empty string for "from . import x" n = read_int(data) names = [] for _ in range(n): - # Read imported name name = read_str(data) - # Read optional alias has_asname = read_bool(data) if has_asname: asname = read_str(data) @@ -603,10 +561,7 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: expect_end_tag(data) return stmt elif tag == nodes.IMPORT_ALL: - # Read module name (empty string for "from . import *") - module_id = read_str(data) - - # Read relative import level + module_id = read_str(data) # Empty string for "from . import *" relative = read_int(data) stmt = ImportAll(module_id, relative) @@ -620,14 +575,12 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: elif tag == nodes.TRY_STMT: return read_try_stmt(state, data) elif tag == nodes.DEL_STMT: - # Read the target expression expr = read_expression(state, data) stmt = DelStmt(expr) read_loc(data, stmt) expect_end_tag(data) return stmt elif tag == nodes.GLOBAL_DECL: - # Read number of names n = read_int(data) decl_names = [] for _ in range(n): @@ -637,7 +590,6 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: expect_end_tag(data) return stmt elif tag == nodes.NONLOCAL_DECL: - # Read number of names n = read_int(data) decl_names = [] for _ in range(n): @@ -647,25 +599,20 @@ def read_statement(state: State, data: ReadBuffer) -> Statement: expect_end_tag(data) return stmt elif tag == nodes.MATCH_STMT: - # Read subject expression subject = read_expression(state, data) - # Read number of cases n_cases = read_int(data) patterns = [] guards: list[Expression | None] = [] bodies = [] for _ in range(n_cases): - # Read pattern pattern = read_pattern(state, data) patterns.append(pattern) - # Read optional guard has_guard = read_bool(data) if has_guard: guard = read_expression(state, data) guards.append(guard) else: guards.append(None) - # Read body body = read_block(state, data) bodies.append(body) stmt = MatchStmt(subject, patterns, guards, bodies) @@ -689,16 +636,13 @@ def read_parameters(state: State, data: ReadBuffer) -> tuple[list[Argument], boo for _ in range(n_args): arg_name = read_str(data) arg_kind_int = read_int(data) - # Convert integer to ArgKind enum using ARG_KINDS tuple arg_kind = ARG_KINDS[arg_kind_int] - # Read type annotation has_type = read_bool(data) if has_type: ann = read_type(state, data) has_ann = True else: ann = None - # Read default value has_default = read_bool(data) if has_default: default = read_expression(state, data) @@ -727,25 +671,18 @@ def read_type_params(state: State, data: ReadBuffer) -> list[TypeParam]: type_params: list[TypeParam] = [] n = read_int_bare(data) for _ in range(n): - # Read type param kind kind = read_int(data) - - # Read name name = read_str(data) - - # Read upper bound has_bound = read_bool(data) if has_bound: upper_bound = read_type(state, data) else: upper_bound = None - # Read values (for constrained TypeVar) expect_tag(data, LIST_GEN) n_values = read_int_bare(data) values = [read_type(state, data) for _ in range(n_values)] - # Read default has_default = read_bool(data) if has_default: default = read_type(state, data) @@ -760,10 +697,7 @@ def read_type_params(state: State, data: ReadBuffer) -> list[TypeParam]: def read_func_def(state: State, data: ReadBuffer) -> FuncDef: state.num_funcs += 1 - # Function name name = read_str(data) - - # Parameters arguments, has_ann = read_parameters(state, data) if special_function_elide_names(name): @@ -771,7 +705,6 @@ def read_func_def(state: State, data: ReadBuffer) -> FuncDef: arg.pos_only = True body = read_block(state, data) - is_async = read_bool(data) # Type parameters (PEP 695) @@ -781,7 +714,6 @@ def read_func_def(state: State, data: ReadBuffer) -> FuncDef: else: type_params = None - # Return type annotation has_return_type = read_bool(data) if has_return_type: return_type = read_type(state, data) @@ -818,16 +750,10 @@ def read_func_def(state: State, data: ReadBuffer) -> FuncDef: def read_class_def(state: State, data: ReadBuffer) -> ClassDef: - # Class name name = read_str(data) - - # Body body = read_block(state, data) - - # Base classes base_type_exprs = read_expression_list(state, data) - # Decorators expect_tag(data, LIST_GEN) n_decorators = read_int_bare(data) decorators = [read_expression(state, data) for _ in range(n_decorators)] @@ -869,22 +795,15 @@ def read_class_def(state: State, data: ReadBuffer) -> ClassDef: def read_type_alias_stmt(state: State, data: ReadBuffer) -> TypeAliasStmt: """Read PEP 695 type alias statement.""" - # Read name (NameExpr) name = read_expression(state, data) assert isinstance(name, NameExpr), f"Expected NameExpr for type alias name, got {type(name)}" - # Read type parameters n_type_params = read_int_bare(data) if n_type_params > 0: type_params = [] for _ in range(n_type_params): - # Read type param kind kind = read_int(data) - - # Read name param_name = read_str(data) - - # Read upper bound has_bound = read_bool(data) if has_bound: upper_bound = read_type(state, data) @@ -896,7 +815,6 @@ def read_type_alias_stmt(state: State, data: ReadBuffer) -> TypeAliasStmt: n_values = read_int_bare(data) values = [read_type(state, data) for _ in range(n_values)] - # Read default has_default = read_bool(data) if has_default: default = read_type(state, data) @@ -907,7 +825,6 @@ def read_type_alias_stmt(state: State, data: ReadBuffer) -> TypeAliasStmt: else: type_params = [] - # Read value expression (the type on RHS of type alias) value_expr = read_expression(state, data) # Wrap the value expression in a LambdaExpr as expected by TypeAliasStmt @@ -931,13 +848,9 @@ def read_type_alias_stmt(state: State, data: ReadBuffer) -> TypeAliasStmt: def read_try_stmt(state: State, data: ReadBuffer) -> TryStmt: - # Read try body body = read_block(state, data) - - # Read number of except handlers num_handlers = read_int(data) - # Read exception types for each handler types_list: list[Expression | None] = [] for _ in range(num_handlers): has_type = read_bool(data) @@ -947,7 +860,6 @@ def read_try_stmt(state: State, data: ReadBuffer) -> TryStmt: else: types_list.append(None) - # Read variable names for each handler vars_list: list[NameExpr | None] = [] for _ in range(num_handlers): has_name = read_bool(data) @@ -958,20 +870,17 @@ def read_try_stmt(state: State, data: ReadBuffer) -> TryStmt: else: vars_list.append(None) - # Read handler bodies handlers = [] for _ in range(num_handlers): handler_body = read_block(state, data) handlers.append(handler_body) - # Read else body (optional) has_else = read_bool(data) if has_else: else_body = read_block(state, data) else: else_body = None - # Read finally body (optional) has_finally = read_bool(data) if has_finally: finally_body = read_block(state, data) @@ -1104,13 +1013,11 @@ def read_pattern(state: State, data: ReadBuffer) -> Pattern: """Read a pattern node from the buffer.""" tag = read_tag(data) if tag == nodes.AS_PATTERN: - # Read optional pattern has_pattern = read_bool(data) if has_pattern: pattern = read_pattern(state, data) else: pattern = None - # Read optional name has_name = read_bool(data) if has_name: name_str = read_str(data) @@ -1123,7 +1030,6 @@ def read_pattern(state: State, data: ReadBuffer) -> Pattern: expect_end_tag(data) return as_pattern elif tag == nodes.OR_PATTERN: - # Read number of patterns n = read_int(data) patterns = [read_pattern(state, data) for _ in range(n)] or_pattern = OrPattern(patterns) @@ -1131,14 +1037,12 @@ def read_pattern(state: State, data: ReadBuffer) -> Pattern: expect_end_tag(data) return or_pattern elif tag == nodes.VALUE_PATTERN: - # Read value expression expr = read_expression(state, data) value_pattern = ValuePattern(expr) read_loc(data, value_pattern) expect_end_tag(data) return value_pattern elif tag == nodes.SINGLETON_PATTERN: - # Read singleton value singleton_tag = read_tag(data) if singleton_tag == LITERAL_NONE: value = None @@ -1150,7 +1054,6 @@ def read_pattern(state: State, data: ReadBuffer) -> Pattern: expect_end_tag(data) return singleton_pattern elif tag == nodes.SEQUENCE_PATTERN: - # Read number of patterns n = read_int(data) patterns = [read_pattern(state, data) for _ in range(n)] sequence_pattern = SequencePattern(patterns) @@ -1171,7 +1074,6 @@ def read_pattern(state: State, data: ReadBuffer) -> Pattern: expect_end_tag(data) return starred_pattern elif tag == nodes.MAPPING_PATTERN: - # Read number of key-value pairs n = read_int(data) keys = [] values = [] @@ -1180,7 +1082,6 @@ def read_pattern(state: State, data: ReadBuffer) -> Pattern: value = read_pattern(state, data) keys.append(key) values.append(value) - # Read optional rest pattern has_rest = read_bool(data) if has_rest: rest_str = read_str(data) @@ -1193,12 +1094,9 @@ def read_pattern(state: State, data: ReadBuffer) -> Pattern: expect_end_tag(data) return mapping_pattern elif tag == nodes.CLASS_PATTERN: - # Read class reference expression class_ref = cast(RefExpr, read_expression(state, data)) - # Read number of positional patterns n_positional = read_int(data) positionals = [read_pattern(state, data) for _ in range(n_positional)] - # Read number of keyword patterns n_keywords = read_int(data) keyword_keys = [] keyword_values = [] @@ -1363,26 +1261,18 @@ def read_expression(state: State, data: ReadBuffer) -> Expression: expect_end_tag(data) return expr elif tag == nodes.DICT_COMPREHENSION: - # Read key expression key = read_expression(state, data) - # Read value expression value = read_expression(state, data) - # Read number of generators n_generators = read_int(data) - # Read all indices (targets) indices = [read_expression(state, data) for _ in range(n_generators)] - # Read all sequences (iters) sequences = [read_expression(state, data) for _ in range(n_generators)] - # Read all condlists (ifs for each generator) condlists = [read_expression_list(state, data) for _ in range(n_generators)] - # Read all is_async flags is_async = [read_bool(data) for _ in range(n_generators)] expr = DictionaryComprehension(key, value, indices, sequences, condlists, is_async) read_loc(data, expr) expect_end_tag(data) return expr elif tag == nodes.YIELD_EXPR: - # Read optional value expression has_value = read_bool(data) if has_value: value = read_expression(state, data) @@ -1393,7 +1283,6 @@ def read_expression(state: State, data: ReadBuffer) -> Expression: expect_end_tag(data) return expr elif tag == nodes.YIELD_FROM_EXPR: - # Read value expression (required for yield from) value = read_expression(state, data) expr = YieldFromExpr(value) read_loc(data, expr) @@ -1436,11 +1325,9 @@ def read_expression(state: State, data: ReadBuffer) -> Expression: return result elif tag == nodes.COMPARISON_EXPR: left = read_expression(state, data) - # Read operators list expect_tag(data, LIST_INT) n_ops = read_int_bare(data) ops = [cmp_ops[read_int_bare(data)] for _ in range(n_ops)] - # Read comparators list comparators = read_expression_list(state, data) assert len(ops) == len(comparators) expr = ComparisonExpr(ops, [left] + comparators) @@ -1455,7 +1342,6 @@ def read_expression(state: State, data: ReadBuffer) -> Expression: expect_end_tag(data) return expr elif tag == nodes.DICT_EXPR: - # Read keys expect_tag(data, LIST_GEN) n_keys = read_int_bare(data) keys: list[Expression | None] = [] @@ -1465,7 +1351,6 @@ def read_expression(state: State, data: ReadBuffer) -> Expression: keys.append(read_expression(state, data)) else: keys.append(None) - # Read values values = read_expression_list(state, data) # Zip keys and values into items items = list(zip(keys, values)) @@ -1474,26 +1359,20 @@ def read_expression(state: State, data: ReadBuffer) -> Expression: expect_end_tag(data) return expr elif tag == nodes.COMPLEX_EXPR: - # Read real part expect_tag(data, LITERAL_FLOAT) real = read_float_bare(data) - # Read imaginary part expect_tag(data, LITERAL_FLOAT) imag = read_float_bare(data) - # Create complex value value = complex(real, imag) expr = ComplexExpr(value) read_loc(data, expr) expect_end_tag(data) return expr elif tag == nodes.SLICE_EXPR: - # Read begin_index (lower in Ruff) has_begin = read_bool(data) begin_index = read_expression(state, data) if has_begin else None - # Read end_index (upper in Ruff) has_end = read_bool(data) end_index = read_expression(state, data) if has_end else None - # Read stride (step in Ruff) has_stride = read_bool(data) stride = read_expression(state, data) if has_stride else None expr = SliceExpr(begin_index, end_index, stride) @@ -1511,11 +1390,8 @@ def read_expression(state: State, data: ReadBuffer) -> Expression: expect_end_tag(data) return expr elif tag == nodes.CONDITIONAL_EXPR: - # Read if_expr (value when condition is true) if_expr = read_expression(state, data) - # Read cond (the condition) cond = read_expression(state, data) - # Read else_expr (value when condition is false) else_expr = read_expression(state, data) expr = ConditionalExpr(cond, if_expr, else_expr) read_loc(data, expr) @@ -1540,13 +1416,9 @@ def read_expression(state: State, data: ReadBuffer) -> Expression: expect_end_tag(data) return expr elif tag == nodes.LAMBDA_EXPR: - # Read parameters arguments, has_ann = read_parameters(state, data) - - # Read body block body = read_block(state, data) - # Create lambda expression if has_ann: typ = CallableType( [ @@ -1567,9 +1439,7 @@ def read_expression(state: State, data: ReadBuffer) -> Expression: expect_end_tag(data) return expr elif tag == nodes.NAMED_EXPR: - # Read target expression target = read_expression(state, data) - # Read value expression value = read_expression(state, data) # AssignmentExpr expects target to be a NameExpr if not isinstance(target, NameExpr): @@ -1583,7 +1453,6 @@ def read_expression(state: State, data: ReadBuffer) -> Expression: expect_end_tag(data) return expr elif tag == nodes.STAR_EXPR: - # Read the wrapped expression wrapped_expr = read_expression(state, data) expr = StarExpr(wrapped_expr) read_loc(data, expr) @@ -1597,7 +1466,6 @@ def read_expression(state: State, data: ReadBuffer) -> Expression: expect_end_tag(data) return expr elif tag == nodes.AWAIT_EXPR: - # Read awaited expression value = read_expression(state, data) expr = AwaitExpr(value) read_loc(data, expr) @@ -1695,17 +1563,11 @@ def read_expression_list(state: State, data: ReadBuffer) -> list[Expression]: def read_generator_expr(state: State, data: ReadBuffer) -> GeneratorExpr: """Helper function to read comprehension data (shared by Generator, ListComp, SetComp)""" - # Read element expression left_expr = read_expression(state, data) - # Read number of generators n_generators = read_int(data) - # Read all indices (targets) indices = [read_expression(state, data) for _ in range(n_generators)] - # Read all sequences (iters) sequences = [read_expression(state, data) for _ in range(n_generators)] - # Read all condlists (ifs for each generator) condlists = [read_expression_list(state, data) for _ in range(n_generators)] - # Read all is_async flags is_async = [read_bool(data) for _ in range(n_generators)] return GeneratorExpr(left_expr, indices, sequences, condlists, is_async) @@ -1746,7 +1608,6 @@ def read_call_type(state: State, data: ReadBuffer) -> Type: This performs validation and error reporting similar to mypy/fastparse.py. """ - # Read callee callee_type = read_type(state, data) # Read positional arguments From fd23cb8758145c614cf0ff594b7d9d5f5c5ed7a5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 15 Feb 2026 15:41:47 +0000 Subject: [PATCH 172/192] Use native parser in testcheck.py only if env variable TEST_NATIVE_PARSER is set Example: `TEST_NATIVE_PARSER=1 pytest mypy/test/testheck.py`. --- mypy/test/testcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 47e6e87e3d545..3326d642c6a40 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -136,7 +136,7 @@ def run_case_once( options = parse_options(original_program_text, testcase, incremental_step) options.use_builtins_fixtures = True options.show_traceback = True - options.native_parser = True # XXX remove + options.native_parser = bool(os.environ.get("TEST_NATIVE_PARSER")) if options.num_workers: options.fixed_format_cache = True From 98d9a1b72bdba2287f4ae8d12ebd5c40cc0225bb Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 15 Feb 2026 15:44:09 +0000 Subject: [PATCH 173/192] Lint --- test-data/unit/native-parser.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-data/unit/native-parser.test b/test-data/unit/native-parser.test index c211021cbde9e..b9c2e3fe09390 100644 --- a/test-data/unit/native-parser.test +++ b/test-data/unit/native-parser.test @@ -1175,7 +1175,7 @@ MypyFile:1( @dec def foo(): pass -class C: +class C: @dec() @staticmethod def foo(): From 3e87c1a5ea6c8287e2f34f733abcc043bdb026fd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 15 Feb 2026 16:04:13 +0000 Subject: [PATCH 174/192] Update semantic analyzer test outputs --- test-data/unit/semanal-abstractclasses.test | 4 ++++ test-data/unit/semanal-classes.test | 3 +++ 2 files changed, 7 insertions(+) diff --git a/test-data/unit/semanal-abstractclasses.test b/test-data/unit/semanal-abstractclasses.test index b0cb00e82106c..6e317e8d05ac1 100644 --- a/test-data/unit/semanal-abstractclasses.test +++ b/test-data/unit/semanal-abstractclasses.test @@ -13,6 +13,7 @@ MypyFile:1( Import:2(typing) ClassDef:4( A + Keywords(metaclass=NameExpr(ABCMeta [abc.ABCMeta])) Metaclass(NameExpr(ABCMeta [abc.ABCMeta])) Decorator:5( Var(g) @@ -49,10 +50,12 @@ MypyFile:1( Import:2(typing) ClassDef:4( A + Keywords(metaclass=NameExpr(ABCMeta [abc.ABCMeta])) Metaclass(NameExpr(ABCMeta [abc.ABCMeta])) PassStmt:4()) ClassDef:5( B + Keywords(metaclass=NameExpr(ABCMeta [abc.ABCMeta])) Metaclass(NameExpr(ABCMeta [abc.ABCMeta])) PassStmt:5()) ClassDef:6( @@ -106,6 +109,7 @@ MypyFile:1( Import:3(typing) ClassDef:5( A + Keywords(metaclass=NameExpr(ABCMeta [abc.ABCMeta])) Metaclass(NameExpr(ABCMeta [abc.ABCMeta])) Decorator:6( Var(g) diff --git a/test-data/unit/semanal-classes.test b/test-data/unit/semanal-classes.test index c43d7754c75f4..e30dbac8ecbb0 100644 --- a/test-data/unit/semanal-classes.test +++ b/test-data/unit/semanal-classes.test @@ -413,6 +413,9 @@ MypyFile:1( Import:1(abc) ClassDef:2( A + Keywords(metaclass=MemberExpr:2( + NameExpr(abc) + ABCMeta [abc.ABCMeta])) Metaclass(MemberExpr:2( NameExpr(abc) ABCMeta [abc.ABCMeta])) From 840bd1ed9c9eae8bf8fc443d791040ec9c396159 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 15 Feb 2026 16:11:11 +0000 Subject: [PATCH 175/192] Fix class keywords in tree transform --- mypy/treetransform.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/treetransform.py b/mypy/treetransform.py index 1a76a50a2d94e..fb783a0ce7596 100644 --- a/mypy/treetransform.py +++ b/mypy/treetransform.py @@ -259,12 +259,14 @@ def visit_overloaded_func_def(self, node: OverloadedFuncDef) -> OverloadedFuncDe return new def visit_class_def(self, node: ClassDef) -> ClassDef: + keywords = [(key, self.expr(value)) for key, value in node.keywords.items()] new = ClassDef( node.name, self.block(node.defs), node.type_vars, self.expressions(node.base_type_exprs), self.optional_expr(node.metaclass), + keywords, ) new.fullname = node.fullname new.info = node.info From 144f6d6ef53ae8a00edadeef0f80c0d42ac23d5e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 18 Feb 2026 01:00:05 +0000 Subject: [PATCH 176/192] Fix an obvious ordering bug to correctly set line --- mypy/nativeparse.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 598614abc4914..fcbd86319a9d2 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -736,15 +736,15 @@ def read_func_def(state: State, data: ReadBuffer) -> FuncDef: typ = None func_def = FuncDef(name, arguments, body, typ=typ, type_args=type_params) - if typ: - # TODO: This seems wasteful, can we avoid it? - func_def.unanalyzed_type = typ.copy_modified() - - typ.definition = func_def - typ.line = func_def.line if is_async: func_def.is_coroutine = True read_loc(data, func_def) + if typ: + typ.line = func_def.line + typ.column = func_def.column + typ.definition = func_def + # TODO: This seems wasteful, can we avoid it? + func_def.unanalyzed_type = typ.copy_modified() expect_end_tag(data) return func_def From 67ff2e555e384941e7a14157db81715e083cc678 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 18 Feb 2026 02:00:33 +0000 Subject: [PATCH 177/192] Always allow new union syntax inside strings/comments --- mypy/nativeparse.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index fcbd86319a9d2..0131d1c6906ae 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -955,6 +955,8 @@ def read_type(state: State, data: ReadBuffer) -> Type: union = UnionType(items, uses_pep604_syntax=uses_pep604_syntax) union.original_str_expr = original_str_expr union.original_str_fallback = original_str_fallback + if original_str_expr is not None: + union.is_evaluated = False read_loc(data, union) expect_end_tag(data) return union From 4270ff8190e92d296c7754973d7d091e93abd3af Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 18 Feb 2026 10:28:28 +0000 Subject: [PATCH 178/192] Couple tuple-related fixes (#20840) This is the mypy counter-part of https://github.com/mypyc/ast_serialize/pull/12. Depends on that PR to work. --- mypy/nativeparse.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 0131d1c6906ae..ece6595ecf382 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -143,6 +143,7 @@ EllipsisType, Instance, RawExpressionType, + TupleType, Type, TypeList, TypeOfAny, @@ -904,6 +905,7 @@ def read_type(state: State, data: ReadBuffer) -> Type: expect_tag(data, LIST_GEN) n = read_int_bare(data) args = tuple(read_type(state, data) for i in range(n)) + empty_tuple_index = read_bool(data) # Read optional original_str_expr t = read_tag(data) if t == LITERAL_NONE: @@ -923,6 +925,7 @@ def read_type(state: State, data: ReadBuffer) -> Type: unbound = UnboundType( name, args, + empty_tuple_index=empty_tuple_index, original_str_expr=original_str_expr, original_str_fallback=original_str_fallback, ) @@ -969,6 +972,16 @@ def read_type(state: State, data: ReadBuffer) -> Type: read_loc(data, type_list) expect_end_tag(data) return type_list + elif tag == types.TUPLE_TYPE: + # Read items list + expect_tag(data, LIST_GEN) + n = read_int_bare(data) + items = [read_type(state, data) for i in range(n)] + implicit = read_bool(data) + tuple_type = TupleType(items, _dummy_fallback, implicit=implicit) + read_loc(data, tuple_type) + expect_end_tag(data) + return tuple_type elif tag == types.ELLIPSIS_TYPE: # EllipsisType has no attributes ellipsis_type = EllipsisType() From 3756c17e88db9dde6b560c6fb0c0dbb0ac5af533 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 18 Feb 2026 15:09:12 +0000 Subject: [PATCH 179/192] Detect partial stub packages (#20844) This is the mypy counter-part of https://github.com/mypyc/ast_serialize/pull/13 (I am not actually using the new flag yet in `build.py`, I will do this later when the branch is in master) --- mypy/nativeparse.py | 11 +++++++---- mypy/test/test_nativeparse.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index ece6595ecf382..bb13b121a9e6c 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -300,7 +300,9 @@ def native_parse( node.path = filename return node, [], [] - b, errors, ignores, import_bytes = parse_to_binary_ast(filename, options, skip_function_bodies) + b, errors, ignores, import_bytes, is_partial_package = parse_to_binary_ast( + filename, options, skip_function_bodies + ) data = ReadBuffer(b) n = read_int(data) state = State(options) @@ -310,6 +312,7 @@ def native_parse( node = MypyFile(defs, imports) node.path = filename + node.is_partial_stub_package = is_partial_package # Merge deserialization errors with parsing errors all_errors = errors + state.errors return node, all_errors, ignores @@ -329,11 +332,11 @@ def read_statements(state: State, data: ReadBuffer, n: int) -> list[Statement]: def parse_to_binary_ast( filename: str, options: Options, skip_function_bodies: bool = False -) -> tuple[bytes, list[dict[str, Any]], TypeIgnores, bytes]: - ast_bytes, errors, ignores, import_bytes = ast_serialize.parse( +) -> tuple[bytes, list[dict[str, Any]], TypeIgnores, bytes, bool]: + ast_bytes, errors, ignores, import_bytes, is_partial_package = ast_serialize.parse( filename, skip_function_bodies, python_version=options.python_version ) - return ast_bytes, errors, ignores, import_bytes + return ast_bytes, errors, ignores, import_bytes, is_partial_package def read_statement(state: State, data: ReadBuffer) -> Statement: diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index c13cd65861b44..13777d59cf4b3 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -235,7 +235,7 @@ def locs(start_line: int, start_column: int, end_line: int, end_column: int) -> ] with temp_source("print('hello')") as fnam: - b, _, _, _ = parse_to_binary_ast(fnam, Options()) + b, _, _, _, _ = parse_to_binary_ast(fnam, Options()) assert list(b) == ( [LITERAL_INT, 22, nodes.EXPR_STMT, nodes.CALL_EXPR] + [nodes.NAME_EXPR, LITERAL_STR] From 854eea826f491204eeb28a923bd801b97ee3e833 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 19 Feb 2026 12:32:20 +0000 Subject: [PATCH 180/192] Add support for inline TypedDicts (#20847) This is the mypy counterpart of https://github.com/mypyc/ast_serialize/pull/17 --- mypy/nativeparse.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index bb13b121a9e6c..7b4570856e0f1 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -44,6 +44,7 @@ read_bool, read_int, read_str, + read_str_opt, read_tag, ) from mypy.nodes import ( @@ -142,9 +143,11 @@ CallableType, EllipsisType, Instance, + ProperType, RawExpressionType, TupleType, Type, + TypedDictType, TypeList, TypeOfAny, UnboundType, @@ -985,6 +988,26 @@ def read_type(state: State, data: ReadBuffer) -> Type: read_loc(data, tuple_type) expect_end_tag(data) return tuple_type + elif tag == types.TYPED_DICT_TYPE: + expect_tag(data, LIST_GEN) + n = read_int_bare(data) + keys = [read_str_opt(data) for i in range(n)] + expect_tag(data, LIST_GEN) + n = read_int_bare(data) + values = [read_type(state, data) for i in range(n)] + td_items = {} + extra_items_from = [] + for key, val in zip(keys, values): + if key is None: + assert isinstance(val, ProperType) + extra_items_from.append(val) + else: + td_items[key] = val + typeddict_type = TypedDictType(td_items, set(), set(), _dummy_fallback) + typeddict_type.extra_items_from = extra_items_from + read_loc(data, typeddict_type) + expect_end_tag(data) + return typeddict_type elif tag == types.ELLIPSIS_TYPE: # EllipsisType has no attributes ellipsis_type = EllipsisType() From fa08d892c74967e079ebc5b0e242384a4c0ba45a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 19 Feb 2026 17:49:23 +0000 Subject: [PATCH 181/192] Fix couple more edge cases type comments/strings (#20848) This is mypy counterpart for https://github.com/mypyc/ast_serialize/pull/18 --- mypy/nativeparse.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 7b4570856e0f1..13676c9058b53 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -964,8 +964,7 @@ def read_type(state: State, data: ReadBuffer) -> Type: union = UnionType(items, uses_pep604_syntax=uses_pep604_syntax) union.original_str_expr = original_str_expr union.original_str_fallback = original_str_fallback - if original_str_expr is not None: - union.is_evaluated = False + union.is_evaluated = read_bool(data) read_loc(data, union) expect_end_tag(data) return union From 3869d792e1a39fc455b355d147b78ca7e54ac85f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 21 Feb 2026 11:15:00 +0000 Subject: [PATCH 182/192] Update docstring --- mypy/nativeparse.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 13676c9058b53..651f26d105d20 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -5,13 +5,13 @@ to a mypy AST. NOTE: This is heavily work in progress. To use this, you need to manually build the - ast_serialize Rust extension, see instructions in https://github.com/mypyc/ast_serialize. + ast_serialize Rust extension. More information at https://github.com/mypyc/ast_serialize. Expected benefits over mypy.fastparse: - * No intermediate non-mypyc AST created, to improve performance + * No intermediate non-mypyc Python-level AST created, to improve performance * Parsing doesn't need GIL => use multithreading to construct serialized ASTs in parallel * Produce import dependencies without having to build an AST => helps parallel type checking - * Support all Python syntax even if running mypy on an older Python version + * Support all Python syntax even if mypy is running on an older Python version * Generate an AST even if there are syntax errors * Potential to support incremental parsing (quickly process modified sections in a file) * Stripping function bodies in third-party code can happen earlier, for extra performance From 512bf1587ed3ead14f88128def9f889dee41e560 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 21 Feb 2026 11:15:17 +0000 Subject: [PATCH 183/192] Remove test cases that aren't useful any more --- mypy/test/test_nativeparse.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index 13777d59cf4b3..f4b56778eb5ba 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -220,8 +220,11 @@ def format_reachable_imports(node: MypyFile) -> list[str]: @unittest.skipUnless(has_nativeparse, "nativeparse not available") -class TestNativeParse(unittest.TestCase): +class TestNativeParserBinaryFormat(unittest.TestCase): def test_trivial_binary_data(self) -> None: + # A quick sanity check to ensure the serialized data looks as expected. Only covers + # a few AST nodes. + def int_enc(n: int) -> int: return (n + 10) << 1 @@ -255,18 +258,6 @@ def locs(start_line: int, start_column: int, end_line: int, end_column: int) -> + [END_TAG, END_TAG] ) - def test_deserialize_hello(self) -> None: - with temp_source("print('hello')") as fnam: - node, _, _ = native_parse(fnam, Options()) - assert isinstance(node, MypyFile) - - def test_deserialize_member_expr(self) -> None: - with temp_source("foo_bar.xyz2") as fnam: - node, _, _ = native_parse(fnam, Options()) - assert isinstance(node, MypyFile) - assert isinstance(node.defs[0], ExpressionStmt) - assert isinstance(node.defs[0].expr, MemberExpr) - @contextlib.contextmanager def temp_source(text: str) -> Iterator[str]: From 9c2ca82410269304c0cf5fad2c557fe0e0a78597 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 21 Feb 2026 11:15:44 +0000 Subject: [PATCH 184/192] Update comments --- test-data/unit/native-parser-imports.test | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test-data/unit/native-parser-imports.test b/test-data/unit/native-parser-imports.test index a5c908b9f6a9c..2f2be69627059 100644 --- a/test-data/unit/native-parser-imports.test +++ b/test-data/unit/native-parser-imports.test @@ -112,7 +112,7 @@ import reachable 6: import reachable [case testUnreachableWithTopLevel] -# Unreachable imports are still marked as top_level correctly +# Unreachable imports together with top_level flag handling def f(): if PY2: import unreachable_func @@ -135,8 +135,7 @@ else: 2: import sys 4: import reachable_version_check -[case testAllBranchesUnreachable] -# When all branches are unreachable, no imports appear +[case testMultipeBranchesUnreachable] if PY2: import unreachable_if elif PY2: From f7926c170cdbe3f264d09c992f12b873de416103 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 21 Feb 2026 11:16:41 +0000 Subject: [PATCH 185/192] Remove type ignore that isn't needed any more --- mypy/plugins/attrs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/plugins/attrs.py b/mypy/plugins/attrs.py index cc3f13ce591ad..47c6ad9f305a2 100644 --- a/mypy/plugins/attrs.py +++ b/mypy/plugins/attrs.py @@ -1107,7 +1107,7 @@ def _meet_fields(types: list[Mapping[str, Type]]) -> Mapping[str, Type]: return { name: ( - get_proper_type(reduce(meet_types, f_types)) # type: ignore[call-arg, misc, unused-ignore] # HAX + get_proper_type(reduce(meet_types, f_types)) if len(f_types) == len(types) else UninhabitedType() ) From f09d7ba5d3674c446581975da66146a8b31af331 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 21 Feb 2026 11:21:03 +0000 Subject: [PATCH 186/192] Fix self check --- mypy/nativeparse.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 651f26d105d20..f97cc1353411f 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -339,7 +339,13 @@ def parse_to_binary_ast( ast_bytes, errors, ignores, import_bytes, is_partial_package = ast_serialize.parse( filename, skip_function_bodies, python_version=options.python_version ) - return ast_bytes, errors, ignores, import_bytes, is_partial_package + return ( + ast_bytes, + cast("list[dict[str, Any]]", errors), + ignores, + import_bytes, + is_partial_package, + ) def read_statement(state: State, data: ReadBuffer) -> Statement: From ca7d19d020a930ecc6b1ae20583185f96ce9932d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 21 Feb 2026 11:41:17 +0000 Subject: [PATCH 187/192] Lint --- mypy/test/test_nativeparse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index f4b56778eb5ba..d0de95d812d6a 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -25,7 +25,7 @@ ) from mypy.config_parser import parse_mypy_comments from mypy.errors import CompileError -from mypy.nodes import ExpressionStmt, MemberExpr, MypyFile +from mypy.nodes import MypyFile from mypy.options import Options from mypy.test.data import DataDrivenTestCase, DataSuite from mypy.test.helpers import assert_string_arrays_equal From 61478c2ca010c6c55b05d03124fdf8dafd458a02 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 21 Feb 2026 11:41:38 +0000 Subject: [PATCH 188/192] Add reachability tests --- test-data/unit/native-parser-imports.test | 403 +++++++++++++++++++++- 1 file changed, 402 insertions(+), 1 deletion(-) diff --git a/test-data/unit/native-parser-imports.test b/test-data/unit/native-parser-imports.test index 2f2be69627059..b8dcac283a525 100644 --- a/test-data/unit/native-parser-imports.test +++ b/test-data/unit/native-parser-imports.test @@ -143,7 +143,7 @@ elif PY2: else: import reachable_else [out] -7: import reachable_else +6: import reachable_else [case testMypyOnlyImport] # Imports in TYPE_CHECKING blocks are mypy_only @@ -330,3 +330,404 @@ if PY2: from reachable import * [out] 3: from reachable import * + +[case testNotPY2] +# not PY2 should be always true +if not PY2: + import always_reachable +[out] +3: import always_reachable + +[case testNotPY3] +# not PY3 should be always false +if not PY3: + import unreachable +import reachable +[out] +4: import reachable + +[case testNotPY3Else] +# not PY3 else branch should be reachable +if not PY3: + import unreachable_if +else: + import reachable_else +[out] +5: import reachable_else + +[case testBoolAndAlwaysTrueWithMypyTrue] +# PY3 and TYPE_CHECKING - both true, result is mypy_true +from typing import TYPE_CHECKING +if PY3 and TYPE_CHECKING: + import mypy_only_import +[out] +2: from typing import TYPE_CHECKING +4: import mypy_only_import [mypy_only] + +[case testBoolAndAlwaysTrueWithUnknown] +# PY3 and unknown - result is unknown, both branches reachable +if PY3 and some_var: + import maybe_reachable +else: + import also_maybe_reachable +[out] +3: import maybe_reachable +5: import also_maybe_reachable + +[case testBoolAndAlwaysFalseWithAnything] +# PY2 and anything - result is always false +if PY2 and some_var: + import unreachable +import reachable +[out] +4: import reachable + +[case testBoolAndMypyFalseWithAlwaysTrue] +# not TYPE_CHECKING and PY3 - result is mypy_false, else is mypy_only +from typing import TYPE_CHECKING +if not TYPE_CHECKING and PY3: + import runtime_only +else: + import mypy_only_import +[out] +2: from typing import TYPE_CHECKING +6: import mypy_only_import [mypy_only] + +[case testBoolAndMypyTrueOnly] +# MYPY and TYPE_CHECKING - both mypy_true, result is mypy_true +from typing import TYPE_CHECKING +if MYPY and TYPE_CHECKING: + import mypy_only_import +[out] +2: from typing import TYPE_CHECKING +4: import mypy_only_import [mypy_only] + +[case testBoolOrAlwaysTrueWithAnything] +# PY3 or anything - result is always true +if PY3 or some_var: + import always_reachable +else: + import unreachable_else +[out] +3: import always_reachable + +[case testBoolOrMypyTrueWithAlwaysFalse] +# TYPE_CHECKING or PY2 - result is mypy_true +from typing import TYPE_CHECKING +if TYPE_CHECKING or PY2: + import mypy_only_import +[out] +2: from typing import TYPE_CHECKING +4: import mypy_only_import [mypy_only] + +[case testBoolOrMypyFalseOnly] +# not MYPY or not TYPE_CHECKING - all mypy_false, result is mypy_false, else is mypy_only +from typing import TYPE_CHECKING +if not MYPY or not TYPE_CHECKING: + import runtime_only +else: + import mypy_only_import +[out] +2: from typing import TYPE_CHECKING +6: import mypy_only_import [mypy_only] + +[case testBoolOrAlwaysFalseWithUnknown] +# PY2 or unknown - result is unknown, both branches reachable +if PY2 or some_var: + import maybe_reachable +else: + import also_maybe_reachable +[out] +3: import maybe_reachable +5: import also_maybe_reachable + +[case testBoolOrAllFalse] +# PY2 or not TYPE_CHECKING - all false values, result is always_false, else is unreachable +from typing import TYPE_CHECKING +if PY2 or not TYPE_CHECKING: + import runtime_only +else: + import mypy_only_import +[out] +2: from typing import TYPE_CHECKING +6: import mypy_only_import + +[case testBoolComplexAndOr] +# Complex: (PY3 and TYPE_CHECKING) or PY2 - evaluates to mypy_true +from typing import TYPE_CHECKING +if (PY3 and TYPE_CHECKING) or PY2: + import mypy_only_import +[out] +2: from typing import TYPE_CHECKING +4: import mypy_only_import [mypy_only] + +[case testBoolMultipleOr] +# Multiple or: PY2 or PY2 or TYPE_CHECKING - result is mypy_true +from typing import TYPE_CHECKING +if PY2 or PY2 or TYPE_CHECKING: + import mypy_only_import +[out] +2: from typing import TYPE_CHECKING +4: import mypy_only_import [mypy_only] + +[case testBoolMultipleAnd] +# Multiple and: PY3 and PY3 and PY3 - result is always_true +if PY3 and PY3 and PY3: + import always_reachable +else: + import unreachable_else +[out] +3: import always_reachable + +[case testVersionInfoIndexMajor] +# sys.version_info[0] == 3 should be always true for Python 3.10 +import sys +if sys.version_info[0] == 3: + import reachable_py3 +else: + import unreachable_not_py3 +[out] +2: import sys +4: import reachable_py3 + +[case testVersionInfoIndexMajorNotEqual] +# sys.version_info[0] == 2 should be always false for Python 3.10 +import sys +if sys.version_info[0] == 2: + import unreachable_py2 +import reachable +[out] +2: import sys +5: import reachable + +[case testVersionInfoIndexMinor] +# sys.version_info[1] >= 10 should be always true for Python 3.10 +import sys +if sys.version_info[1] >= 10: + import reachable_310_plus +else: + import unreachable_pre_310 +[out] +2: import sys +4: import reachable_310_plus + +[case testVersionInfoIndexMinorLess] +# sys.version_info[1] < 10 should be always false for Python 3.10 +import sys +if sys.version_info[1] < 10: + import unreachable_pre_310 +import reachable +[out] +2: import sys +5: import reachable + +[case testVersionInfoIndexMinorGreater] +# sys.version_info[1] > 9 should be always true for Python 3.10 +import sys +if sys.version_info[1] > 9: + import reachable_310_plus +[out] +2: import sys +4: import reachable_310_plus + +[case testVersionInfoIndexLessOrEqual] +# sys.version_info[0] <= 3 should be always true for Python 3.10 +import sys +if sys.version_info[0] <= 3: + import reachable_py3_or_less +[out] +2: import sys +4: import reachable_py3_or_less + +[case testVersionInfoIndexNotEqual] +# sys.version_info[0] != 2 should be always true for Python 3.10 +import sys +if sys.version_info[0] != 2: + import reachable_not_py2 +[out] +2: import sys +4: import reachable_not_py2 + +[case testVersionInfoSliceExplicit] +# sys.version_info[:2] >= (3, 10) should be always true for Python 3.10 +import sys +if sys.version_info[:2] >= (3, 10): + import reachable_310_plus +else: + import unreachable_pre_310 +[out] +2: import sys +4: import reachable_310_plus + +[case testVersionInfoSliceExplicitBothBounds] +# sys.version_info[0:2] >= (3, 10) should be always true for Python 3.10 +import sys +if sys.version_info[0:2] >= (3, 10): + import reachable_310_plus +[out] +2: import sys +4: import reachable_310_plus + +[case testVersionInfoSliceExplicitLess] +# sys.version_info[:2] < (3, 10) should be always false for Python 3.10 +import sys +if sys.version_info[:2] < (3, 10): + import unreachable_pre_310 +import reachable +[out] +2: import sys +5: import reachable + +[case testVersionInfoSliceSingleElement] +# sys.version_info[:1] >= (3,) should be always true for Python 3.10 +import sys +if sys.version_info[:1] >= (3,): + import reachable_py3 +[out] +2: import sys +4: import reachable_py3 + +[case testVersionInfoSliceSingleElementLess] +# sys.version_info[:1] < (3,) should be always false for Python 3.10 +import sys +if sys.version_info[:1] < (3,): + import unreachable_py2 +import reachable +[out] +2: import sys +5: import reachable + +[case testVersionInfoReversedOperands] +# (3, 8) <= sys.version_info should be always true for Python 3.10 +import sys +if (3, 8) <= sys.version_info: + import reachable_38_plus +[out] +2: import sys +4: import reachable_38_plus + +[case testVersionInfoReversedGreater] +# (3, 11) > sys.version_info should be always true for Python 3.10 +import sys +if (3, 11) > sys.version_info: + import reachable_pre_311 +[out] +2: import sys +4: import reachable_pre_311 + +[case testVersionInfoReversedEquals] +# (3, 10) == sys.version_info should be always true for Python 3.10 +import sys +if (3, 10) == sys.version_info: + import reachable_exactly_310 +[out] +2: import sys +4: import reachable_exactly_310 + +[case testVersionInfoEquals] +# sys.version_info == (3, 10) should be always true for Python 3.10 +import sys +if sys.version_info == (3, 10): + import reachable_exactly_310 +else: + import unreachable_not_310 +[out] +2: import sys +4: import reachable_exactly_310 + +[case testVersionInfoNotEquals] +# sys.version_info != (3, 9) should be always true for Python 3.10 +import sys +if sys.version_info != (3, 9): + import reachable_not_39 +[out] +2: import sys +4: import reachable_not_39 + +[case testVersionInfoNotEqualsExact] +# sys.version_info != (3, 10) should be always false for Python 3.10 +import sys +if sys.version_info != (3, 10): + import unreachable_not_310 +import reachable +[out] +2: import sys +5: import reachable + +[case testVersionInfoSliceMinorOnly] +# sys.version_info[1:2] >= (10,) should be always true for Python 3.10 +import sys +if sys.version_info[1:2] >= (10,): + import reachable_minor_10_plus +[out] +2: import sys +4: import reachable_minor_10_plus + +[case testAttributeTypeChecking] +# typing.TYPE_CHECKING should be recognized as mypy_true +import typing +if typing.TYPE_CHECKING: + import mypy_only_import +[out] +2: import typing +4: import mypy_only_import [mypy_only] + +[case testAttributeMYPY] +# foo.MYPY should be recognized as mypy_true +import foo +if foo.MYPY: + import mypy_only_import +[out] +2: import foo +4: import mypy_only_import [mypy_only] + +[case testAttributePY2] +# bar.PY2 should be recognized as always_false +import bar +if bar.PY2: + import unreachable +import reachable +[out] +2: import bar +5: import reachable + +[case testAttributePY3] +# baz.PY3 should be recognized as always_true +import baz +if baz.PY3: + import always_reachable +[out] +2: import baz +4: import always_reachable + +[case testVersionInfoWithBoolAnd] +# Combining version check with TYPE_CHECKING using and +import sys +from typing import TYPE_CHECKING +if sys.version_info >= (3, 8) and TYPE_CHECKING: + import mypy_only_import +[out] +2: import sys +3: from typing import TYPE_CHECKING +5: import mypy_only_import [mypy_only] + +[case testVersionInfoWithBoolOr] +# Combining version check with PY2 using or - should be always_true +import sys +if sys.version_info >= (3, 8) or PY2: + import always_reachable +else: + import unreachable_else +[out] +2: import sys +4: import always_reachable + +[case testComplexBoolVersionMypyUnknown] +# Complex: (MYPY or unknown) evaluates to MYPY_TRUE, and version_info >= (3,8) is ALWAYS_TRUE +# ALWAYS_TRUE and MYPY_TRUE evaluates to MYPY_TRUE +import sys +if sys.version_info >= (3, 8) and (MYPY or unknown_var): + import mypy_only_import +[out] +3: import sys +5: import mypy_only_import [mypy_only] From 99da49b8c337cc9f90ef89c6720c5d7468022042 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 21 Feb 2026 11:44:53 +0000 Subject: [PATCH 189/192] Update docstring --- mypy/nativeparse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index f97cc1353411f..5a85ce67883ae 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -4,8 +4,8 @@ Use a Rust extension to generate a serialized AST, and deserialize the AST directly to a mypy AST. -NOTE: This is heavily work in progress. To use this, you need to manually build the - ast_serialize Rust extension. More information at https://github.com/mypyc/ast_serialize. +NOTE: This is work in progress. To use this, you need to manually build the + ast_serialize Rust extension. See the README at https://github.com/mypyc/ast_serialize. Expected benefits over mypy.fastparse: * No intermediate non-mypyc Python-level AST created, to improve performance From 659ecbf3e7ada4df991f2b85af73edc4cea6cfde Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 21 Feb 2026 11:49:29 +0000 Subject: [PATCH 190/192] Add docstring and reorganize functions --- mypy/nativeparse.py | 214 ++++++++++++++++++++++++-------------------- 1 file changed, 116 insertions(+), 98 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 5a85ce67883ae..ae5eca36eb168 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -192,110 +192,27 @@ def add_error( ) -def expect_end_tag(data: ReadBuffer) -> None: - assert read_tag(data) == END_TAG - - -def expect_tag(data: ReadBuffer, tag: Tag) -> None: - assert read_tag(data) == tag - - -def _read_and_set_import_metadata(data: ReadBuffer, stmt: Import | ImportFrom | ImportAll) -> None: - """Read location and metadata flags from buffer and set them on the import statement. - - Args: - data: Buffer containing serialized data - stmt: Import, ImportFrom, or ImportAll statement to populate with location and metadata - """ - read_loc(data, stmt) - - # Metadata flags as a single integer bitfield - flags = read_int(data) - - # Extract individual flags using bitwise operations - # Bit 0: is_top_level - # Bit 1: is_unreachable - # Bit 2: is_mypy_only - stmt.is_top_level = (flags & 0x01) != 0 - stmt.is_unreachable = (flags & 0x02) != 0 - stmt.is_mypy_only = (flags & 0x04) != 0 - +def native_parse( + filename: str, options: Options, skip_function_bodies: bool = False +) -> tuple[MypyFile, list[dict[str, Any]], TypeIgnores]: + """Parse a Python file using the native Rust-based parser. -def deserialize_imports(import_bytes: bytes) -> list[ImportBase]: - """Deserialize import metadata from bytes into mypy AST nodes. + Uses the ast_serialize Rust extension to parse Python code and deserialize + the resulting AST directly into mypy's native AST representation. Args: - import_bytes: Serialized import metadata from the Rust parser + filename: Path to the Python source file to parse + options: Mypy options affecting parsing behavior (e.g., Python version) + skip_function_bodies: If True, many function and method bodies are omitted from + the AST, useful for parsing stubs or extracting signatures without full + implementation details Returns: - List of Import and ImportFrom AST nodes with location and metadata + A tuple containing: + - MypyFile: The parsed AST as a mypy AST node + - list[dict[str, Any]]: List of parse errors and deserialization errors + - TypeIgnores: List of (line_number, ignored_codes) tuples for type: ignore comments """ - if not import_bytes: - return [] - - data = ReadBuffer(import_bytes) - - expect_tag(data, LIST_GEN) - n_imports = read_int_bare(data) - - imports: list[ImportBase] = [] - - for _ in range(n_imports): - tag = read_tag(data) - - if tag == IMPORT_METADATA: - name = read_str(data) - relative = read_int(data) - - has_asname = read_bool(data) - if has_asname: - asname = read_str(data) - else: - asname = None - - # Note: relative imports are handled via ImportFrom, so relative should be 0 here - stmt = Import([(name, asname)]) - _read_and_set_import_metadata(data, stmt) - imports.append(stmt) - - elif tag == IMPORTFROM_METADATA: - module = read_str(data) - relative = read_int(data) - - expect_tag(data, LIST_GEN) - n_names = read_int_bare(data) - names: list[tuple[str, str | None]] = [] - - for _ in range(n_names): - name = read_str(data) - has_asname = read_bool(data) - if has_asname: - asname = read_str(data) - else: - asname = None - names.append((name, asname)) - - stmt = ImportFrom(module, relative, names) - _read_and_set_import_metadata(data, stmt) - imports.append(stmt) - - elif tag == IMPORTALL_METADATA: - module = read_str(data) - relative = read_int(data) - - stmt = ImportAll(module, relative) - _read_and_set_import_metadata(data, stmt) - imports.append(stmt) - - else: - raise ValueError(f"Unexpected tag in import metadata: {tag}") - - return imports - - -def native_parse( - filename: str, options: Options, skip_function_bodies: bool = False -) -> tuple[MypyFile, list[dict[str, Any]], TypeIgnores]: # If the path is a directory, return empty AST (matching fastparse behavior) # This can happen for packages that only contain .pyc files without source if os.path.isdir(filename): @@ -321,6 +238,14 @@ def native_parse( return node, all_errors, ignores +def expect_end_tag(data: ReadBuffer) -> None: + assert read_tag(data) == END_TAG + + +def expect_tag(data: ReadBuffer, tag: Tag) -> None: + assert read_tag(data) == tag + + def read_statements(state: State, data: ReadBuffer, n: int) -> list[Statement]: defs: list[Statement] = [] old_num_funcs = state.num_funcs @@ -2030,3 +1955,96 @@ def fix_function_overloads(state: State, stmts: list[Statement]) -> list[Stateme elif last_if_stmt is not None: ret.append(last_if_stmt) return ret + + +def deserialize_imports(import_bytes: bytes) -> list[ImportBase]: + """Deserialize import metadata from bytes into mypy AST nodes. + + Args: + import_bytes: Serialized import metadata from the Rust parser + + Returns: + List of Import and ImportFrom AST nodes with location and metadata + """ + if not import_bytes: + return [] + + data = ReadBuffer(import_bytes) + + expect_tag(data, LIST_GEN) + n_imports = read_int_bare(data) + + imports: list[ImportBase] = [] + + for _ in range(n_imports): + tag = read_tag(data) + + if tag == IMPORT_METADATA: + name = read_str(data) + relative = read_int(data) + + has_asname = read_bool(data) + if has_asname: + asname = read_str(data) + else: + asname = None + + # Note: relative imports are handled via ImportFrom, so relative should be 0 here + stmt = Import([(name, asname)]) + _read_and_set_import_metadata(data, stmt) + imports.append(stmt) + + elif tag == IMPORTFROM_METADATA: + module = read_str(data) + relative = read_int(data) + + expect_tag(data, LIST_GEN) + n_names = read_int_bare(data) + names: list[tuple[str, str | None]] = [] + + for _ in range(n_names): + name = read_str(data) + has_asname = read_bool(data) + if has_asname: + asname = read_str(data) + else: + asname = None + names.append((name, asname)) + + stmt = ImportFrom(module, relative, names) + _read_and_set_import_metadata(data, stmt) + imports.append(stmt) + + elif tag == IMPORTALL_METADATA: + module = read_str(data) + relative = read_int(data) + + stmt = ImportAll(module, relative) + _read_and_set_import_metadata(data, stmt) + imports.append(stmt) + + else: + raise ValueError(f"Unexpected tag in import metadata: {tag}") + + return imports + + +def _read_and_set_import_metadata(data: ReadBuffer, stmt: Import | ImportFrom | ImportAll) -> None: + """Read location and metadata flags from buffer and set them on the import statement. + + Args: + data: Buffer containing serialized data + stmt: Import, ImportFrom, or ImportAll statement to populate with location and metadata + """ + read_loc(data, stmt) + + # Metadata flags as a single integer bitfield + flags = read_int(data) + + # Extract individual flags using bitwise operations + # Bit 0: is_top_level + # Bit 1: is_unreachable + # Bit 2: is_mypy_only + stmt.is_top_level = (flags & 0x01) != 0 + stmt.is_unreachable = (flags & 0x02) != 0 + stmt.is_mypy_only = (flags & 0x04) != 0 From 8b318555a4c5f399de9cc2077215e2c45c33605a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 21 Feb 2026 11:54:55 +0000 Subject: [PATCH 191/192] Move some type read functions to be close to each other --- mypy/nativeparse.py | 264 ++++++++++++++++++++++---------------------- 1 file changed, 132 insertions(+), 132 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index ae5eca36eb168..7b6dec8cde172 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -980,6 +980,138 @@ def read_type(state: State, data: ReadBuffer) -> Type: assert False, tag +def stringify_type_name(typ: Type) -> str | None: + """Extract qualified name from a type (for Arg constructor detection).""" + if isinstance(typ, UnboundType): + return typ.name + return None + + +def extract_arg_name(typ: Type) -> str | None: + """Extract argument name from a type (for Arg name parameter).""" + if isinstance(typ, RawExpressionType) and typ.base_type_name == "builtins.str": + return typ.literal_value # type: ignore[return-value] + elif isinstance(typ, UnboundType): + # String literals in type context are parsed as UnboundType (forward references) + # For Arg names, these are typically simple names without dots + if typ.name == "None": + return None + # Return the name as-is (it's the argument name) + return typ.name + return None # Invalid, but let validation handle it + + +def read_call_type(state: State, data: ReadBuffer) -> Type: + """Read Call in type context - check if it's an Arg/DefaultArg/VarArg/KwArg constructor. + + This performs validation and error reporting similar to mypy/fastparse.py. + """ + callee_type = read_type(state, data) + + # Read positional arguments + expect_tag(data, LIST_GEN) + n_args = read_int_bare(data) + args = [read_type(state, data) for _ in range(n_args)] + + # Read keyword arguments + expect_tag(data, LIST_GEN) + n_kwargs = read_int_bare(data) + kwargs = [] + for _ in range(n_kwargs): + tag_kw = read_tag(data) + if tag_kw == LITERAL_NONE: + kw_name = None + elif tag_kw == LITERAL_STR: + kw_name = read_str_bare(data) + else: + assert False, f"Unexpected tag for keyword name: {tag_kw}" + kw_value = read_type(state, data) + kwargs.append((kw_name, kw_value)) + + # Try to detect Arg/DefaultArg/VarArg/KwArg pattern + constructor = stringify_type_name(callee_type) + + # We'll read location before processing errors so we can report them correctly + invalid = AnyType(TypeOfAny.from_error) + read_loc(data, invalid) + expect_end_tag(data) + + if not constructor: + # ARG_CONSTRUCTOR_NAME_EXPECTED + state.add_error( + message_registry.ARG_CONSTRUCTOR_NAME_EXPECTED.value, + invalid.line, + invalid.column, + blocker=True, + code="misc", + ) + return invalid + + # Extract type and name from arguments + name: str | None = None + name_set_from_positional = False + default_type = AnyType(TypeOfAny.special_form) + typ: Type = default_type + typ_set_from_positional = False + + # Process positional arguments + for i, arg in enumerate(args): + if i == 0: + typ = arg + typ_set_from_positional = True + elif i == 1: + name = extract_arg_name(arg) + name_set_from_positional = True + else: + # ARG_CONSTRUCTOR_TOO_MANY_ARGS + state.add_error( + message_registry.ARG_CONSTRUCTOR_TOO_MANY_ARGS.value, + invalid.line, + invalid.column, + blocker=True, + code="misc", + ) + + # Process keyword arguments + for kw_name, kw_value in kwargs: + if kw_name == "name": + # MULTIPLE_VALUES_FOR_NAME_KWARG + if name is not None and name_set_from_positional: + state.add_error( + message_registry.MULTIPLE_VALUES_FOR_NAME_KWARG.format(constructor).value, + invalid.line, + invalid.column, + blocker=True, + code="misc", + ) + name = extract_arg_name(kw_value) + elif kw_name == "type": + # MULTIPLE_VALUES_FOR_TYPE_KWARG + if typ is not default_type and typ_set_from_positional: + state.add_error( + message_registry.MULTIPLE_VALUES_FOR_TYPE_KWARG.format(constructor).value, + invalid.line, + invalid.column, + blocker=True, + code="misc", + ) + typ = kw_value + else: + # ARG_CONSTRUCTOR_UNEXPECTED_ARG + state.add_error( + message_registry.ARG_CONSTRUCTOR_UNEXPECTED_ARG.format(kw_name).value, + invalid.line, + invalid.column, + blocker=True, + code="misc", + ) + + # Create CallableArgument + call_arg = CallableArgument(typ, name, constructor) + set_line_column_range(call_arg, invalid) + return call_arg + + def read_pattern(state: State, data: ReadBuffer) -> Pattern: """Read a pattern node from the buffer.""" tag = read_tag(data) @@ -1553,138 +1685,6 @@ def read_loc(data: ReadBuffer, node: Context) -> None: node.end_column = column + read_int_bare(data) -def stringify_type_name(typ: Type) -> str | None: - """Extract qualified name from a type (for Arg constructor detection).""" - if isinstance(typ, UnboundType): - return typ.name - return None - - -def extract_arg_name(typ: Type) -> str | None: - """Extract argument name from a type (for Arg name parameter).""" - if isinstance(typ, RawExpressionType) and typ.base_type_name == "builtins.str": - return typ.literal_value # type: ignore[return-value] - elif isinstance(typ, UnboundType): - # String literals in type context are parsed as UnboundType (forward references) - # For Arg names, these are typically simple names without dots - if typ.name == "None": - return None - # Return the name as-is (it's the argument name) - return typ.name - return None # Invalid, but let validation handle it - - -def read_call_type(state: State, data: ReadBuffer) -> Type: - """Read Call in type context - check if it's an Arg/DefaultArg/VarArg/KwArg constructor. - - This performs validation and error reporting similar to mypy/fastparse.py. - """ - callee_type = read_type(state, data) - - # Read positional arguments - expect_tag(data, LIST_GEN) - n_args = read_int_bare(data) - args = [read_type(state, data) for _ in range(n_args)] - - # Read keyword arguments - expect_tag(data, LIST_GEN) - n_kwargs = read_int_bare(data) - kwargs = [] - for _ in range(n_kwargs): - tag_kw = read_tag(data) - if tag_kw == LITERAL_NONE: - kw_name = None - elif tag_kw == LITERAL_STR: - kw_name = read_str_bare(data) - else: - assert False, f"Unexpected tag for keyword name: {tag_kw}" - kw_value = read_type(state, data) - kwargs.append((kw_name, kw_value)) - - # Try to detect Arg/DefaultArg/VarArg/KwArg pattern - constructor = stringify_type_name(callee_type) - - # We'll read location before processing errors so we can report them correctly - invalid = AnyType(TypeOfAny.from_error) - read_loc(data, invalid) - expect_end_tag(data) - - if not constructor: - # ARG_CONSTRUCTOR_NAME_EXPECTED - state.add_error( - message_registry.ARG_CONSTRUCTOR_NAME_EXPECTED.value, - invalid.line, - invalid.column, - blocker=True, - code="misc", - ) - return invalid - - # Extract type and name from arguments - name: str | None = None - name_set_from_positional = False - default_type = AnyType(TypeOfAny.special_form) - typ: Type = default_type - typ_set_from_positional = False - - # Process positional arguments - for i, arg in enumerate(args): - if i == 0: - typ = arg - typ_set_from_positional = True - elif i == 1: - name = extract_arg_name(arg) - name_set_from_positional = True - else: - # ARG_CONSTRUCTOR_TOO_MANY_ARGS - state.add_error( - message_registry.ARG_CONSTRUCTOR_TOO_MANY_ARGS.value, - invalid.line, - invalid.column, - blocker=True, - code="misc", - ) - - # Process keyword arguments - for kw_name, kw_value in kwargs: - if kw_name == "name": - # MULTIPLE_VALUES_FOR_NAME_KWARG - if name is not None and name_set_from_positional: - state.add_error( - message_registry.MULTIPLE_VALUES_FOR_NAME_KWARG.format(constructor).value, - invalid.line, - invalid.column, - blocker=True, - code="misc", - ) - name = extract_arg_name(kw_value) - elif kw_name == "type": - # MULTIPLE_VALUES_FOR_TYPE_KWARG - if typ is not default_type and typ_set_from_positional: - state.add_error( - message_registry.MULTIPLE_VALUES_FOR_TYPE_KWARG.format(constructor).value, - invalid.line, - invalid.column, - blocker=True, - code="misc", - ) - typ = kw_value - else: - # ARG_CONSTRUCTOR_UNEXPECTED_ARG - state.add_error( - message_registry.ARG_CONSTRUCTOR_UNEXPECTED_ARG.format(kw_name).value, - invalid.line, - invalid.column, - blocker=True, - code="misc", - ) - - # Create CallableArgument - call_arg = CallableArgument(typ, name, constructor) - set_line_column_range(call_arg, invalid) - return call_arg - - def strip_contents_from_if_stmt(stmt: IfStmt) -> None: """Remove contents from IfStmt. From 0017e18e4f98f1de6eb959f9d535a0bbb68f8e66 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 21 Feb 2026 12:02:44 +0000 Subject: [PATCH 192/192] Clean up test code --- mypy/test/test_nativeparse.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index d0de95d812d6a..71f9c06892dd2 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -90,11 +90,7 @@ def test_parser(testcase: DataDrivenTestCase) -> None: try: with temp_source(source) as fnam: - try: - node, errors, type_ignores = native_parse(fnam, options, skip_function_bodies) - except ValueError as e: - print(f"Parse failed: {e}") - assert False + node, errors, type_ignores = native_parse(fnam, options, skip_function_bodies) node.path = "main" a = node.str_with_options(options).split("\n") a = [format_error(err) for err in errors] + a @@ -132,11 +128,7 @@ def test_parser_imports(testcase: DataDrivenTestCase) -> None: try: with temp_source(source) as fnam: - try: - node, errors, type_ignores = native_parse(fnam, options) - except ValueError as e: - print(f"Parse failed: {e}") - assert False + node, errors, type_ignores = native_parse(fnam, options) # Extract and format reachable imports a = format_reachable_imports(node)